Skip to content

Instantly share code, notes, and snippets.

@luser

luser/Untitled.ipynb

Created February 1, 2017 13:17
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save luser/6708c7f165d44b7010cbd4c70f45ac22 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save luser/6708c7f165d44b7010cbd4c70f45ac22 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
Untitled.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from pyspark.sql import functions\n",
"from datetime import datetime, timedelta"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"num_days = 1\n",
"now = datetime.utcnow()\n",
"days = [now - timedelta(days=i) for i in range(1, num_days + 1)]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"dataset = SQLContext(sc).read.load(['s3://telemetry-parquet/socorro_crash/v1/crash_date=' + day.strftime('%Y%m%d') for day in days], 'parquet')"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"crashes = dataset\\\n",
".filter(dataset['product'] == 'Firefox')\\\n",
".select('signature', functions.explode(dataset['json_dump']['crashing_thread']['frames']).alias('frame')).rdd\\\n",
".filter(lambda r: r['frame']['file'] and 'media/webrtc' in r['frame']['file'])\\\n",
".groupBy(lambda r: r['signature'])\\\n",
".collect()"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"90"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(crashes)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"[u'mozilla::dom::RTCCertificateBinding::Wrap',\n",
" u'mozilla::dom::CallbackObject::CallSetup::CallSetup',\n",
" u'WebRtcAgc_InitDigital',\n",
" u'msvcr120.dll@0xf20c | webrtc::I420VideoFrame::CreateFrame',\n",
" u'js::MarkAtoms',\n",
" u'nr_ice_component_consent_timer_cb',\n",
" u'moz_malloc_usable_size',\n",
" u'CStdMarshal::DisconnectSrvIPIDs',\n",
" u'mozilla::widget::NativeKey::GetFollowingCharMessage',\n",
" u'OOM | large | mozalloc_abort | mozalloc_handle_oom | moz_xmalloc | mozilla::MakeUnique<T>',\n",
" u'_wcsicmp',\n",
" u'mozilla::PeerConnectionImpl::Initialize',\n",
" u'moz_abort | pages_commit',\n",
" u'js::InternalCallOrConstruct',\n",
" u'vflydcam.ax@0x9650',\n",
" u'js::TenuringTracer::traverse<T>',\n",
" u'webrtc::DesktopAndCursorComposer::OnCaptureCompleted',\n",
" u'GraphWalker<T>::DoWalk',\n",
" u'nsNPAPIPluginInstance::Stop',\n",
" u'webrtc::AudioDeviceWindowsCore::InitCaptureThreadPriority',\n",
" u'RuleHash_ClassCSMatchEntry',\n",
" u'AudioSrvBinding',\n",
" u'hang | NtUserPeekMessage | NtUserPeekMessage | NtUserPeekMessage | mozilla::ipc::NeuteredWindowRegion::PumpOnce',\n",
" u'nsCycleCollector::ScanWhiteNodes',\n",
" u'nsACString_internal::Assign | nsACString_internal::Assign | mozilla::gmp::GetContentParentFromDone::Done',\n",
" u'nsIDocument::WarnOnceAbout',\n",
" u'webrtc::videocapturemodule::DeviceInfoDS::CreateCapabilityMap',\n",
" u'Blt32to32BA_SSE2_Simple',\n",
" u'js::jit::DoGetPropFallback',\n",
" u'js::MapObject::mark',\n",
" u'hang | npswf32_15_0_0_246.dll@0x6f3840',\n",
" u'Abort | Could not initialize mScreenReferenceDrawTarget | mozalloc_abort | NS_DebugBreak | gfxPlatform::Init',\n",
" u'IPCError-content | (msgtype=0x820017,name=???) Route error: message sent to unknown actor ID',\n",
" u'nsTArray_Impl<T>::DestructRange | nsTArray_Impl<T>::RemoveElementsAt | mozilla::MediaEngineSource::Deallocate',\n",
" u'mozilla::net::nsProtocolProxyService::LoadHostFilters',\n",
" u\"`anonymous namespace''::CloseEventRunnable::WorkerRun\",\n",
" u'webrtc::ThreadWindows::~ThreadWindows',\n",
" u'memcpy | webrtc::I420VideoFrame::CreateFrame',\n",
" u'qipcap.dll@0x31100',\n",
" u'mozilla::WebrtcVideoConduit::SendPacket',\n",
" u'IPCError-browser | PDocAccessibleParent::AddChildDoc binding to nonexistant proxy!',\n",
" u'CPerChannelDbControl::GetDetails',\n",
" u'nsCOMPtr_base::assign_from_gs_contractid_with_error | mozilla::DataChannelConnection::Init',\n",
" u'IsAboutToBeFinalizedInternal<T>',\n",
" u'memset | vp8_post_proc_frame',\n",
" u'init_frame',\n",
" u'mozilla::EverySecondTelemetryCallback_s',\n",
" u'hang | PeekMessageW',\n",
" u'CVideoControlInterfaceHandler::GetFrameRateList',\n",
" u'nsCSSStyleSheet::GetOwningDocument',\n",
" u'webrtc::AudioDeviceWindowsCore::DoCaptureThread',\n",
" u'IPCError-browser | ShutDownKill',\n",
" u'_moz_cairo_scaled_font_get_font_options',\n",
" u'mozilla::WebrtcGmpVideoDecoder::ReceivedDecodedReferenceFrame',\n",
" u'webrtc::ProcessThreadImpl::Process',\n",
" u'mozilla::DataChannelConnection::Open',\n",
" u'std::_Tree<T>::_Lbound',\n",
" u'RGB24ToARGBRow_SSSE3',\n",
" u'wsck32.dll@0x27263',\n",
" u'OOM | large | mozalloc_abort | mozalloc_handle_oom | moz_xmalloc | mozilla::camera::DeliverFrameRunnable::DeliverFrameRunnable',\n",
" u'js::GCMarker::lazilyMarkChildren',\n",
" u'webrtc::Trace::SetTraceFile',\n",
" u'webrtc::DelayManager::UpdateHistogram',\n",
" u'nsRuleNode::GetStyleDisplay<T>',\n",
" u'sipcc::EverySecondTelemetryCallback_s',\n",
" u'OOM | large | mozalloc_abort | mozalloc_handle_oom | moz_xrealloc | nsTArray_base<T>::EnsureCapacity<T> | nsTArray_Impl<T>::AppendElements<T> | mozilla::MediaSegmentBase<T>::AppendFromInternal',\n",
" u'nsRefPtr<T>::~nsRefPtr<T> | NativeInterfaceSweeper',\n",
" u'general_composite_rect',\n",
" u'lavsplitter.ax@0x28c11',\n",
" u'CSFLog',\n",
" u'OOM | small',\n",
" u'JSScript::traceChildren',\n",
" u'sdp_validate_sdp',\n",
" u'nsRefPtr<T>::assign_with_AddRef | mozilla::net::nsHttpChannel::OnStopRequest',\n",
" u'mozilla::dom::CreateInterfaceObjects',\n",
" u'_PeekMessage',\n",
" u'CSessMixer::GetSimpleVolume',\n",
" u'js::IdIsIndex',\n",
" u'zzz_AsmCodeRange_Begin | RtlpWaitOnCriticalSection | webrtc::CriticalSectionWindows::Leave',\n",
" u'webrtc::VCMEncodedFrameCallback::Encoded',\n",
" u\"mozilla::dom::indexedDB::`anonymous namespace''::DatabaseConnection::GetCachedStatement\",\n",
" u'mozalloc_abort | NS_DebugBreak | mozilla::ipc::FatalError | mozilla::dom::PContentChild::SendPWebrtcGlobalConstructor',\n",
" u'CNativeAudioParse::Init',\n",
" u'js::ConstraintTypeSet::sweep',\n",
" u'mozilla::SrtpFlow::ProtectRtcp',\n",
" u'hang | NtUserMessageCall | RealDefWindowProcW',\n",
" u'OOM | large | js::TraceRootRange<T>',\n",
" u'PLDHashTable::Add | cft1st',\n",
" u'OOM | large | mozalloc_abort | mozalloc_handle_oom | moz_xmalloc | mozilla::layers::BufferRecycleBin::GetBuffer',\n",
" u'Interpret']"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"[c[0] for c in crashes]"
]
},
{
"cell_type": "raw",
"metadata": {},
"source": []
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python [conda root]",
"language": "python",
"name": "conda-root-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.12"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment