v4_beta_from_ES_with_accumulators.ipynb

{"nbformat_minor": 0, "cells": [{"execution_count": 1, "cell_type": "code", "source": "from moztelemetry import get_records", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 8, "cell_type": "code", "source": "summary = get_records(sc, 'telemetry-executive-summary',\n                      submissionDate=(\"20150808\",\"20150902\"))", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 5, "cell_type": "code", "source": "# The latest v2 sample has activity through August 30\nimport time, datetime as DT\naug31 = DT.datetime(2015, 8, 31, 0, 0, 0)\nACTIVITY_CUTOFF = time.mktime(aug31.timetuple()) * (10**9)\n\ndef make_walker():\n    counters = {\n        'beta_41': sc.accumulator(0),\n        'crashes': sc.accumulator(0),\n        'yahoo': sc.accumulator(0),\n        'hours': sc.accumulator(0.0),\n        'default': sc.accumulator(0),\n        'parsing_errors': sc.accumulator(0)\n        }\n    def walk(d):\n        d = d.get('meta', d)\n        if d['version'] != '41.0':\n            return False\n        if d['channel'] != 'beta':\n            return False\n        if d['activityTimestamp'] > ACTIVITY_CUTOFF:\n            return False\n        counters['beta_41'].add(1)\n        if d['docType'] == 'crash':\n            counters['crashes'].add(1)\n            return True\n        counters['yahoo'].add(d['yahoo'])\n        counters['hours'].add(d['hours'])\n        counters['default'].add(int(d['default']))\n        return True\n    return walk, counters", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 9, "cell_type": "code", "source": "import sys\nwalk, counters = make_walker()\nprint DT.datetime.now()\nsys.stdout.flush()\nbeta41 = summary.map(walk).collect()\nprint DT.datetime.now()\nfor name, obj in counters.items():\n    print name, obj.value", "outputs": [{"output_type": "stream", "name": "stdout", "text": "2015-09-08 15:34:54.231340\n2015-09-08 17:01:50.545621\ncrashes 1141738\ndefault 82521247\nyahoo 2417683\nhours 192192829.499\nparsing_errors 0\nbeta_41 129812397\n"}], "metadata": {"collapsed": false, "trusted": true}}], "nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 2", "name": "python2", "language": "python"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "version": "2.7.9", "name": "python", "file_extension": ".py", "pygments_lexer": "ipython2", "codemirror_mode": {"version": 2, "name": "ipython"}}}}