Skip to content

Instantly share code, notes, and snippets.

@SamPenrose

SamPenrose/executive summary choropleth.ipynb

Last active December 8, 2015 19:21
Show Gist options
  • Save SamPenrose/77aa362ba1d40ec77257 to your computer and use it in GitHub Desktop.
Save SamPenrose/77aa362ba1d40ec77257 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
executive summary choropleth.ipynb
{"nbformat_minor": 0, "cells": [{"execution_count": 1, "cell_type": "code", "source": "os.system('pip install folium')\nimport folium", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 3, "cell_type": "code", "source": "from pyspark.sql import SQLContext\nfrom pyspark.sql.types import *\n\ndataset = sqlContext.load(\"s3://telemetry-parquet/executive-stream/telemetry-executive-summary-3/submissionDateS3=20151130\", \"parquet\")", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 4, "cell_type": "code", "source": "dataset.count()", "outputs": [{"execution_count": 4, "output_type": "execute_result", "data": {"text/plain": "404338319L"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 9, "cell_type": "code", "source": "countries = dataset.groupBy('country').count().collect()", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 17, "cell_type": "code", "source": "# https://raw.githubusercontent.com/python-visualization/folium/master/examples/world-countries.json\n# edited to include a 2-letter country tag that matches what's in the ES\nimport ujson as json\nimport urllib\nu = urllib.urlopen('https://raw.githubusercontent.com/python-visualization/folium/master/examples/world-countries.json')\nmap_geo_data = json.load(u)\nfor d in map_geo_data['features']:\n d['id2'] = d['id'][:2]\nwith open('world-countries-2.json', 'w') as f:\n json.dump(map_geo_data, f)", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 13, "cell_type": "code", "source": "import numpy, pandas as pd\ncdf = pd.DataFrame([r.asDict() for r in countries])", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 22, "cell_type": "code", "source": "countries.sort(key=lambda r: r.count)\ncolumns = ['country', 'count']\nthreshold_scale = numpy.linspace(countries[0].count, countries[-1].count, 6, dtype=int).tolist()\nmapa = folium.Map(tiles=\"Mapbox Bright\", zoom_start=3)\nmapa.geo_json(geo_path='world-countries-2.json', data=cdf, columns=columns, fill_color='YlGn',\n threshold_scale=threshold_scale, key_on='feature.id2')\nmapa", "outputs": [{"execution_count": 22, "output_type": "execute_result", "data": {"text/plain": "<folium.folium.Map at 0x7ffb14183ad0>", "text/html": "<iframe srcdoc=\"<!DOCTYPE html>\n<head>\n <meta http-equiv=&quot;content-type&quot; content=&quot;text/html; charset=UTF-8&quot; />\n <link rel=&quot;stylesheet&quot; href=&quot;https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.3/leaflet.css&quot; />\n <script src=&quot;https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.3/leaflet.js&quot;></script>\n <script src=&quot;https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js&quot; charset=&quot;utf-8&quot;></script>\n <script src=&quot;https://cdnjs.cloudflare.com/ajax/libs/queue-async/1.0.7/queue.min.js&quot;></script>\n\n \n \n \n \n\n <style>\n\n html, body {\n width: 100%;\n height: 100%;\n margin: 0;\n padding: 0;\n }\n\n .legend {\n padding: 0px 0px;\n font: 10px sans-serif;\n background: white;\n background: rgba(255,255,255,0.8);\n box-shadow: 0 0 15px rgba(0,0,0,0.2);\n border-radius: 5px;\n }\n\n .key path {\n display: none;\n }\n\n </style>\n</head>\n\n<body>\n\n <div id=&quot;map&quot; style=&quot;width: 100%; height: 100%&quot;></div>\n\n <script>\n\n queue()\n .defer(function(callback){callback(null, JSON.parse('[{&quot;BD&quot;: 2054650, &quot;BE&quot;: 3228503, &quot;BF&quot;: 71764, &quot;BG&quot;: 2179362, &quot;BA&quot;: 626974, &quot;BB&quot;: 28352, &quot;WF&quot;: 1548, &quot;BL&quot;: 5, &quot;BM&quot;: 7905, &quot;BN&quot;: 47292, &quot;BO&quot;: 445625, &quot;BH&quot;: 56042, &quot;BI&quot;: 28220, &quot;BJ&quot;: 54509, &quot;BT&quot;: 18887, &quot;JM&quot;: 91614, &quot;BW&quot;: 59655, &quot;WS&quot;: 2678, &quot;BQ&quot;: 2367, &quot;BR&quot;: 18178398, &quot;BS&quot;: 21019, &quot;JE&quot;: 12187, &quot;BY&quot;: 1000470, &quot;BZ&quot;: 25720, &quot;LV&quot;: 801151, &quot;RW&quot;: 103953, &quot;RS&quot;: 1745287, &quot;LT&quot;: 1291108, &quot;RE&quot;: 235855, &quot;LU&quot;: 255966, &quot;TJ&quot;: 13616, &quot;RO&quot;: 2882883, &quot;TK&quot;: 2, &quot;GW&quot;: 2893, &quot;GU&quot;: 17551, &quot;GT&quot;: 284383, &quot;GR&quot;: 3591144, &quot;GQ&quot;: 3353, &quot;GP&quot;: 73294, &quot;JP&quot;: 7284242, &quot;GY&quot;: 28903, &quot;GG&quot;: 8151, &quot;GF&quot;: 39117, &quot;GE&quot;: 258529, &quot;GD&quot;: 6019, &quot;GB&quot;: 9815089, &quot;GA&quot;: 36540, &quot;GN&quot;: 28458, &quot;GM&quot;: 20970, &quot;GL&quot;: 5390, &quot;KW&quot;: 107461, &quot;GI&quot;: 6026, &quot;GH&quot;: 344110, &quot;OM&quot;: 90215, &quot;IL&quot;: 549003, &quot;JO&quot;: 163595, &quot;HR&quot;: 1251285, &quot;HT&quot;: 32681, &quot;HU&quot;: 4183152, &quot;HK&quot;: 869743, &quot;HN&quot;: 132796, &quot;KR&quot;: 456602, &quot;CR&quot;: 354955, &quot;VE&quot;: 2759655, &quot;PR&quot;: 214594, &quot;PS&quot;: 334302, &quot;PW&quot;: 981, &quot;PT&quot;: 1596399, &quot;KN&quot;: 3837, &quot;VG&quot;: 2363, &quot;IQ&quot;: 510370, &quot;PA&quot;: 154692, &quot;PF&quot;: 49330, &quot;PG&quot;: 12638, &quot;PE&quot;: 1161249, &quot;PK&quot;: 1700292, &quot;PH&quot;: 2310127, &quot;PN&quot;: 3, &quot;PL&quot;: 19083203, &quot;PM&quot;: 4532, &quot;ZM&quot;: 85466, &quot;RU&quot;: 17451840, &quot;EE&quot;: 526629, &quot;EG&quot;: 4994451, &quot;ZA&quot;: 1495490, &quot;EC&quot;: 2291905, &quot;IT&quot;: 13051670, &quot;VN&quot;: 3229957, &quot;KZ&quot;: 1397153, &quot;ET&quot;: 263054, &quot;ZW&quot;: 128897, &quot;KY&quot;: 9441, &quot;ES&quot;: 8685366, &quot;ER&quot;: 1942, &quot;ME&quot;: 96271, &quot;MD&quot;: 339566, &quot;MG&quot;: 167703, &quot;MF&quot;: 1994, &quot;MA&quot;: 1222729, &quot;MC&quot;: 16827, &quot;UZ&quot;: 103138, &quot;MM&quot;: 189478, &quot;ML&quot;: 56220, &quot;MO&quot;: 40162, &quot;MN&quot;: 396697, &quot;MH&quot;: 3043, &quot;MK&quot;: 342921, &quot;MU&quot;: 137396, &quot;MT&quot;: 64397, &quot;MW&quot;: 33583, &quot;MV&quot;: 46409, &quot;MQ&quot;: 56420, &quot;MP&quot;: 4642, &quot;MS&quot;: 195, &quot;MR&quot;: 26670, &quot;IM&quot;: 13649, &quot;UG&quot;: 197539, &quot;MY&quot;: 2177671, &quot;MX&quot;: 5099746, &quot;MZ&quot;: 74680, &quot;FR&quot;: 27617202, &quot;IO&quot;: 645, &quot;FI&quot;: 2769993, &quot;FJ&quot;: 58308, &quot;FK&quot;: 383, &quot;FM&quot;: 5786, &quot;FO&quot;: 11965, &quot;NI&quot;: 150601, &quot;NL&quot;: 4310689, &quot;NO&quot;: 1078271, &quot;NA&quot;: 57958, &quot;VU&quot;: 4966, &quot;NC&quot;: 75095, &quot;NE&quot;: 25586, &quot;NF&quot;: 5, &quot;NG&quot;: 827538, &quot;NZ&quot;: 852959, &quot;NP&quot;: 330411, &quot;NR&quot;: 324, &quot;NU&quot;: 40, &quot;CK&quot;: 1646, &quot;CI&quot;: 255145, &quot;CH&quot;: 3036279, &quot;CO&quot;: 1888221, &quot;CN&quot;: 10058044, &quot;CM&quot;: 213998, &quot;CL&quot;: 1319480, &quot;CC&quot;: 3, &quot;CA&quot;: 8076629, &quot;LB&quot;: 195588, &quot;CG&quot;: 24553, &quot;CF&quot;: 2344, &quot;CD&quot;: 60887, &quot;CZ&quot;: 4176570, &quot;CY&quot;: 182742, &quot;CX&quot;: 10, &quot;UY&quot;: 414786, &quot;PY&quot;: 240182, &quot;CW&quot;: 14250, &quot;CV&quot;: 14410, &quot;CU&quot;: 65185, &quot;SZ&quot;: 9536, &quot;SY&quot;: 234407, &quot;SX&quot;: 4341, &quot;KG&quot;: 61738, &quot;KE&quot;: 859212, &quot;SS&quot;: 6346, &quot;SR&quot;: 23313, &quot;KI&quot;: 741, &quot;KH&quot;: 361972, &quot;SV&quot;: 243632, &quot;KM&quot;: 4054, &quot;ST&quot;: 2188, &quot;SK&quot;: 1740217, &quot;SJ&quot;: 6, &quot;SI&quot;: 943810, &quot;KP&quot;: 334, &quot;SO&quot;: 23991, &quot;SN&quot;: 216188, &quot;SM&quot;: 9222, &quot;SL&quot;: 13954, &quot;SC&quot;: 10499, &quot;SB&quot;: 2644, &quot;SA&quot;: 979030, &quot;SG&quot;: 979943, &quot;SE&quot;: 1998135, &quot;SD&quot;: 164655, &quot;DO&quot;: 250851, &quot;DM&quot;: 4678, &quot;DJ&quot;: 16315, &quot;DK&quot;: 1120380, &quot;DE&quot;: 40933979, &quot;YE&quot;: 61579, &quot;Other&quot;: 393658, &quot;DZ&quot;: 2261662, &quot;US&quot;: 61763898, &quot;YT&quot;: 18115, &quot;TZ&quot;: 224451, &quot;LC&quot;: 9550, &quot;LA&quot;: 67895, &quot;TV&quot;: 172, &quot;TW&quot;: 1832289, &quot;TT&quot;: 89211, &quot;TR&quot;: 3566119, &quot;LK&quot;: 559257, &quot;LI&quot;: 16419, &quot;TN&quot;: 459905, &quot;TO&quot;: 3077, &quot;TL&quot;: 13004, &quot;TM&quot;: 4496, &quot;LR&quot;: 5651, &quot;LS&quot;: 8504, &quot;TH&quot;: 2107862, &quot;TG&quot;: 51106, &quot;TD&quot;: 1818, &quot;TC&quot;: 2480, &quot;LY&quot;: 114683, &quot;VA&quot;: 2288, &quot;VC&quot;: 4254, &quot;AE&quot;: 582969, &quot;AD&quot;: 13667, &quot;AG&quot;: 7872, &quot;AF&quot;: 61900, &quot;AI&quot;: 781, &quot;VI&quot;: 15614, &quot;IS&quot;: 105121, &quot;IR&quot;: 4526109, &quot;AM&quot;: 211656, &quot;AL&quot;: 191560, &quot;AO&quot;: 51656, &quot;AS&quot;: 3286, &quot;AR&quot;: 3963509, &quot;AU&quot;: 4497309, &quot;AT&quot;: 3353821, &quot;AW&quot;: 9878, &quot;IN&quot;: 15732508, &quot;AX&quot;: 8571, &quot;AZ&quot;: 110656, &quot;IE&quot;: 970957, &quot;ID&quot;: 10727092, &quot;UA&quot;: 5730368, &quot;QA&quot;: 137277}]'))})\n .defer(d3.json, 'world-countries-2.json')\n .await(makeMap)\n\n function onEachFeature(feature, layer) {\n // does this feature have a property named popupContent?\n if (feature.properties && feature.properties.popupContent) {\n layer.bindPopup(feature.properties.popupContent);\n }\n };\n\n function makeMap(error, data_1,gjson_1) {\n\n \n\n \n\n function matchKey(datapoint, key_variable){\n if (typeof key_variable[0][datapoint] === 'undefined') {\n return null;\n }\n else {\n return parseFloat(key_variable[0][datapoint]);\n };\n };\n\n \n var color = d3.scale.threshold()\n .domain([13667, 36713, 59759, 82805, 105851, 128897])\n .range(['#FFFFCC', '#D9F0A3', '#ADDD8E', '#78C679', '#41AB5D', '#238443', '#005A32']);\n \n\n var map = L.map('map').setView([0, 0], 1);\n\n L.tileLayer('https://{s}.tiles.mapbox.com/v3/mapbox.world-bright/{z}/{x}/{y}.png', {\n maxZoom: 18,\n minZoom: 1,\n attribution: 'Map tiles by <a href=&quot;http://www.mapbox.com/m&quot;>Mapbox</a> Data by <a href=&quot;http://openstreetmap.org&quot;>OpenStreetMap</a>, under <a href=&quot;http://creativecommons.org/licenses/by-sa/3.0&quot;>CC BY SA</a>.'\n }).addTo(map);\n\n \n function style_1(feature) {\n return {\n fillColor: color(matchKey(feature.id2, data_1)),\n weight: 1,\n opacity: 1,\n color: 'black',\n fillOpacity: 0.6\n };\n}\n \n\n \n gJson_layer_1 = L.geoJson(gjson_1, {style: style_1,onEachFeature: onEachFeature}).addTo(map)\n \n\n \n var legend = L.control({position: 'topright'});\n\n legend.onAdd = function (map) {var div = L.DomUtil.create('div', 'legend'); return div};\n\n legend.addTo(map);\n\n var x = d3.scale.linear()\n .domain([0, 141786])\n .range([0, 400]);\n\n var xAxis = d3.svg.axis()\n .scale(x)\n .orient(&quot;top&quot;)\n .tickSize(1)\n .tickValues([13667, 36713, 59759, 82805, 105851, 128897]);\n\n var svg = d3.select(&quot;.legend.leaflet-control&quot;).append(&quot;svg&quot;)\n .attr(&quot;id&quot;, 'legend')\n .attr(&quot;width&quot;, 450)\n .attr(&quot;height&quot;, 40);\n\n var g = svg.append(&quot;g&quot;)\n .attr(&quot;class&quot;, &quot;key&quot;)\n .attr(&quot;transform&quot;, &quot;translate(25,16)&quot;);\n\n g.selectAll(&quot;rect&quot;)\n .data(color.range().map(function(d, i) {\n return {\n x0: i ? x(color.domain()[i - 1]) : x.range()[0],\n x1: i < color.domain().length ? x(color.domain()[i]) : x.range()[1],\n z: d\n };\n }))\n .enter().append(&quot;rect&quot;)\n .attr(&quot;height&quot;, 10)\n .attr(&quot;x&quot;, function(d) { return d.x0; })\n .attr(&quot;width&quot;, function(d) { return d.x1 - d.x0; })\n .style(&quot;fill&quot;, function(d) { return d.z; });\n\n g.call(xAxis).append(&quot;text&quot;)\n .attr(&quot;class&quot;, &quot;caption&quot;)\n .attr(&quot;y&quot;, 21)\n .text('count');\n \n\n \n\n \n\n \n\n \n\n \n\n var autobounds = L.featureGroup([]).getBounds()\n\n\n\n\n\nmap.fitBounds(autobounds,\n {&quot;padding&quot;: [30, 30]}\n);\n\n };\n\n </script>\n</body>\" style=\"width: 100%; height: 500px; border: none\"></iframe>"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": null, "cell_type": "code", "source": "", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}], "nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 2", "name": "python2", "language": "python"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "version": "2.7.9", "name": "python", "file_extension": ".py", "pygments_lexer": "ipython2", "codemirror_mode": {"version": 2, "name": "ipython"}}}}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment