-
-
Save dblackdblack/5a6b115aa2fcce0b158cb6e299547d6a to your computer and use it in GitHub Desktop.
covid scraper
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import logging | |
| import json | |
| import requests | |
| import collections | |
| import socket | |
| datadog_host = '127.0.0.1' | |
| datadog_port = 8125 | |
| sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) | |
| logging.basicConfig(level=logging.INFO) | |
| def send_datadog(stat): | |
| sock.sendto(str.encode(stat), (datadog_host, datadog_port)) | |
| def main(): | |
| countries_data = requests.get('https://services1.arcgis.com/0MSEUqKaxRlEPj5g/arcgis/rest/services/ncov_cases/FeatureServer/2/query?f=json&where=Confirmed%20%3E%200&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Confirmed%20desc&resultOffset=0&resultRecordCount=100&cacheHint=true').json() | |
| states_data = requests.get('https://services9.arcgis.com/N9p5hsImWXAccRNI/arcgis/rest/services/Z7biAeD8PAkqgmWhxG2A/FeatureServer/1/query?f=json&where=Confirmed%20%3E%200&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Confirmed%20desc%2CCountry_Region%20asc%2CProvince_State%20asc&resultOffset=0&resultRecordCount=250&cacheHint=true', headers={'referer': 'https://www.arcgis.com/'}).json() | |
| all_countries = [r['attributes']['Country_Region'] for r in countries_data['features']] | |
| totals = collections.defaultdict(int) | |
| fields = ['Active', 'Confirmed', 'Deaths', 'Recovered'] | |
| def countries(rec): | |
| country = rec['Country_Region'].replace(" ", "").replace(",", "") | |
| for f in fields: | |
| count = rec[f] | |
| totals[f] += count | |
| stat = f"covid.{f}:{count}|g|#country:{country}" | |
| send_datadog(stat) | |
| logging.info(stat) | |
| def states(rec): | |
| country = rec['Country_Region'].replace(" ", "").replace(",", "") | |
| state = rec['Province_State'] | |
| if not state: | |
| return | |
| state = state.replace(" ", "").replace(",", "").replace(".", "") | |
| if country not in ('US', 'Canada'): | |
| return | |
| for f in fields: | |
| count = rec[f] | |
| stat = f"covid.{f}.by-state:{count}|g|#country:{country},state:{state}" | |
| send_datadog(stat) | |
| logging.info(stat) | |
| [countries(r['attributes']) for r in countries_data['features']] | |
| # report country totals | |
| for f in fields: | |
| count = totals[f] | |
| stat = f"covid.{f}:{count}|g|#country:total" | |
| logging.info(stat) | |
| send_datadog(stat) | |
| [states(r['attributes']) for r in states_data['features']] | |
| if __name__ == '__main__': | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment