Created
August 7, 2014 11:15
-
-
Save johnyzed/d5e87d16b0ee3d38bcc3 to your computer and use it in GitHub Desktop.
Python script that collect data from Incapsula API and returns it as JSON for Splunk integration
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import pycurl | |
| import cStringIO | |
| import json | |
| import base64 | |
| import sys | |
| import re | |
| from pprint import pprint | |
| from datetime import datetime | |
| from calendar import timegm | |
| def get_curl(api,params_string): | |
| buf = cStringIO.StringIO() | |
| api_id="ID" | |
| api_key="KEY" | |
| url="https://my.incapsula.com" | |
| curl = pycurl.Curl() | |
| curl.setopt(pycurl.URL,"%s/%s" % (url,api)) | |
| if not params_string: | |
| curl.setopt(pycurl.POSTFIELDS, "api_id=%s&api_key=%s" % (api_id.decode('base64'),api_key.decode('base64')) ) | |
| else: | |
| curl.setopt(pycurl.POSTFIELDS, "api_id=%s&api_key=%s&%s" % (api_id.decode('base64'),api_key.decode('base64'),params_string) ) | |
| curl.setopt(pycurl.WRITEFUNCTION, buf.write) | |
| curl.perform() | |
| json_result = buf.getvalue() | |
| return json.loads(buf.getvalue()) | |
| def main(): | |
| site_id_api="api/prov/v1/sites/list" | |
| domain_name_api="api/prov/v1/sites/status" | |
| stats_api="api/stats/v1" | |
| statistics_array=["visits_timeseries","hits_timeseries","bandwidth_timeseries"] | |
| index_length = 120 | |
| data = get_curl(site_id_api , "") | |
| sites_num=len(data['sites']) | |
| site_id_array = [None] * sites_num | |
| for x in range(0,sites_num): | |
| site_id_array[x] = data['sites'][x]['site_id'] | |
| site_dict={} | |
| for site_id_num in site_id_array: | |
| data=get_curl(domain_name_api, "site_id=%s" % site_id_num) | |
| site_dict[site_id_num]=data['domain'] | |
| for stat_key in range(0,len(statistics_array)): | |
| statistics=statistics_array[stat_key] | |
| if statistics == "bandwidth_timeseries": | |
| for key in site_dict: | |
| site=key | |
| data=get_curl( stats_api, "site_id=%s&time_range=today&stats=%s" % (key, | |
| statistics)) | |
| if not data[statistics][0]["data"]: | |
| continue | |
| else : | |
| bandwitch_data=data[statistics][0]["data"] | |
| bps_data =data[statistics][1]["data"] | |
| bandwitch_dict={} | |
| bps_dict={} | |
| for couple in range(0,len(bandwitch_data)): | |
| bandwitch_dict[bandwitch_data[couple][0]]=bandwitch_data[couple][1] | |
| bps_dict[bandwitch_data[couple][0]]=bps_data[couple][1] | |
| for key in bandwitch_dict.keys(): | |
| json_string="{\"_time\":\"%s\",\"site\":\"%s\",\"statistics\":\"%s\",\"bandwitch\":\"%s\",\"bps\":\"%s\"}" % (key/1000,site_dict[site],statistics,bandwitch_dict[key],bps_dict[key]) | |
| print json_string | |
| for key in site_dict: | |
| site=key | |
| data=get_curl( stats_api, "site_id=%s&time_range=today&stats=%s" % (key, | |
| statistics)) | |
| if not data[statistics][0]["data"]: | |
| continue | |
| else : | |
| human_data=data[statistics][0]["data"] | |
| bots_data =data[statistics][1]["data"] | |
| human_dict={} | |
| bot_dict={} | |
| for couple in range(0,len(human_data)): | |
| human_dict[human_data[couple][0]]=human_data[couple][1] | |
| bot_dict[human_data[couple][0]]=bots_data[couple][1] | |
| for key in human_dict.keys(): | |
| json_string="{\"_time\":\"%s\",\"site\":\"%s\",\"statistics\":\"%s\",\"human\":\"%s\",\"bots\":\"%s\"}" % (key/1000,site_dict[site],statistics,human_dict[key],bot_dict[key]) | |
| print json_string | |
| if __name__ == '__main__': | |
| main() |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hi Guys,
First I am very glad, this script can help you.
Secondly, I wrote it a while ago, and since then I left the company where it was running. Nowaday I am working in a company that doesn't use Incapsula, so it is very difficult for me to help you guys using the script when I dont have access to an API key/id to test it. And there is the possibility that since then , Incapsula changed their API (thing that I could not be aware of, since I am no longer registered to their newsletter).
To answer to @hardrock1234 ,yes you just replace the key and the id, but after encrypting them to base64 (more info here https://www.safaribooksonline.com/library/view/python-cookbook-3rd/9781449357337/ch06s10.html) .
Hope it helps.