Created
August 7, 2014 11:15
-
-
Save johnyzed/d5e87d16b0ee3d38bcc3 to your computer and use it in GitHub Desktop.
Python script that collect data from Incapsula API and returns it as JSON for Splunk integration
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import pycurl | |
| import cStringIO | |
| import json | |
| import base64 | |
| import sys | |
| import re | |
| from pprint import pprint | |
| from datetime import datetime | |
| from calendar import timegm | |
| def get_curl(api,params_string): | |
| buf = cStringIO.StringIO() | |
| api_id="ID" | |
| api_key="KEY" | |
| url="https://my.incapsula.com" | |
| curl = pycurl.Curl() | |
| curl.setopt(pycurl.URL,"%s/%s" % (url,api)) | |
| if not params_string: | |
| curl.setopt(pycurl.POSTFIELDS, "api_id=%s&api_key=%s" % (api_id.decode('base64'),api_key.decode('base64')) ) | |
| else: | |
| curl.setopt(pycurl.POSTFIELDS, "api_id=%s&api_key=%s&%s" % (api_id.decode('base64'),api_key.decode('base64'),params_string) ) | |
| curl.setopt(pycurl.WRITEFUNCTION, buf.write) | |
| curl.perform() | |
| json_result = buf.getvalue() | |
| return json.loads(buf.getvalue()) | |
| def main(): | |
| site_id_api="api/prov/v1/sites/list" | |
| domain_name_api="api/prov/v1/sites/status" | |
| stats_api="api/stats/v1" | |
| statistics_array=["visits_timeseries","hits_timeseries","bandwidth_timeseries"] | |
| index_length = 120 | |
| data = get_curl(site_id_api , "") | |
| sites_num=len(data['sites']) | |
| site_id_array = [None] * sites_num | |
| for x in range(0,sites_num): | |
| site_id_array[x] = data['sites'][x]['site_id'] | |
| site_dict={} | |
| for site_id_num in site_id_array: | |
| data=get_curl(domain_name_api, "site_id=%s" % site_id_num) | |
| site_dict[site_id_num]=data['domain'] | |
| for stat_key in range(0,len(statistics_array)): | |
| statistics=statistics_array[stat_key] | |
| if statistics == "bandwidth_timeseries": | |
| for key in site_dict: | |
| site=key | |
| data=get_curl( stats_api, "site_id=%s&time_range=today&stats=%s" % (key, | |
| statistics)) | |
| if not data[statistics][0]["data"]: | |
| continue | |
| else : | |
| bandwitch_data=data[statistics][0]["data"] | |
| bps_data =data[statistics][1]["data"] | |
| bandwitch_dict={} | |
| bps_dict={} | |
| for couple in range(0,len(bandwitch_data)): | |
| bandwitch_dict[bandwitch_data[couple][0]]=bandwitch_data[couple][1] | |
| bps_dict[bandwitch_data[couple][0]]=bps_data[couple][1] | |
| for key in bandwitch_dict.keys(): | |
| json_string="{\"_time\":\"%s\",\"site\":\"%s\",\"statistics\":\"%s\",\"bandwitch\":\"%s\",\"bps\":\"%s\"}" % (key/1000,site_dict[site],statistics,bandwitch_dict[key],bps_dict[key]) | |
| print json_string | |
| for key in site_dict: | |
| site=key | |
| data=get_curl( stats_api, "site_id=%s&time_range=today&stats=%s" % (key, | |
| statistics)) | |
| if not data[statistics][0]["data"]: | |
| continue | |
| else : | |
| human_data=data[statistics][0]["data"] | |
| bots_data =data[statistics][1]["data"] | |
| human_dict={} | |
| bot_dict={} | |
| for couple in range(0,len(human_data)): | |
| human_dict[human_data[couple][0]]=human_data[couple][1] | |
| bot_dict[human_data[couple][0]]=bots_data[couple][1] | |
| for key in human_dict.keys(): | |
| json_string="{\"_time\":\"%s\",\"site\":\"%s\",\"statistics\":\"%s\",\"human\":\"%s\",\"bots\":\"%s\"}" % (key/1000,site_dict[site],statistics,human_dict[key],bot_dict[key]) | |
| print json_string | |
| if __name__ == '__main__': | |
| main() |
Author
Hi Guys,
First I am very glad, this script can help you.
Secondly, I wrote it a while ago, and since then I left the company where it was running. Nowaday I am working in a company that doesn't use Incapsula, so it is very difficult for me to help you guys using the script when I dont have access to an API key/id to test it. And there is the possibility that since then , Incapsula changed their API (thing that I could not be aware of, since I am no longer registered to their newsletter).
To answer to @hardrock1234 ,yes you just replace the key and the id, but after encrypting them to base64 (more info here https://www.safaribooksonline.com/library/view/python-cookbook-3rd/9781449357337/ch06s10.html) .
Hope it helps.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hi Johnyzed: Trying to get your scrip running , however i get the following errors , Please advise ..
Traceback (most recent call last):
File "C:/Users/sza/Desktop/Accelop-Improvements/d5e87d16b0ee3d38bcc3-75e7c30e4c35b4a9bb6b828bc3934ea25d495274/incapsula.py", line 97, in
main()
File "C:/Users/sza/Desktop/Accelop-Improvements/d5e87d16b0ee3d38bcc3-75e7c30e4c35b4a9bb6b828bc3934ea25d495274/incapsula.py", line 38, in main
data = get_curl(site_id_api , "")
File "C:/Users/sza/Desktop/Accelop-Improvements/d5e87d16b0ee3d38bcc3-75e7c30e4c35b4a9bb6b828bc3934ea25d495274/incapsula.py", line 20, in get_curl
curl.setopt(pycurl.POSTFIELDS, "api_id=%s&api_key=%s" % (api_id.decode('base64'),api_key.decode('base64')))
AttributeError: 'int' object has no attribute 'decode'
Much appreciate your help :)