Skip to content

Instantly share code, notes, and snippets.

@stormpython

stormpython/README.md

Last active March 21, 2022 13:45
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save stormpython/8796112 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save stormpython/8796112 to your computer and use it in GitHub Desktop.
Download ZIP
Data Visualization with Elasticsearch Aggregations and D3 (Tutorial)
Raw
README.md

Data Visualization with Elasticsearch Aggregations and D3

Link to the Elasticsearch Blog.

Raw
index.html
<!DOCTYPE html>
<html>
<head>
<title>Elastic Aggregations</title>
<script src="scripts/require.js"></script>
<script>require(["scripts/main"], function () {})</script>
<style>
body {
font: 14px sans-serif;
}
.arc path {
stroke: #fff;
stroke-width: 3px;
}
.node circle {
fill: #fff;
stroke: steelblue;
stroke-width: 1.5px;
}
.node {
font: 10px sans-serif;
}
.link {
fill: none;
stroke: #ccc;
stroke-width: 1.5px;
}
</style>
</head>
<body>
<div id="donut-chart"></div>
<div id="dendrogram"></div>
</body>
</html>
Raw
main.js
define(['scripts/d3.v3', 'scripts/elasticsearch'], function (d3, elasticsearch) {
"use strict";
var client = new elasticsearch.Client();
client.search({
index: 'nfl',
size: 5,
body: {
// Begin query.
query: {
// Boolean query for matching and excluding items.
bool: {
must: { match: { "description": "TOUCHDOWN" }},
must_not: { match: { "qtr": 5 }}
}
},
// Aggregate on the results
aggs: {
touchdowns: {
terms: {
field: "qtr",
order: { "_term" : "asc" }
}
}
}
// End query.
}
}).then(function (resp) {
console.log(resp);
// D3 code goes here.
var touchdowns = resp.aggregations.touchdowns.buckets;
// d3 donut chart
var width = 600,
height = 300,
radius = Math.min(width, height) / 2;
var color = ['#ff7f0e', '#d62728', '#2ca02c', '#1f77b4'];
var arc = d3.svg.arc()
.outerRadius(radius - 60)
.innerRadius(120);
var pie = d3.layout.pie()
.sort(null)
.value(function (d) { return d.doc_count; });
var svg = d3.select("#donut-chart").append("svg")
.attr("width", width)
.attr("height", height)
.append("g")
.attr("transform", "translate(" + width/1.4 + "," + height/2 + ")");
var g = svg.selectAll(".arc")
.data(pie(touchdowns))
.enter()
.append("g")
.attr("class", "arc");
g.append("path")
.attr("d", arc)
.style("fill", function (d, i) {
return color[i];
});
g.append("text")
.attr("transform", function (d) { return "translate(" + arc.centroid(d) + ")"; })
.attr("dy", ".35em")
.style("text-anchor", "middle")
.style("fill", "white")
.text(function (d) { return d.data.key; });
});
client.search({
index: 'nfl',
size: 5,
body: {
query: {
bool: {
must: { match: { "description": "TOUCHDOWN"}},
must_not: [
{ match: { "description": "intercepted"}},
{ match: { "description": "incomplete"}},
{ match: { "description": "FUMBLES"}},
{ match: { "description": "NULLIFIED"}}
]
}
},
aggs: {
teams: {
terms: {
field: "off",
exclude: "",
size: 5
},
aggs: {
players: {
terms: {
field: "description",
include: "([a-z]?[.][a-z]+)",
size: 200
},
aggs: {
qtrs: {
terms: {
field: "qtr",
order: { "_term": "asc" }
}
}
}
}
}
}
}
}
}).then(function (resp) {
console.log(resp);
// D3 code goes here.
var root = createChildNodes(resp);
// d3 dendrogram
var width = 600,
height = 2000;
var color = ['#ff7f0e', '#d62728', '#2ca02c', '#1f77b4'];
var cluster = d3.layout.cluster()
.size([height, width - 200]);
var diagonal = d3.svg.diagonal()
.projection(function(d) { return [d.y, d.x]; });
var svg = d3.select("#dendrogram").append("svg")
.attr("width", width)
.attr("height", height)
.append("g")
.attr("transform", "translate(120,0)");
var nodes = cluster.nodes(root),
links = cluster.links(nodes);
var link = svg.selectAll(".link")
.data(links)
.enter().append("path")
.attr("class", "link")
.attr("d", diagonal);
var node = svg.selectAll(".node")
.data(nodes)
.enter().append("g")
.attr("class", "node")
.attr("transform", function(d) { return "translate(" + d.y + "," + d.x + ")"; });
node.append("circle")
.attr("r", 4.5)
.style("fill", function (d) {
return d.children ? "#ffffff" : color[d.key - 1];
})
.style("stroke", function (d) {
return d.children ? "#4682B4" : color[d.key - 1];
});
node.append("text")
.attr("dx", function(d) { return d.children ? -8 : 8; })
.attr("dy", 3)
.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
.text(function(d) { return d.children? d.key : d.key + ": " + d.doc_count; });
d3.select(self.frameElement).style("height", height + "px");
function createChildNodes(dataObj) {
var root = {};
root.key = "NFL";
root.children = dataObj.aggregations.teams.buckets;
root.children.forEach(function (d) { d.children = d.players.buckets; });
root.children.forEach(function (d) {
d.children.forEach(function (d) {
d.children = d.qtrs.buckets;
});
});
return root;
}
});
});
@roccia
Copy link

roccia commented Jan 5, 2016

OK I rewrite the mapping.json, it looks like this:

{
"mappings" : {

 "my_type": {

    "properties" : {

        "gameid" : {
            "type" : "string",
            "index" : "not_analyzed",
            "store" : "true"
        },
        "qtr" : {
            "type" : "short",
            "store" : "true"
        },

        "min" : {
            "type" : "short",
            "store" : "true",
             "ignore_malformed": "true"
            },

        "sec" : {
            "type" : "short",
            "store" : "true"

        },
        "off" : {
            "type" : "string"

        },
        "def" : {
            "type" : "string"

        },
        "down" : {
            "type" : "short",
            "store" : "true",
            "ignore_malformed": "true"

        },
        "togo" : {
            "type" : "short",
            "store" : "true",
            "ignore_malformed": "true"

        },
        "ydline" : {
            "type" : "short",
            "store" : "true",
            "ignore_malformed": "true"

        },
        "scorediff" : {
            "type" : "short",
            "store" : "true"
        },
        "series1stdn" : {
            "type" : "short",
            "store" : "true"
        },
        "description" : {
            "type" : "string",
            "store" : "true"
        },
        "scorechange" : {
            "type" : "short",
            "store" : "true"
        },
        "nextscore" : {
            "type" : "short",
            "store" : "true"
        },
        "teamwin" : {
            "type" : "short",
            "store" : "true"
        },
        "offscore" : {
            "type" : "short",
            "store" : "true"
        },
        "defscore" : {
            "type" : "short",
            "store" : "true"
        },
        "season" : {
            "type" : "date",
            "format" : "YYYY",
           "ignore_malformed": "true"


        }
    }
   }
}

}

but I'm still getting "reason" : "Root mapping definition has unsupported parameters: error , any idea why? need your help! thank you

@Veer-Abhimanyu
Copy link

Veer-Abhimanyu commented Mar 22, 2016

@roccia try this..

@Veer-Abhimanyu
Copy link

Veer-Abhimanyu commented Mar 28, 2016

{
"2013" : {
"properties" : {
"gameid" : {
"type" : "string",
"index" : "not_analyzed",
"store" : "yes"
},
"qtr" : {
"type" : "short",
"store" : "yes"
},
"min" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"sec" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"off" : {
"type" : "string",
"index" : "not_analyzed"
},
"def" : {
"type" : "string",
"index" : "not_analyzed"
},
"down" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"togo" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"ydline" : {
"type" : "short",
"store" : "yes",
"ignore_malformed" : "true"
},
"scorediff" : {
"type" : "short",
"store" : "yes"
},
"series1stdn" : {
"type" : "short",
"store" : "yes"
},
"description" : {
"type" : "string",
"store" : "yes"
},
"scorechange" : {
"type" : "short",
"store" : "yes"
},
"nextscore" : {
"type" : "short",
"store" : "yes"
},
"teamwin" : {
"type" : "short",
"store" : "yes"
},
"offscore" : {
"type" : "short",
"store" : "yes"
},
"defscore" : {
"type" : "short",
"store" : "yes"
},
"season" : {
"type" : "date",
"format" : "YYYY",
"ignore_malformed" : "true"
}
}
}
}

@tiwarisahil91
Copy link

tiwarisahil91 commented Nov 21, 2016 •
edited
Loading

Hi,while I am trying to view the chart on the http-server, I created using node js and with the following port no. as http://127.0.0.1:8080, I am getting this kind of error in my console.
XMLHttpRequest cannot load http://localhost:9200/nfl/2013/_search?size=5. No 'Access-Control-Allow-Origin' header is present on the requested resource. Origin 'http://127.0.0.1:8080' is therefore not allowed access.
Anybody knowing how to remove this

@aakritip12
Copy link

aakritip12 commented Jun 14, 2017

@tiwarisahil91 open run and enter chrome.exe --user-data-dir="C:/Chrome dev session" --disable-web-security . A new chrome browser will open run your url in that window

@rolfrennemo
Copy link

rolfrennemo commented Feb 16, 2018

Hi,

I have a problem with this example. Please have a look at the screeen shots enclosed.
image

image

Thanks!

@VSJTech
Copy link

VSJTech commented Apr 19, 2018

hi

i am getting the following error while launching the server:

elasticsearch.js:19450 ERROR: 2018-04-19T10:24:55Z
Error: Request error, retrying -- Request failed to complete.
at Log.error (http://localhost:8001/scripts/elasticsearch.js:19151:56)
at checkRespForFailure (http://localhost:8001/scripts/elasticsearch.js:19817:18)
at XMLHttpRequest.xhr.onreadystatechange (http://localhost:8001/scripts/elasticsearch.js:18655:7)

@VSJTech
Copy link

VSJTech commented Apr 19, 2018

the following helped to get the issue cleared...

var elasticsearch = require('elasticsearch');
var AgentKeepAlive = require('agentkeepalive');
var client = new elasticsearch.Client({

host: 'localhost:9200',
requestTimeout: Infinity, // Tested
keepAlive: true ,// Tested

    maxRetries: 10,
    keepAlive: true,
    maxSockets: 10,
    minSockets: 10,
    createNodeAgent: function (connection, config) {
      return new AgentKeepAlive(connection.makeAgentConfig(config));
    }
});

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment