Created
November 10, 2021 08:36
-
-
Save zorteran/dfe922a839ed7c437aa46dadc285799a to your computer and use it in GitHub Desktop.
Filebeat - Haproxy module - ingest pipeline - TCP log
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| PUT _ingest/pipeline/filebeat-7.15.1-haproxy-log-pipeline-custom | |
| { | |
| "description" : "Pipeline for parsing HAProxy http, tcp and default logs. Requires the geoip plugin.", | |
| "processors" : [ | |
| { | |
| "set" : { | |
| "field" : "event.ingested", | |
| "value" : "{{_ingest.timestamp}}" | |
| } | |
| }, | |
| { | |
| "grok" : { | |
| "field" : "message", | |
| "patterns" : [ | |
| """%{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: (%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy_server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.total_session_duration_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}""", | |
| """%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{GREEDYDATA} (%{IPORHOST:source.address}|-):%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\)""", | |
| "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} (%{IPORHOST:destination.address} )?%{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", | |
| """(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}""", | |
| """%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}""" | |
| ], | |
| "ignore_missing" : false, | |
| "pattern_definitions" : { | |
| "HAPROXY_DATE" : "(%{MONTHDAY}[/-]%{MONTH}[/-]%{YEAR}:%{HOUR}:%{MINUTE}:%{SECOND})|%{SYSLOGTIMESTAMP}" | |
| } | |
| } | |
| }, | |
| { | |
| "date" : { | |
| "field" : "haproxy.request_date", | |
| "target_field" : "@timestamp", | |
| "formats" : [ | |
| "dd/MMM/yyyy:HH:mm:ss.SSS", | |
| "MMM dd HH:mm:ss" | |
| ], | |
| "on_failure" : [ | |
| { | |
| "append" : { | |
| "field" : "error.message", | |
| "value" : "{{ _ingest.on_failure_message }}" | |
| } | |
| } | |
| ], | |
| "if" : "ctx.event.timezone == null" | |
| } | |
| }, | |
| { | |
| "date" : { | |
| "target_field" : "@timestamp", | |
| "formats" : [ | |
| "dd/MMM/yyyy:HH:mm:ss.SSS", | |
| "MMM dd HH:mm:ss" | |
| ], | |
| "timezone" : "{{ event.timezone }}", | |
| "on_failure" : [ | |
| { | |
| "append" : { | |
| "field" : "error.message", | |
| "value" : "{{ _ingest.on_failure_message }}" | |
| } | |
| } | |
| ], | |
| "if" : "ctx.event.timezone != null", | |
| "field" : "haproxy.request_date" | |
| } | |
| }, | |
| { | |
| "remove" : { | |
| "field" : "haproxy.request_date" | |
| } | |
| }, | |
| { | |
| "remove" : { | |
| "field" : "message" | |
| } | |
| }, | |
| { | |
| "grok" : { | |
| "field" : "haproxy.http.request.raw_request_line", | |
| "patterns" : [ | |
| "%{WORD:http.request.method}%{SPACE}%{URIPATHPARAM:url.original}%{SPACE}HTTP/%{NUMBER:http.version}" | |
| ], | |
| "ignore_missing" : true | |
| } | |
| }, | |
| { | |
| "uri_parts" : { | |
| "field" : "url.original", | |
| "ignore_failure" : true, | |
| "if" : "ctx?.url?.original != null" | |
| } | |
| }, | |
| { | |
| "grok" : { | |
| "ignore_failure" : true, | |
| "patterns" : [ | |
| "^%{IP:source.ip}$" | |
| ], | |
| "field" : "source.address" | |
| } | |
| }, | |
| { | |
| "grok" : { | |
| "field" : "destination.address", | |
| "patterns" : [ | |
| "^%{IP:destination.ip}$" | |
| ], | |
| "on_failure" : [ | |
| { | |
| "set" : { | |
| "field" : "destination.domain", | |
| "value" : "{{destination.address}}", | |
| "ignore_empty_value" : true | |
| } | |
| } | |
| ] | |
| } | |
| }, | |
| { | |
| "geoip" : { | |
| "field" : "source.ip", | |
| "target_field" : "source.geo", | |
| "ignore_missing" : true | |
| } | |
| }, | |
| { | |
| "geoip" : { | |
| "database_file" : "GeoLite2-ASN.mmdb", | |
| "field" : "source.ip", | |
| "target_field" : "source.as", | |
| "properties" : [ | |
| "asn", | |
| "organization_name" | |
| ], | |
| "ignore_missing" : true | |
| } | |
| }, | |
| { | |
| "rename" : { | |
| "ignore_missing" : true, | |
| "field" : "source.as.asn", | |
| "target_field" : "source.as.number" | |
| } | |
| }, | |
| { | |
| "rename" : { | |
| "field" : "source.as.organization_name", | |
| "target_field" : "source.as.organization.name", | |
| "ignore_missing" : true | |
| } | |
| }, | |
| { | |
| "split" : { | |
| "ignore_failure" : true, | |
| "field" : "haproxy.http.request.captured_headers", | |
| "separator" : """\|""" | |
| } | |
| }, | |
| { | |
| "split" : { | |
| "field" : "haproxy.http.response.captured_headers", | |
| "separator" : """\|""", | |
| "ignore_failure" : true | |
| } | |
| }, | |
| { | |
| "script" : { | |
| "lang" : "painless", | |
| "source" : "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)", | |
| "params" : { | |
| "scale" : 1000000 | |
| }, | |
| "if" : "ctx.temp?.duration != null" | |
| } | |
| }, | |
| { | |
| "remove" : { | |
| "field" : "temp.duration", | |
| "ignore_missing" : true | |
| } | |
| }, | |
| { | |
| "convert" : { | |
| "field" : "haproxy.bytes_read", | |
| "target_field" : "http.response.bytes", | |
| "type" : "long", | |
| "if" : "ctx.containsKey('http')" | |
| } | |
| }, | |
| { | |
| "append" : { | |
| "if" : "ctx?.source?.ip != null", | |
| "field" : "related.ip", | |
| "value" : "{{source.ip}}" | |
| } | |
| }, | |
| { | |
| "append" : { | |
| "field" : "related.ip", | |
| "value" : "{{destination.ip}}", | |
| "if" : "ctx?.destination?.ip != null" | |
| } | |
| }, | |
| { | |
| "append" : { | |
| "field" : "related.hosts", | |
| "value" : "{{destination.domain}}", | |
| "if" : "ctx?.destination?.domain != null" | |
| } | |
| }, | |
| { | |
| "set" : { | |
| "field" : "event.kind", | |
| "value" : "event" | |
| } | |
| }, | |
| { | |
| "append" : { | |
| "value" : "web", | |
| "if" : "ctx?.haproxy?.mode == 'HTTP' || ctx?.haproxy?.http != null", | |
| "field" : "event.category" | |
| } | |
| }, | |
| { | |
| "append" : { | |
| "field" : "event.category", | |
| "value" : "network", | |
| "if" : "ctx?.source.ip != null && ctx?.destination?.ip != null" | |
| } | |
| }, | |
| { | |
| "append" : { | |
| "field" : "event.type", | |
| "value" : "connection", | |
| "if" : "ctx?.source.ip != null && ctx?.destination?.ip != null" | |
| } | |
| }, | |
| { | |
| "set" : { | |
| "field" : "event.outcome", | |
| "value" : "success", | |
| "if" : "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" | |
| } | |
| }, | |
| { | |
| "set" : { | |
| "field" : "event.outcome", | |
| "value" : "failure", | |
| "if" : "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" | |
| } | |
| }, | |
| { | |
| "script" : { | |
| "lang" : "painless", | |
| "description" : "This script processor iterates over the whole document to remove fields with null values.", | |
| "source" : """void handleMap(Map map) { | |
| for (def x : map.values()) { | |
| if (x instanceof Map) { | |
| handleMap(x); | |
| } else if (x instanceof List) { | |
| handleList(x); | |
| } | |
| } | |
| map.values().removeIf(v -> v == null); | |
| } | |
| void handleList(List list) { | |
| for (def x : list) { | |
| if (x instanceof Map) { | |
| handleMap(x); | |
| } else if (x instanceof List) { | |
| handleList(x); | |
| } | |
| } | |
| } | |
| handleMap(ctx); | |
| """ | |
| } | |
| } | |
| ], | |
| "on_failure" : [ | |
| { | |
| "set" : { | |
| "field" : "error.message", | |
| "value" : "{{ _ingest.on_failure_message }}" | |
| } | |
| } | |
| ] | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment