Created
July 15, 2019 11:50
-
-
Save nehiljain/a3a973fe6ff41454e9e8e8ee41805ed9 to your computer and use it in GitHub Desktop.
Setup Fluentd on Elasticbeanstalk 2019
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| files: | |
| "/etc/td-agent/td-agent.conf": | |
| owner: root | |
| group: root | |
| content: | | |
| <source> | |
| @type tail | |
| path /var/log/eb-docker/containers/log.file | |
| exclude_path ["/var/log/eb-docker/containers/exception.file"] | |
| pos_file /var/log/td-agent/fluentd.log.pos | |
| tag schema | |
| <parse> | |
| @type json | |
| </parse> | |
| </source> | |
| <match schema> | |
| @type rewrite_tag_filter | |
| <rule> | |
| key "schema" | |
| pattern /^(\w+)/ | |
| tag "<%text>${tag}</%text>_$1" | |
| </rule> | |
| </match> | |
| <match schema_*> | |
| @type "s3" | |
| aws_key_id ${PIPELINE_LOG_AWS_KEY} | |
| aws_sec_key ${PIPELINE_LOG_AWS_SECRET} | |
| s3_bucket pipeline-logs | |
| s3_region "us-east-1" | |
| path "<%text>${tag}</%text>/%Y/%m/%d/" | |
| s3_object_key_format "%{path}_#{worker_id}_%{time_slice}_%{index}.%{file_extension}" | |
| store_as gzip_command | |
| format json | |
| include_time_key true | |
| time_key fluent_log_time | |
| <buffer tag, time> | |
| @type "file" | |
| flush_thread_count 8 | |
| path "/var/log/s3" | |
| timekey 600 | |
| timekey_wait 300 | |
| timekey_use_utc true | |
| chunk_limit_size 15m | |
| </buffer> | |
| </match> | |
| "/tmp/install_td_agent.sh": | |
| mode: "000755" | |
| owner: root | |
| group: root | |
| content: | | |
| echo "==============================" | |
| echo " td-agent Installation Script " | |
| echo "==============================" | |
| echo "This script requires superuser access to install rpm packages." | |
| echo "You will be prompted for your password by sudo." | |
| # clear any previous sudo permission | |
| sudo -k | |
| # run inside sudo | |
| sudo sh <<SCRIPT | |
| # add GPG key | |
| rpm --import https://packages.treasuredata.com/GPG-KEY-td-agent | |
| # add treasure data repository to yum | |
| cat >/etc/yum.repos.d/td.repo <<'EOF'; | |
| [treasuredata] | |
| name=TreasureData | |
| # NOTE: this might be have changed. | |
| baseurl=http://packages.treasuredata.com.s3.amazonaws.com/3/amazon/1/2018.03/x86_64 | |
| gpgcheck=1 | |
| gpgkey=https://packages.treasuredata.com/GPG-KEY-td-agent | |
| EOF | |
| # update your sources | |
| yum check-update | |
| # install the toolbelt | |
| yes | yum install -y td-agent | |
| SCRIPT | |
| # message | |
| echo "" | |
| echo "Installation completed. Happy Logging!" | |
| echo "" | |
| mkdir /var/log/s3 | |
| chmod 777 /var/log/s3 | |
| commands: | |
| 01-command: | |
| command: echo 'Defaults:root !requiretty' >> /etc/sudoers | |
| 02-command: | |
| command: sh /tmp/install_td_agent.sh | |
| 03-command: | |
| command: td-agent-gem install fluent-plugin-rewrite-tag-filter | |
| 04-command: | |
| command: td-agent-gem install fluent-plugin-s3 -v 1.0.0 | |
| 05-command: | |
| command: /etc/init.d/td-agent restart |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment