{"id":383,"date":"2015-11-25T15:32:17","date_gmt":"2015-11-25T22:32:17","guid":{"rendered":"http:\/\/infinitedisorder.com\/?p=383"},"modified":"2015-12-01T16:19:32","modified_gmt":"2015-12-01T23:19:32","slug":"filebeat-and-logstash-2-0-with-log4j-setup","status":"publish","type":"post","link":"https:\/\/infinitedisorder.com\/?p=383","title":{"rendered":"filebeat and logstash 2.0 with log4j setup"},"content":{"rendered":"<p>logstash config:<\/p>\n<pre>\r\n<code>input {\r\n#  tcp {\r\n#    port =&gt; 5000\r\n#    type =&gt; syslog\r\n#  }\r\n#  udp {\r\n#    port =&gt; 5000\r\n#    type =&gt; syslog\r\n#  }\r\n#  lumberjack {\r\n#    port =&gt; 5001\r\n#    type =&gt; \"logs\"\r\n#    ssl_certificate =&gt; \"\/etc\/pki\/tls\/certs\/elk-staging.crt\"\r\n#    ssl_key =&gt; \"\/etc\/pki\/tls\/private\/elk-staging.key\"\r\n#  }\r\n  beats {\r\n    port =&gt; 5018\r\n    type =&gt; \"log4j\"\r\n    ssl =&gt; true\r\n    ssl_certificate =&gt; \"\/etc\/pki\/tls\/certs\/elk-staging.crt\"\r\n    ssl_key =&gt; \"\/etc\/pki\/tls\/private\/elk-staging.key\"\r\n    codec =&gt; multiline {\r\n      # Grok pattern names are valid! :)\r\n      pattern =&gt; \"^%{TIMESTAMP_ISO8601} \"\r\n      negate =&gt; true\r\n      what =&gt; previous\r\n    }\r\n  }\r\n}\r\n\r\noutput {\r\n  elasticsearch {\r\n    hosts =&gt; [\"127.0.0.1:9200\"]\r\n    index =&gt; \"%{[@metadata][beat]}-%{+YYYY.MM.dd}\"\r\n    document_type =&gt; \"%{[@metadata][type]}\"\r\n  }\r\n}<\/code>\r\n<\/pre>\n<p>filebeat.yml:<\/p>\n<pre>\r\n<code>filebeat:\r\n  prospectors:\r\n    -\r\n      paths:\r\n        - \/myapp\/log\/*.log\r\n        - \/myOtherApp\/log\/production.log\r\n      input_type: log\r\n  registry_file: \/var\/lib\/filebeat\/registry\r\noutput:\r\n  logstash:\r\n    hosts: [\"elk-staging.infinitedisorder.com:5018\"]\r\n    tls:\r\n      certificate_authorities: [\"\/etc\/pki\/tls\/certs\/elk-staging.crt\"]\r\n      certificate: \"\/etc\/pki\/tls\/certs\/elk-staging.crt\"\r\n      certificate_key: \"\/etc\/pki\/tls\/private\/elk-staging.key\"\r\nshipper:\r\nlogging:\r\n  files:\r\nlogging:\r\n  level: warning\r\n  to_files: true\r\n  to_syslog: false\r\n  files:\r\n    path: \/var\/log\/filebeat\r\n    name: filebeat.log\r\n    keepfiles: 7<\/code>\r\n<\/pre>\n<p>Automated install from ansible playbook:<\/p>\n<pre>\r\n<code>---\r\n- hosts: tag_Name_*hosts*\r\nsudo: True\r\nuser: ubuntu\r\ntasks:\r\n- command: mkdir -p \/etc\/pki\/tls\/certs\r\n- command: mkdir -p \/etc\/pki\/tls\/private\r\n- copy: src=.\/elk-staging.crt dest=\/etc\/pki\/tls\/certs\/elk-staging.crt\r\n- copy: src=.\/elk-staging.key dest=\/etc\/pki\/tls\/private\/elk-staging.key\r\n- command: chmod 444 \/etc\/pki\/tls\/private\/elk-staging.key\r\n- shell: wget https:\/\/download.elastic.co\/beats\/filebeat\/filebeat_1.0.0-rc2_amd64.deb\r\n- shell: dpkg -i filebeat_1.0.0-rc2_amd64.deb\r\n- copy: src=.\/filebeat.yml dest=\/etc\/filebeat\/filebeat.yml\r\n- shell: curl -XPUT 'http:\/\/elk-staging.infinitedisorder.com:9200\/_template\/filebeat?pretty' -d@\/etc\/filebeat\/filebeat.template.json\r\n- service: name=filebeat state=restarted<\/code>\r\n<\/pre>\n<p>Don&#8217;t forget to install the logstash-input-filebeat plugin<\/p>\n","protected":false},"excerpt":{"rendered":"<p>logstash config: input { # tcp { # port =&gt; 5000 # type =&gt; syslog # } # udp { # port =&gt; 5000 # type =&gt; syslog # } # lumberjack { # port =&gt; 5001 # type =&gt; &#8220;logs&#8221; # ssl_certificate =&gt; &#8220;\/etc\/pki\/tls\/certs\/elk-staging.crt&#8221; # ssl_key =&gt; &#8220;\/etc\/pki\/tls\/private\/elk-staging.key&#8221; # } beats { port =&gt; [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-383","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"_links":{"self":[{"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=\/wp\/v2\/posts\/383","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=383"}],"version-history":[{"count":8,"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=\/wp\/v2\/posts\/383\/revisions"}],"predecessor-version":[{"id":400,"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=\/wp\/v2\/posts\/383\/revisions\/400"}],"wp:attachment":[{"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=383"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=383"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/infinitedisorder.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=383"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}