Infinite Disorder

logstash config:

input {
#  tcp {
#    port => 5000
#    type => syslog
#  }
#  udp {
#    port => 5000
#    type => syslog
#  }
#  lumberjack {
#    port => 5001
#    type => "logs"
#    ssl_certificate => "/etc/pki/tls/certs/elk-staging.crt"
#    ssl_key => "/etc/pki/tls/private/elk-staging.key"
#  }
  beats {
    port => 5018
    type => "log4j"
    ssl => true
    ssl_certificate => "/etc/pki/tls/certs/elk-staging.crt"
    ssl_key => "/etc/pki/tls/private/elk-staging.key"
    codec => multiline {
      # Grok pattern names are valid! :)
      pattern => "^%{TIMESTAMP_ISO8601} "
      negate => true
      what => previous
    }
  }
}

output {
  elasticsearch {
    hosts => ["127.0.0.1:9200"]
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

filebeat.yml:

filebeat:
  prospectors:
    -
      paths:
        - /myapp/log/*.log
        - /myOtherApp/log/production.log
      input_type: log
  registry_file: /var/lib/filebeat/registry
output:
  logstash:
    hosts: ["elk-staging.infinitedisorder.com:5018"]
    tls:
      certificate_authorities: ["/etc/pki/tls/certs/elk-staging.crt"]
      certificate: "/etc/pki/tls/certs/elk-staging.crt"
      certificate_key: "/etc/pki/tls/private/elk-staging.key"
shipper:
logging:
  files:
logging:
  level: warning
  to_files: true
  to_syslog: false
  files:
    path: /var/log/filebeat
    name: filebeat.log
    keepfiles: 7

Automated install from ansible playbook:

---
- hosts: tag_Name_*hosts*
sudo: True
user: ubuntu
tasks:
- command: mkdir -p /etc/pki/tls/certs
- command: mkdir -p /etc/pki/tls/private
- copy: src=./elk-staging.crt dest=/etc/pki/tls/certs/elk-staging.crt
- copy: src=./elk-staging.key dest=/etc/pki/tls/private/elk-staging.key
- command: chmod 444 /etc/pki/tls/private/elk-staging.key
- shell: wget https://download.elastic.co/beats/filebeat/filebeat_1.0.0-rc2_amd64.deb
- shell: dpkg -i filebeat_1.0.0-rc2_amd64.deb
- copy: src=./filebeat.yml dest=/etc/filebeat/filebeat.yml
- shell: curl -XPUT 'http://elk-staging.infinitedisorder.com:9200/_template/filebeat?pretty' -d@/etc/filebeat/filebeat.template.json
- service: name=filebeat state=restarted

Don’t forget to install the logstash-input-filebeat plugin