From 6515fde45500e20a86145344b821288dad027b7c Mon Sep 17 00:00:00 2001
From: "antoine.roux@zenika.com" <antoinroux@hotmail.fr>
Date: Thu, 7 Feb 2019 00:50:26 +0100
Subject: [PATCH] Add ELK stack

---
 Makefile                                      |  6 +-
 test.dockerapp/docker-compose.yml             | 58 ++++++++++++++++++-
 .../logstash/pipelines/elasticsearch.conf     | 14 +++++
 test.dockerapp/logstash/pipelines/stdin.conf  |  9 +++
 test.dockerapp/logstash/pipelines/telnet.conf | 30 ++++++++++
 test.dockerapp/parameters.yml                 |  5 +-
 6 files changed, 119 insertions(+), 3 deletions(-)
 create mode 100644 test.dockerapp/logstash/pipelines/elasticsearch.conf
 create mode 100644 test.dockerapp/logstash/pipelines/stdin.conf
 create mode 100644 test.dockerapp/logstash/pipelines/telnet.conf

diff --git a/Makefile b/Makefile
index e26e46e..d315847 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 
 run:
-	@docker-app render | docker-compose -f - up -d --build && docker logs docker_hello_1 -f
+	@docker-app render | docker-compose -f - up -d --build && docker logs hello -f
 
 rm:
 	@docker rm -f $$(docker ps -a --format="{{.Names}}")
@@ -13,3 +13,7 @@ put:
 
 del:
 	@etcdctl del /dev/etcd/loaded
+
+
+status:
+	curl http://127.0.0.1:9200/_cat/health
diff --git a/test.dockerapp/docker-compose.yml b/test.dockerapp/docker-compose.yml
index 96d8b45..aa7784c 100644
--- a/test.dockerapp/docker-compose.yml
+++ b/test.dockerapp/docker-compose.yml
@@ -7,6 +7,7 @@ services:
       args:
         httpEchoServerVersion: ${image-version}
         alpineVersion: ${alpine-version}
+    container_name: hello      
     environment:
       HTTPTEXT: ${response-text}
       # etcdWait parameter
@@ -17,14 +18,69 @@ services:
       - ${echo-port}:5678
   
   etcd:
+    container_name: etcd
     image: bitnami/etcd
     environment:
       - ALLOW_NONE_AUTHENTICATION=yes
       # - ETCD_ADVERTISE_CLIENT_URLS=http://0.0.0.0:2379
     restart: always
     ports:
-      - "2379:2379/tcp"
+      - ${etcd-port}:2379/tcp
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${elk-version}
+    container_name: elasticsearch
+    environment:
+      - cluster.name=docker-cluster
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - esdata1:/usr/share/elasticsearch/data
+    ports:
+      - ${es-port}:9200
+    networks:
+      - esnet
+  
+  logstash:
+    image: docker.elastic.co/logstash/logstash:${elk-version}
+    container_name: logstash
+    environment:
+      XPACK_MONITORING_ELASTICSEARCH_URL: http://elasticsearch:9200
+    volumes:
+      - pipelines:/usr/share/logstash/pipeline/
+    ports:
+      - 5000:5000
+    depends_on:
+      - elasticsearch
+    networks:
+      - esnet
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:${elk-version}
+    container_name: kibana
+    environment:
+      SERVER_NAME: local.localhost.com
+      ELASTICSEARCH_HOSTS: http://elasticsearch
+    ports:
+      - ${kibana-port}:5601
+    depends_on:
+      - elasticsearch
+    networks:
+      - esnet
 
 volumes:
   data-volume:
     driver: local
+  esdata1:
+    driver: local
+  pipelines:
+    driver_opts:
+      type: none
+      device: $$PWD/test.dockerapp/logstash/pipelines/
+      o: bind
+networks:
+  esnet:
diff --git a/test.dockerapp/logstash/pipelines/elasticsearch.conf b/test.dockerapp/logstash/pipelines/elasticsearch.conf
new file mode 100644
index 0000000..137c9cf
--- /dev/null
+++ b/test.dockerapp/logstash/pipelines/elasticsearch.conf
@@ -0,0 +1,14 @@
+input {
+  beats {
+    port => 5044
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => "elasticsearch:9200"
+    manage_template => false
+    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" 
+    document_type => "%{[@metadata][type]}" 
+  }
+}
diff --git a/test.dockerapp/logstash/pipelines/stdin.conf b/test.dockerapp/logstash/pipelines/stdin.conf
new file mode 100644
index 0000000..af24607
--- /dev/null
+++ b/test.dockerapp/logstash/pipelines/stdin.conf
@@ -0,0 +1,9 @@
+input { 
+  stdin { } 
+}
+
+output {
+  elasticsearch {
+    hosts => ["elasticsearch:9200"] 
+  }
+}
\ No newline at end of file
diff --git a/test.dockerapp/logstash/pipelines/telnet.conf b/test.dockerapp/logstash/pipelines/telnet.conf
new file mode 100644
index 0000000..f73c4ac
--- /dev/null
+++ b/test.dockerapp/logstash/pipelines/telnet.conf
@@ -0,0 +1,30 @@
+input {
+  tcp {
+    port => 5000
+    type => syslog
+  }
+  udp {
+    port => 5000
+    type => syslog
+  }
+}
+
+filter {
+  if [type] == "syslog" {
+    grok {
+      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{host}" ]
+    }
+    date {
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+    }
+  }
+}
+
+output {
+    elasticsearch {
+        hosts => ["elasticsearch:9200"] 
+    }
+    stdout { codec => rubydebug }
+}
\ No newline at end of file
diff --git a/test.dockerapp/parameters.yml b/test.dockerapp/parameters.yml
index 27f39dc..b52e684 100644
--- a/test.dockerapp/parameters.yml
+++ b/test.dockerapp/parameters.yml
@@ -1,5 +1,7 @@
 echo-port: 5678
 etcd-port: 2379
+es-port: 9200
+kibana-port: 5601
 
 etcdWait-platform: dev
 etcdWait-max-retry: 100
@@ -10,4 +12,5 @@ response-text: Hello world custom
 etcdApiVersion: 3.3
 etcd-version: 3.3.11
 image-version: latest
-alpine-version: 3.7
\ No newline at end of file
+alpine-version: 3.7
+elk-version: 6.6.0
\ No newline at end of file