fix index name remove date in pattern, drop filebeat event, add default to ignore_older

filebeat-docker.yml

@@ -7,6 +7,11 @@ filebeat.autodiscover:
   providers:
     - type: docker
       hints.enabled: true
+      hints.default_config:
+        type: container
+        paths:
+          - /var/lib/docker/containers/${data.container.id}/*.log
+        ignore_older: 24h
 
 processors:
   - add_cloud_metadata: ~
@@ -16,7 +21,14 @@ processors:
       host: "unix:///var/run/docker.sock"
   - add_host_metadata: ~
   - dissect:
-      tokenizer: '%{nginx.remote_addr} - %{nginx.remote_user} [%{nginx.time}] %{nginx.host} "%{nginx.request}" %{nginx.status|integer} %{nginx.http_referer} "%{nginx.http_user_agent}" %{nginx.http_x_forwarded_for} %{nginx.request_id} "%{nginx.geoip_country_name}" %{nginx.geoip_country_code} %{nginx.geoip_latitude} %{nginx.geoip_longitude}'
+      tokenizer: '%{nginx.remote_addr} - %{nginx.remote_user} [%{nginx.time}] %{nginx.host} "%{nginx.request}" %{nginx.status|integer} %{nginx.http_referer} "%{nginx.http_user_agent}" %{nginx.http_x_forwarded_for} %{nginx.request_id} "%{nginx.geoip_country_name}" %{nginx.geoip_country_code} %{nginx.geoip.lat|double} %{nginx.geoip.lon|double}'
+      target_prefix: ""
+      field: "message"
+      when:
+        equals:
+          container.name: nginxfront
+  - dissect:
+      tokenizer: '%{php.time} [%{php.status}] %{php.info}: %{php.message}, client: %{php.client}, server: %{php.server}, request: "%{php.request}", upstream: "%{php.upstream}", host: "%{php.host}"'
       target_prefix: ""
       field: "message"
       when:
@@ -32,6 +44,10 @@ processors:
       when:
         equals:
           container.name: nginxfront
+  - drop_event:
+      when:
+        equals:
+          container.name: filebeat
 
 setup:
   kibana:
@@ -50,10 +66,11 @@ setup:
         type: keyword
       - name: kubernetes.container.name
         type: keyword
+      - name: nginx.geoip
+        type: geo_point
 
   dashboards:
     enabled: false
-    #index: "filebeat-%{[agent.version]}-*"
 
 output.elasticsearch:
   hosts: '[${ELASTICSEARCH_HOSTS:elasticsearch:9200}]'
@@ -62,7 +79,8 @@ output.elasticsearch:
   ssl:
     verification_mode: none
   indices:
-    - index: "filebeat-%{[agent.version]}-%{[container.name]:common}-%{+yyyy.MM.dd}"
+    #- index: "filebeat-%{[agent.version]}-%{[container.name]:common}-%{+yyyy.MM.dd}"
+    - index: "filebeat-%{[agent.version]}-%{[container.name]:common}"
 
 
 #logging.json: true

filebeat-kubernetes.yml

@@ -22,6 +22,7 @@ filebeat.autodiscover:
         type: container
         paths:
           - /var/log/containers/*-${data.kubernetes.container.id}.log
+        ignore_older: 24h
 
 processors:
   - add_cloud_metadata: ~
@@ -46,6 +47,10 @@ processors:
           to: "container.name"
       ignore_missing: true
       fail_on_error: false
+  - drop_event:
+      when:
+        equals:
+          container.name: filebeat
 
 setup:
   kibana:
@@ -67,7 +72,6 @@ setup:
 
   dashboards:
     enabled: false
-    #index: "filebeat-%{[agent.version]}-*"
 
 output.elasticsearch:
   hosts: '[${ELASTICSEARCH_HOSTS:elasticsearch:9200}]'
@@ -76,7 +80,8 @@ output.elasticsearch:
   ssl:
     verification_mode: none
   indices:
-    - index: "filebeat-%{[agent.version]}-%{[container.name]:common}-%{+yyyy.MM.dd}"
+    #- index: "filebeat-%{[agent.version]}-%{[container.name]:common}-%{+yyyy.MM.dd}"
+    - index: "filebeat-%{[agent.version]}-%{[container.name]:common}"
 
 
 #logging.json: true