第17章 ELK采集Pod日志

两种模式

1.边车模式

2.ds模式

边车模式

nginx.conf

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;

events {
    worker_connections  1024;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    log_format json '{ "time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"http_user_agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr",'
                          '"up_host": "$upstream_http_host",'
                          '"upstream_time": "$upstream_response_time",'
                          '"request_time": "$request_time"'
    ' }';

    access_log  /var/log/nginx/access.log  json;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}

filebeat.yaml

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["es-svc:9200"]
  index: "nginx-access-%{[agent.version]}-%{+yyyy.MM}"

setup.ilm.enabled: false
setup.template.enabled: false

nginx-dp

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-dp
  labels:
    app: nginx-dp
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx-dp
  template:
    metadata:
      labels:
        app: nginx-dp
    spec:
      volumes:
      - name: data
        emptyDir: {}
      - name: nginx-cm
        configMap:
          name: nginx-cm
      - name: filebeat-cm
        configMap:
          name: filebeat-cm
      containers:
      - name: nginx-dp
        image: luffy.com/base/nginx:latest
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 80
          name: nginx-port
        volumeMounts:
        - name: data
          mountPath: /var/log/nginx
        - name: nginx-cm
          mountPath: /etc/nginx/nginx.conf
          subPath: nginx.conf

      - name: filebeat-dp
        image: luffy.com/base/filebeat:7.9.1
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: data
          mountPath: /var/log/nginx
        - name: filebeat-cm
          mountPath: /usr/share/filebeat/filebeat.yml
          subPath: filebeat.yml

svc

apiVersion: v1
kind: Service
metadata:
  name: nginx-svc
  labels:
    app: nginx-svc
spec:
  ports:
  - port: 80
    name: web
    targetPort: 80
  selector:
    app: nginx-dp

ds模式

nginx.conf

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;

events {
    worker_connections  1024;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    log_format json '{ "time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"http_user_agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr",'
                          '"up_host": "$upstream_http_host",'
                          '"upstream_time": "$upstream_response_time",'
                          '"request_time": "$request_time"'
    ' }';

    access_log  /var/log/nginx/access.log  json;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}

nginx-dp

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-dp
  labels:
    app: nginx-dp
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx-dp
  template:
    metadata:
      labels:
        app: nginx-dp
    spec:
      volumes:
      - name: nginx-data
        hostPath:
          path: /data/log/
          type: DirectoryOrCreate
      - name: nginx-cm
        configMap:
          name: nginx-cm
      containers:
      - name: nginx-dp
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        image: luffy.com/base/nginx:latest
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 80
          name: nginx-port
        volumeMounts:
        - name: nginx-data
          mountPath: /var/log/nginx
          subPathExpr: $(POD_NAME)
        - name: nginx-cm
          mountPath: /etc/nginx/nginx.conf
          subPath: nginx.conf

filebeat.yaml

filebeat.yaml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /data/log/*/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["es-svc:9200"]
  index: "nginx-access-%{[agent.version]}-%{+yyyy.MM}"

setup.ilm.enabled: false
setup.template.enabled: false

更新: 2024-09-12 15:19:53