Logstash收集Nginx访问日志

241次阅读

共计 4707 个字符，预计需要花费 12 分钟才能阅读完成。

Logstash 收集 Nginx 访问日志

# 直接 yum 安装：
[root@elk-node1 ~]# yum install nginx  -y
 官方文档：http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format
# 修改配置文件的日志格式：
vim /etc/nginx/nginx.conf  
# 在 http 模块中添加 
          log_format json ‘{“@timestamp”:”$time_iso8601″,’
                          ‘”@version”:”1″,’
                          ‘”client”:”$remote_addr”,’
                          ‘”url”:”$uri”,’
                          ‘”status”:”$status”,’
                          ‘”domain”:”$host”,’
                          ‘”host”:”$server_addr”,’
                          ‘”size”:$body_bytes_sent,’
                          ‘”responsetime”:$request_time,’
                          ‘”referer”: “$http_referer”,’
                          ‘”ua”: “$http_user_agent”‘
              ‘}’;
# 在 server 模块中添加 
access_log  /var/log/nginx/access_json.log  json;
# 修改后的 nginx.conf 文件 
[root@elk-node1 ~]# grep -Ev “#|^&” /etc/nginx/nginx.conf
 
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
 
include /usr/share/nginx/modules/*.conf;
 
events {
    worker_connections 1024;
}
 
http {
    log_format  main  ‘$remote_addr – $remote_user [$time_local] “$request” ‘
                      ‘$status $body_bytes_sent “$http_referer” ‘
                      ‘”$http_user_agent” “$http_x_forwarded_for”‘;
          log_format json ‘{“@timestamp”:”$time_iso8601″,’
                          ‘”@version”:”1″,’
                          ‘”client”:”$remote_addr”,’
                          ‘”url”:”$uri”,’
                          ‘”status”:”$status”,’
                          ‘”domain”:”$host”,’
                          ‘”host”:”$server_addr”,’
                          ‘”size”:$body_bytes_sent,’
                          ‘”responsetime”:$request_time,’
                          ‘”referer”: “$http_referer”,’
                          ‘”ua”: “$http_user_agent”‘
              ‘}’;
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay        on;
    keepalive_timeout  65;
    types_hash_max_size 2048;
 
    include            /etc/nginx/mime.types;
    default_type        application/octet-stream;
 
    include /etc/nginx/conf.d/*.conf;
 
    server {
        listen      80 default_server;
        listen      [::]:80 default_server;
        server_name  _;
        root        /usr/share/nginx/html;
 
        include /etc/nginx/default.d/*.conf;
    access_log  /var/log/nginx/access_json.log  json;
 
        location / {
        }
 
        error_page 404 /404.html;
            location = /40x.html {
        }
 
        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }
 
 
}
# 启动：
[root@controller ~]# systemctl start nginx
root@elk-node1 ~]# ss -lntp|grep 80
LISTEN    0      511          *:80                      *:*                  users:((“nginx”,pid=8045,fd=6),(“nginx”,pid=8044,fd=6),(“nginx”,pid=8043,fd=6))
LISTEN    0      511        :::80                      :::*                  users:((“nginx”,pid=8045,fd=7),(“nginx”,pid=8044,fd=7),(“nginx”,pid=8043,fd=7))

浏览器访问：http://192.168.247.135/
Logstash 收集 Nginx 访问日志

查看 nginx 日志

Logstash 收集 Nginx 访问日志

# 添加 nginx 日志格式到之前 logstash 的 elk-log.yml
[root@elk-node1 ~]# cat /etc/logstash/conf.d/elk_log.conf
input {
    file {
      path => “/var/log/messages”
      type => “system”
      start_position => “beginning”
    }
    file {
      path => “/var/log/elasticsearch/hejianlai.log”
      type => “es-error”
      start_position => “beginning”
      codec => multiline {
          pattern => “^\[“
          negate => true
          what => “previous”
        }
    }
      file {
      path => “/var/log/nginx/access_json.log”
      codec => json
      start_position => “beginning”
      type => “nginx-log”
    }
}
output {
    
    if [type] == “system”{
        elasticsearch {
          hosts => [“192.168.247.135:9200”]
          index => “systemlog-%{+YYYY.MM.dd}”
        }
    }
  
    if [type] == “es-error”{
        elasticsearch {
          hosts => [“192.168.247.135:9200”]
          index => “es-error-%{+YYYY.MM.dd}”
        }
    }
      if [type] == “nginx-log”{
        elasticsearch {
          hosts => [“192.168.247.135:9200”]
          index => “nginx-log-%{+YYYY.MM.dd}”
        }
    }
}

#添加 –configtest 参数检查配置语法是否有误！！！
[root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf –configtest
Configuration OK
# 把之前后台运行的进程 kill 掉重启：
[root@elk-node1 ~]# ps aux|grep elk
root 3248 0.8 6.0 3632960 234924 pts/2 Sl 11:25 1:10 /usr/local/java/jdk1.8.0_171/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xss2048k -Djffi.boot.library.path=/opt/logstash/vendor/jruby/lib/jni -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/logstash/heapdump.hprof -Xbootclasspath/a:/opt/logstash/vendor/jruby/lib/jruby.jar -classpath ::/usr/local/java/jdk1.8.0_171/lib:/usr/local/java/jdk1.8.0_171/jre/lib -Djruby.home=/opt/logstash/vendor/jruby -Djruby.lib=/opt/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main –1.9 /opt/logstash/lib/bootstrap/environment.rb logstash/runner.rb agent -f /etc/logstash/conf.d/elk_log.conf
root 8135 0.0 0.0 112704 976 pts/0 S+ 13:38 0:00 grep –color=auto elk
[root@elk-node1 ~]# kill -9 3248
You have new mail in /var/spool/mail/root
[root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf &
[1] 8178

首先在 es 插件中我们能看到 nginx-log 的索引

Logstash 收集 Nginx 访问日志

设置 kibana

Logstash 收集 Nginx 访问日志

：

正文完

星哥玩云-微信公众号