ELK 处理分析日志(nginx,syslog)

官方网站:

环境:

CentOS 7.1 x64

elasticsearch-2.3.2

logstash-2.3.2(或

logstash-all-plugins-2.3.1)

kibana-4.5.0

nginx-1.10.0

redis-3.0.7

elasticsearch 192.168.8.101-103

logstash,nginx 192.168.8.105

kibana,redis 192.168.8.254

Elasticsearch集群

请参看

Logstash收集日志

mkdir -p 

/opt/logstash-2.3.2/{config,logs}

cat >

/opt/logstash-2.3.2/config/

first-pipeline.conf

 <<HERE

input {

    file {

        path => "/var/log/nginx/*_access"

        start_position => beginning

    }

}

filter {

    grok {

        match => { "message" => "%{COMBINEDAPACHELOG}"}

    }

    date {

        match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]

 

    } 

    geoip {

        source => "clientip"

    }

}

output {

    elasticsearch {

        hosts => ["192.168.8.10:9200"]

        index => "nginx-%{+YYYY.MM.DD}"

    }

}

HERE

screen -dmS logstash /opt/logstash-2.3.2/bin/logstash agent -f /opt/logstash-2.3.2/config/first-pipeline.conf -l /opt/logstash-2.3.2/logs/logstash.log

提示: screen比nohup功能强大得多，有兴趣的朋友可以参看

Kibana

mv 

/opt/kibana-4.5.0-linux-x64/config/kibana.yml{,.default}

cat  >

/opt/kibana-4.5.0-linux-x64/config/kibana.yml <<HERE

server.port: 5601

server.host: "192.168.8.254"

elasticsearch.url: "http://192.168.8.10:9200"

kibana.index: ".kibana"

HERE

kibana默认监听在所有接口的5601端口，可以设置指定值来指定,最主要的两个参数是

elasticsearch.url #合法的elasticsearch主机查询URL

kibana.index #kibana中的检索主键

http://192.168.8.254:5061

Discover pannel是专门过滤字段的，这里选中agent,和clientip，右边会将检索到的记录高亮显示

日志字段检索重排

示例二:redis日志流中转

Nginx日志格式定制

log_format logstash '$http_host $server_addr $remote_addr [$time_local] "$request" '

                    '$request_body $status $body_bytes_sent "$http_referer" "$http_user_agent" '

                    '$request_time $upstream_response_time';

access_log  /var/log/nginx/www.jlive.com_access  logstash;

Logstash agent将日志导入到redis

cat >/opt/logstash-2.3.2/config/logstash_agent.conf <<HERE

input {

        file {

                type => "nginx_access"

                path => ["/var/log/nginx/www.jlive.com_access"]

                start_position => beginning

        }

}

output {

        redis {

                host => "192.168.8.254"

                data_type => "list"

                key => "logstash:redis"

        }

 

}

HERE

 

screen -dmS logstash /opt/logstash-2.3.2/bin/logstash -f /opt/logstash-2.3.2/config/logstash_agent.conf -l /opt/logstash-2.3.2/logs/logstash_agent.log

Logstash indexer将日志从redis传送到elasticsearch主机组

cat >/opt/logstash-2.3.2/config/logstash_indexer.conf <<HERE

input {

        redis {

                host => "192.168.8.254"

                data_type => "list"

                key => "logstash:redis"

                type => "redis-input"

        }

}

filter {

    grok {

        match => { "message" => "%{WORD:http_host} %{URIHOST:api_domain} %{IP:inner_ip} %{IP:lvs_ip} \[%{HTTPDATE:timestamp}\] \"%{WORD:http_verb} %{URIPATH:baseurl}(?:\?%{NOTSPACE:request}|) HTTP/%{NUMBER:http_version}\" (?:-|%{NOTSPACE:request}) %{NUMBER:http_status_code} (?:%{NUMBER:bytes_read}|-) %{QS:referrer} %{QS:agent} %{NUMBER:time_duration:float} (?:%{NUMBER:time_backend_response:float}|-)"}

    }

    geoip {

        source => "clientip"

    }

    kv {

        prefix => "request."

        field_split => "&"

        source => "request"

    }

    urldecode {

        all_fields => true

    }

}

output {

        elasticsearch {

                hosts => ["192.168.8.10:9200"]

                index => "nginx-%{+YYYY.MM.DD}"

        } 

} 

HERE

/opt/logstash-2.3.2/bin/logstash -f /opt/logstash-2.3.2/config/logstash_indexer.conf -l /opt/logstash-2.3.2/logs/logstash_indexer.log

示例三: syslog

cat >/opt/logstash-2.3.2/config/logstash_rsyslog.conf <<HERE

input {

    tcp {

        port => 514

        type => syslog

    }

    udp {

        port => 514

        type => syslog

    }

}

filter {

    if [type] == "syslog" {

        grok {

            match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }

            add_field => [ "received_at", "%{@timestamp}" ]

            add_field => [ "received_from", "%{host}" ]

        }

        date {

            match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]

        }

    }

}

output {

    elasticsearch { 

    hosts => ["localhost:9200"] 

    index => "rsyslog-%{+YYYY.MM.DD}"

    }

    stdout { codec => rubydebug }

 

}

HERE

root@jlive:~#netstat -tunlp|grep 514

tcp6       0      0 :::514                  :::*                    LISTEN      10314/java          

udp6       0      0 :::514                  :::*                                10314/java 

启动logstash后会监听在tcp/udp的514端口，模拟系统日志

telnet localhost 514

复制如下内容

Dec 23 12:11:43 louis postfix/smtpd[31499]: connect from unknown[95.75.93.154]Dec 23 14:42:56 louis named[16000]: client 199.48.164.7#64817: query (cache) 'amsterdamboothuren.com/MX/IN' deniedDec 23 14:30:01 louis CRON[619]: (www-data) CMD (php /usr/share/cacti/site/poller.php >/dev/null 2>/var/log/cacti/poller-error.log)Dec 22 18:28:06 louis rsyslogd: [origin software="rsyslogd" swVersion="4.2.0" x-pid="2253" x-info="http://www.rsyslog.com"] rsyslogd was HUPed, type 'lightweight'.

 

  root@jlive:~#/opt/logstash-2.3.2/bin/logstash -f /opt/logstash-2.3.2/config/logstash_rsyslog.conf 

Settings: Default pipeline workers: 4

Pipeline main started

{

             "message" => "Dec 23 12:11:43 louis postfix/smtpd[31499]: connect from unknown[95.75.93.154]\r",

            "@version" => "1",

          "@timestamp" => "2016-12-23T04:11:43.000Z",

                "host" => "0:0:0:0:0:0:0:1",

                "port" => 60655,

                "type" => "syslog",

    "syslog_timestamp" => "Dec 23 12:11:43",

     "syslog_hostname" => "louis",

      "syslog_program" => "postfix/smtpd",

          "syslog_pid" => "31499",

      "syslog_message" => "connect from unknown[95.75.93.154]\r",

         "received_at" => "2016-05-11T09:15:58.159Z",

       "received_from" => "0:0:0:0:0:0:0:1"

}

{

             "message" => "Dec 23 14:42:56 louis named[16000]: client 199.48.164.7#64817: query (cache) 'amsterdamboothuren.com/MX/IN' denied\r",

            "@version" => "1",

          "@timestamp" => "2016-12-23T06:42:56.000Z",

                "host" => "0:0:0:0:0:0:0:1",

                "port" => 60655,

                "type" => "syslog",

    "syslog_timestamp" => "Dec 23 14:42:56",

     "syslog_hostname" => "louis",

      "syslog_program" => "named",

          "syslog_pid" => "16000",

      "syslog_message" => "client 199.48.164.7#64817: query (cache) 'amsterdamboothuren.com/MX/IN' denied\r",

         "received_at" => "2016-05-11T09:15:58.160Z",

       "received_from" => "0:0:0:0:0:0:0:1"

}

{

             "message" => "Dec 23 14:30:01 louis CRON[619]: (www-data) CMD (php /usr/share/cacti/site/poller.php >/dev/null 2>/var/log/cacti/poller-error.log)\r",

            "@version" => "1",

          "@timestamp" => "2016-12-23T06:30:01.000Z",

                "host" => "0:0:0:0:0:0:0:1",

                "port" => 60655,

                "type" => "syslog",

    "syslog_timestamp" => "Dec 23 14:30:01",

     "syslog_hostname" => "louis",

      "syslog_program" => "CRON",

          "syslog_pid" => "619",

      "syslog_message" => "(www-data) CMD (php /usr/share/cacti/site/poller.php >/dev/null 2>/var/log/cacti/poller-error.log)\r",

         "received_at" => "2016-05-11T09:15:58.161Z",

       "received_from" => "0:0:0:0:0:0:0:1"

}

{

             "message" => "Dec 22 18:28:06 louis rsyslogd: [origin software=\"rsyslogd\" swVersion=\"4.2.0\" x-pid=\"2253\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed, type 'lightweight'.\r",

            "@version" => "1",

          "@timestamp" => "2016-12-22T10:28:06.000Z",

                "host" => "0:0:0:0:0:0:0:1",

                "port" => 60655,

                "type" => "syslog",

    "syslog_timestamp" => "Dec 22 18:28:06",

     "syslog_hostname" => "louis",

      "syslog_program" => "rsyslogd",

      "syslog_message" => "[origin software=\"rsyslogd\" swVersion=\"4.2.0\" x-pid=\"2253\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed, type 'lightweight'.\r",

         "received_at" => "2016-05-11T09:15:59.515Z",

       "received_from" => "0:0:0:0:0:0:0:1"

}