Logstash 日志采集工具

    Logstash是一個開源的用于收集，分析和存儲日志的工具。

    Logstash: Logstash服務的組件，用于處理傳入的日志。不過是基于Elasticsearch配置使用。

    Elasticsearch: 存儲所有日志。

    做個示例監控test-http、tomcat和test-api等系統的運行狀態，輸出error信息到elasticsearch。需要在每一臺test服務器上安裝配置。

    一、下載

logstash：https://download.elastic.co/logstash/logstash/logstash-2.4.1.zip

unzip logstash-2.4.1.zip

    二、log4j的配置

### 設置###

log4j.rootLogger = debug,stdout,D,E

### 輸出信息到控制抬 ###

log4j.appender.stdout = org.apache.log4j.ConsoleAppender

log4j.appender.stdout.Target = System.out

log4j.appender.stdout.layout = org.apache.log4j.PatternLayout

log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n 

### 輸出DEBUG 級別以上的日志到  path/logs/error.log ###

log4j.appender.D = org.apache.log4j.DailyRollingFileAppender

log4j.appender.D.File = logs/log.log

log4j.appender.D.Append = true

log4j.appender.file.ImmediateFlush=false

log4j.appender.file.BufferedIO=true

log4j.appender.file.BufferSize=8192

log4j.appender.D.Threshold = DEBUG

log4j.appender.D.layout = org.apache.log4j.PatternLayout

log4j.appender.D.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} %c [%t]-[%p] %m%n

### 輸出ERROR 級別以上的日志到  path/logs/error.log ###

log4j.appender.E = org.apache.log4j.DailyRollingFileAppender

log4j.appender.E.File =logs/error.log

log4j.appender.E.Append = true

log4j.appender.E.Threshold = ERROR

log4j.appender.E.layout = org.apache.log4j.PatternLayout

log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} %c [%t]-[%p] %m%n

    三、test系統的logstash配置，注意logs目錄位置和ElasticSearch的Hosts

# vim test-api.conf

input {

    file {

        path => "/opt/test-api/logs/error.log"

        start_position => "beginning"

        type => "test-api"

    }

}

filter {

    multiline {

        pattern => "^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}"

        negate => true

        what => "previous"

    }

    grok {

        match => { "message" => "%{TIMESTAMP_ISO8601:datetime} %{NOTSPACE:clazz} \[%{NOTSPACE:thread-id}\]\-\[%{LOGLEVEL:level}\] %{GREEDYDATA:msg}" }

    }

}

output {

    elasticsearch {

        hosts => ["10.207.101.100:9200","10.207.101.101:9200","10.207.101.102:9200"]

        index => "test_logs-%{+YYYYMMdd}"

        document_type => "logs"

    }

}

# cat test-http.conf

input {

    file {

        path => "/opt/test-http/logs/error.log"

        start_position => "beginning"

        type => "test"

    }

}

filter {

    multiline {

        pattern => "^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}"

        negate => true

        what => "previous"

    }

    grok {

        match => { "message" => "%{TIMESTAMP_ISO8601:datetime} %{NOTSPACE:clazz} \[%{NOTSPACE:thread-id}\]\-\[%{LOGLEVEL:level}\] %{GREEDYDATA:msg}" }

    }

}

output {

    elasticsearch {

        hosts => ["10.207.101.100:9200","10.207.101.101:9200","10.207.101.102:9200"]

        index => "test_logs-%{+YYYYMMdd}"

        document_type => "logs"

    }

}

# cat test_logs-tomcat.conf

input {

    file {

        path => "/opt/server/tomcat/logs/web.log"

        start_position => "beginning"

        type => "tomcat"

    }

}

filter {

    multiline {

        pattern => "^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}"

        negate => true

        what => "previous"

    }

    grok {

        match => { "message" => "%{TIMESTAMP_ISO8601:datetime} \[%{NOTSPACE:thread-id}\] %{LOGLEVEL:level}\s*%{NOTSPACE:clazz} \- %{GREEDYDATA:msg}" }

    }

}

output {

    elasticsearch {

        hosts => ["10.207.101.100:9200","10.207.101.101:9200","10.207.101.102:9200"]

        index => "test_logs-%{+YYYYMMdd}"

        document_type => "logs"

    }

}

設置啟動

# cat  test-start.sh

nohup bin/logstash -f test-http.conf > /dev/null 2>&1 &

nohup bin/logstash -f test-api.conf > /dev/null 2>&1 &

nohup bin/logstash -f test_logs-tomcat.conf > /dev/null 2>&1 &