从零开始搭建一个 ELKB 日志收集系统

$ yum install java-1.8.0

$ rpm --import http://packages.elastic.co/GPG-KEY-elasticsearch
$ echo '[elasticsearch-2.x]
name=Elasticsearch repository for 2.x packages
baseurl=http://packages.elastic.co/elasticsearch/2.x/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
' | tee /etc/yum.repos.d/elasticsearch.repo
$ yum install elasticsearch

$ vim /etc/yum.repos.d/logstash.repo
# 添加以下内容
[logstash-2.4]
name=logstash repository for 2.2 packages
baseurl=http://packages.elasticsearch.org/logstash/2.2/centos
gpgcheck=1
gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch
enabled=1
# 安装
$ yum install logstash

$ vim /etc/yum.repos.d/kibana.repo
# 添加以下内容
[kibana-4.6]
name=Kibana repository for 4.4.x packages
baseurl=http://packages.elastic.co/kibana/4.4/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
# 安装
$ yum install kibana

$ rpm --import http://packages.elastic.co/GPG-KEY-elasticsearch

$ vim /etc/yum.repos.d/elastic-beats.repo
# 添加以下内容
[beats]
name=Elastic Beats Repository
baseurl=https://packages.elastic.co/beats/yum/el/$basearch
enabled=1
gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
gpgcheck=1
# 安装
$ yum install filebeat

network.host: localhost

$ curl localhost:9200/_cat/indices?v # 查看当前所有的 index
health status index pri rep docs.count docs.deleted store.size pri.store.size # 没有任何 index
# 创建 shakespeare 索引，并设置 mapping 信息
# speaker 字段和 play_name 不需要分析，Elastic Search 默认会拆分字符串中的每个词并进行索引
$ curl -XPUT http://localhost:9200/shakespeare -d '
{
 "mappings" : {
  "_default_" : {
   "properties" : {
    "speaker" : {"type": "string", "index" : "not_analyzed" },
    "play_name" : {"type": "string", "index" : "not_analyzed" },
    "line_id" : { "type" : "integer" },
    "speech_number" : { "type" : "integer" }
   }
  }
 }
}
';
$ curl localhost:9200/_cat/indices?v # 查看索引
health status index       pri rep docs.count docs.deleted store.size pri.store.size
yellow open   shakespeare   5   1          0            0       260b           260b
# 下载数据，并将数据集 load 进索引中
$ wget https://www.elastic.co/guide/en/kibana/3.0/snippets/shakespeare.json
$ curl -XPOST 'localhost:9200/shakespeare/_bulk?pretty' --data-binary @shakespeare.json
# 以上操作完成后，Elastic Search 中就已经有了我们 load 的所有数据，并建立好了索引，我们可以开始查询了
# 查询一下含有 'man' 这个词的 text_entry
$ curl -s 'localhost:9200/shakespeare/_search?q=text_entry:man&pretty=1&size=20' | jq '.hits.hits | .[]._source.text_entry'
"man."
"Man?"
"man."
"Why, man?"
"Worthy man!"
"Every man,"
"complete man."
"married man?"
"melancholy man."
"Speak, man."
"Why, man?"
"What, man?"
"prave man."
"Speak, man."
"Why, man?"
"So man and man should be;"
"O, the difference of man and man!"
"The young man is an honest man."
"A gross fat man."
"plain-dealing man?"

$ vim /etc/filebeat/filebeat.yml
...
prospectors:
  -
    paths:
      - /var/log/nginx/access.log
    # 找到 document_type 字段，取消注释，这个字段会告诉 Logstash 日志的类型，对应 Logstash 中的 type 字段
    document_type: nginx
...
# 默认输出为 Elastic Search，注释掉，使用 Logstash
logstash:
  hosts: ["IP:5044"] # 注意更改这里的 IP

$ vim /etc/logstash/conf.d/nginx.conf
input {
  beats {
    port => 5044
  }
}

filter {
  if [type] == "nginx" { # 这里的type是日志类型，我们在后面的filebeat中设定
    grok {
      match => { "message" => "%{COMBINEDAPACHELOG} %{QS:gzip_ratio}" } # 使用自带的pattern即可，注意空格
      remove_field => ["beat", "input_type", "message", "offset", "tags"] # filebeat添加的字段，我们不需要
    }
    
    # 更改匹配到的字段的数据类型
    mutate {
      convert => ["response", "integer"]
      convert => ["bytes", "integer"]
      convert => ["responsetime", "float"]
    }
    
    # 指定时间戳字段以及具体的格式
    date {
      match => ["timestamp", "dd/MMM/YYYY:HH:mm:ss Z"]
      remove_field => ["timestamp"]
    }
  }
}

outpugst {
  elasticsearch {
    hosts => [ "localhost:9200" ]
    index => "%{type}-%{+YYYY.MM.dd}" # index 中含有时间戳
  }
}

curl -XPUT localhost:9200/_template/nginx -d '
{
  "template": "nginx*",
  "mappings": {
    "_default_": {
      "properties": {
        "clientip": {
          "type": "string",
          "index": "not_analyzed"
        },
        "ident": {
          "type": "string"
        },
        "auth": {
          "type": "string"
        },
        "verb": {
          "type": "string"
        },
        "request": {
          "type": "string"
        },
        "httpversion": {
          "type": "string"
        },
        "rawrequest": {
          "type": "string" 
        },
        "response": {
          "type": "string"
        },
        "bytes": {
          "type": "integer"
        },
        "referrer": {
          "type": "string"
        },
        "agent": {
          "type": "string"
        },
        "gzip_ratio": {
          "type": "string"
        }
      }
    }
  }
}
'