Docker部署ELFK采集日志实战
Docker部署ELFK采集日志实战
一、安装Docker
二、安装Docker-compose
三、添加阿里云docker镜像加速
四、安装ELK
五、安装FileBeat
六、安装Elastalert报警
一、安装Docker
1、安装utils工具
[root@localhost ~]# yum -y install yum-utils
2、安装config-manager工具；
[root@localhost ~]# yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
3、安装docker服务
[root@localhost ~]# yum install docker-ce docker-ce-cli containerd.io
4、检查docker
[root@localhost ~]# docker version
Client:
Version: 1.13.1
API version: 1.26
Package version: docker-1.13.1-209.git7d71120.el7.centos.x86_64
Go version: go1.10.3
Git commit: 7d71120/1.13.1
Built: Wed Mar 2 15:25:43 2022
OS/Arch: linux/amd64
Server:
Version: 1.13.1
API version: 1.26 (minimum version 1.12)
Package version: docker-1.13.1-209.git7d71120.el7.centos.x86_64
Go version: go1.10.3
Git commit: 7d71120/1.13.1
Built: Wed Mar 2 15:25:43 2022
OS/Arch: linux/amd64
Experimental: false
二、安装Docker-compose
1.查看Linux版本
[root@localhost ~]# uname -s
Linux
[root@localhost ~]# uname -m
x86_64
2.从git上拉取docker-compose
docker-compose官网 https://github.com/docker/compose/releases
** 注意根据 Linux 版本，选择合适的版本安装**
[root@localhost ~]# sudo curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-linux-x86_64" \
-o /usr/local/bin/docker-compose
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 15.4M 100 15.4M 0 0 153k 0 0:01:42 0:01:42 --:--:-- 3220k
3.授权docker-compose
[root@localhost ~]# chmod +x /usr/local/bin/docker-compose
4.建立软连接
[root@localhost ~]# ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
5.校验是否安装成功
[root@localhost ~]# docker-compose --version
docker-compose version 1.24.1, build 4667896b
三、添加阿里云docker镜像加速
通过修改daemon配置文件/etc/docker/daemon.json来使用加速器
1.创建一个目录
[root@localhost ~]# sudo mkdir -p /etc/docker
2.配置阿里云镜像加速
[root@localhost ~]# sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://leh1vw5w.mirror.aliyuncs.com"]
}
EOF
3.启用阿里云镜像加速
[root@localhost ~]# sudo systemctl daemon-reload
4.重启docker
[root@localhost ~]# sudo systemctl restart docker
四、安装ELK
1.创建ES挂载数据卷
[root@localhost ~]# mkdir -p /depy/docker_data/elasticsearch/plugins
[root@localhost ~]# mkdir -p/depy/docker_data/elasticsearch/data
2.创建Logstash配置文件
[root@localhost ~]# mkdir -p /depy/docker_data/elasticsearch/logstash/
[root@localhost ~]# cd /depy/docker_data/elasticsearch/logstash/
[root@localhost logstash]# touch logstash.conf
3.编写logstash.conf 配置
input {
beats {
port => 5044
}
}
# 2022年10月17日添加过滤器
filter{
grok {
# 自定义时间的正则表达式
pattern_definitions => {"MYSELFTIMESTAMP" => "20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND})"}
# 正则配置异常日志的内容
match => {"message" => ["%{MYSELFTIMESTAMP:createTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%
{DATA:threadName}%{SPACE}%{JAVACLASS:javaClass}%{SPACE}:%{SPACE}%{GREEDYDATA:message}"]}
# 将上面%{GREEDYDATA:message} 标签覆盖到message上
overwrite => ["message"]
}
mutate{
remove_field => ["host"]
remove_field => ["agent"]
remove_field => ["ecs"]
remove_field => ["tags"]
# remove_field => ["fields"]
# remove_field => ["@timestamp"]
remove_field => ["@version"]
remove_field => ["input"]
remove_field => ["log"]
}
}
output {
if [fields][project]== "fire" {
elasticsearch {
#这里可以配置多个
hosts => ["http://192.168.101.94:9200"]
# 索引
index => "dashu-park-error-log"
# 类型
# document_type => "fire"
# 主键
# document_id => "%{id}"
}
# mail插件，可以用来报警发邮件
#email {
#port => "25"
#address => "smtp.163.com"
#username => "[email protected]"
#password => "TSQVPDVKKEMEYWTX"
#authentication => "plain"
#use_tls => false
#from => "[email protected]"
#subject => "dashu-park-zone项目中有error日志信息"
#to => "[email protected]"
#via => "smtp"
#body => "错误日志： \n %{message} "
#}
}
#redis {
#host => ["192.168.101.94"] #这个是标明redis服务的地址
#port => 9001
#codec => plain
#db => 1 #redis中的数据库，select的对象
#key => elk_log #redis中的键值
#data_type => list #一般就是list和channel
#password => DaShuRedisoRhFG9xT6kXIZl5b
#timeout => 5
#workers => 1
#}
stdout { codec => rubydebug }
}
4.创建ELK部署脚本文件
[root@localhost ~]# mkdir -p /depy/docker_data/elasticsearch/elk
[root@localhost ~]# cd /depy/docker_data/elasticsearch/elk/
[root@localhost elk]# touch docker-compose.yml
5.创建ELK使用的容器网络
[root@localhost ~]# docker network create --driver bridge --subnet 172.18.0.0/16 --gateway 172.18.0.1 depy_default
6.编写ELK部署脚本
version: '3'
services:
elasticsearch:
image: elastic/elasticsearch:7.1.0
container_name: elasticsearch
environment:
#集群名称为elasticsearch
- "cluster.name=elasticsearch"
#单节点启动
- "discovery.type=single-node"
#jvm内存分配为512MB
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
volumes:
- /depy/docker_data/elasticsearch/plugins:/usr/share/elasticsearch/plugins
- /depy/docker_data/elasticsearch/data:/usr/share/elasticsearch/data
ports:
- 9200:9200
kibana:
image: elastic/kibana:7.1.0
container_name: kibana
links:
#配置elasticsearch域名为es
- elasticsearch:es
depends_on:
- elasticsearch
environment:
#因为上面配置了域名，所以这里可以简写为http://es:9200
- "elasticsearch.hosts=http://es:9200"
ports:
- 5601:5601
logstash:
image: elastic/logstash:7.1.0
container_name: logstash
volumes:
- /depy/docker_data/elasticsearch/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
depends_on:
- elasticsearch
links:
- elasticsearch:es
ports:
- 5044:5044
#filebeat:
#image: elastic/filebeat:7.1.0
#container_name: filebeat
#restart: always
#depends_on:
# - logstash
#volumes:
# - /depy/docker_data/elasticsearch/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
# - /depy/logs/dashu-park-zone/:/usr/share/filebeat/logs/
#links:
# - logstash:logstash
networks:
default:
external:
name: depy_default
7.启动脚本部署ELK
[root@localhost elk]# docker-compose -f docker-compose.yml up -d
注意：到docker-compose存放目录下执行命令
五、安装FileBeat
1.创建FileBeat挂载数据卷
[root@localhost ~]# mkdir -p /depy/logs/dashu-park-fire/
[root@localhost ~]# mkdir -p /depy/docker_data/filebeat/
2.创建FileBeat配置文件
[root@localhost ~]# cd /depy/docker_data/filebeat/
[root@localhost ~]# touch filebeat.yml
3.创建FileBeat配置文件
#=========================== Filebeat inputs =============================
filebeat.inputs:
# Each - is an input. Most options can be set at the input level, so
# you can use different inputs for various configurations.
# Below are the input specific configurations.
- type: log
# Change to true to enable this input configuration.
enabled: true
# Paths that should be crawled and fetched. Glob based paths.
paths: #配置多个日志路径
# 如果是docker部署，这里就是docker容器内的路径，而且需要和宿主机的路径进行映射
- /usr/share/filebeat/logs/error.log
#- /usr/share/filebeat/logs/info.log
#指定自定义标签
# tags: ["dashu-park-fire"]
#指定自定义字段
fields:
project: "fire" #字段1
#hostName: "192.168.101.94" #字段2
# Exclude lines. A list of regular expressions to match. It drops the lines that are
# matching any regular expression from the list.
#exclude_lines: ['^DBG']
# Include lines. A list of regular expressions to match. It exports the lines that are
# matching any regular expression from the list.
#include_lines: ['^ERR', '^WARN']
# Exclude files. A list of regular expressions to match. Filebeat drops the files that
# are matching any regular expression from the list. By default, no files are dropped.
#exclude_files: ['.gz$']
# Optional additional fields. These fields can be freely picked
# to add additional information to the crawled log files for filtering
#fields:
# level: debug
# review: 1
### Multiline options
# Multiline can be used for log messages spanning multiple lines. This is common
# for Java Stack Traces or C-Line Continuation
# The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
#multiline.pattern: ^\[
# Defines if the pattern set under pattern should be negated or not. Default is false.
#multiline.negate: false
# Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
# that was (not) matched before or after or as long as a pattern is not matched based on negate.
# Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
#multiline.match: after
multiline:
#多行匹配规则
pattern: '^[[:space:]]+(at|\.{3})\b&^Caused by:'
#将不匹配的规则的行合并在一起
negate: true
#合并到匹配规则的上一行末尾
match: after
#================================ Outputs =====================================
#----------------------------- Logstash output --------------------------------
output.logstash:
# The Logstash hosts #配多个logstash使用负载均衡机制
hosts: ["192.168.101.94:5044"]
#使用了负载均衡
loadbalance: true
# Optional SSL. By default is off.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
#output.elasticsearch:
# hosts: 192.168.101.94:9200
# indices:
# - index: "filebeat-%{+yyyy.MM.dd}"
#output.redis:
#hosts: ["192.168.101.94:9001"]
#password: DaShuRedisoRhFG9xT6kXIZl5b
#key: "filebeat-redis"
#db: 1
#timeout: 60
# 控制台输出
#output.console:
#pretty: true
#enable: true
3.创建FileBeat部署脚本
[root@localhost ~]# mkdir -p /depy/docker_data/filebeat
[root@localhost ~]# cd /depy/docker_data/filebeat
[root@localhost filebeat]# touch docker-compose-filebeat.yml
4.编写FileBeat部署脚本
[root@localhost filebeat]# vim docker-compose-filebeat.yml
5.编写FileBeat部署脚本
# 版本
version: "3"
# 服务
services:
filebeat:
# 容器名称
container_name: filebeat
# 主机名称
hostname: filebeat
# 镜像
image: elastic/filebeat:7.1.0
# 重启机制
restart: always
# 启动用户
user: root
# 持久化挂载
volumes:
# 日志文件夹映射到容器中[作为数据源]
- /depy/logs/dashu-park-fire:/usr/share/filebeat/logs/
# 采集日志配置映射配置文件到容器中
- /depy/docker_data/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
# 使用主机网络模式
network_mode: host
6.启动FileBeat部署脚本
[root@localhost filebeat]# docker-compose -f docker-compose.yml up -d
六、安装Elastalert报警
1.下载elastalert
elastalert 官网 https://github.com/bitsensor/elastalert.git
可以从192.168.101.94 服务器 ````/depy/docker_data/elastalert```，复制过去
git clone https://github.com/bitsensor/elastalert.git
# 安装依赖包
[root@nacos-host ~]#yum install -y wget gcc openssl-devel epel-release git python3 python3-devel unzip telnet lrzsz
[root@nacos-host ~]#
[root@nacos-host ~]# pip install -r ./requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
[root@nacos-host ~]# python setup.py install
[root@nacos-host ~]# mkdir -p /depy/docker_data/elastalert/config/
这四个命令作用如下：
1、elastalert
用于根据报警规则进行报警。
2、elastalert-create-index
该命令在执行后，会创建一个索引，elastalert会把执行记录存放到这个索引中。在默认情况下，索引名称为elastalert_status。该索引有4个type，都有自己的时间戳，因此可以使用Kibana来查看该索引的内容。
3、elastalert-rule-from-kibana
该命令用于从Kibana已保存的仪表盘中读取filtering设置，帮助生成配置文件。
4、elastalert-test-rule
该命令用于测试自定义配置中的rule设置。
另一种安装方式
[root@nacos-host ~]# yum -y install wget openssl openssl-devel gcc gcc-c++
[root@nacos-host ~]# wget https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz
[root@nacos-host ~]# tar xf Python-3.6.9.tgz
[root@nacos-host ~]# cd Python-3.6.9./configure --prefix=/usr/local/python --with-openssl
2.配置elastalert启动参数elastalert.yaml
[root@nacos-host ~]# mkdir -p /depy/docker_data/elastalert/config/
[root@nacos-host elastalert]# touch elastalert.yaml
[root@nacos-host elastalert]# chmod 777 elastalert.yaml
[root@nacos-host elastalert]# vim elastalert.yaml
# The elasticsearch hostname for metadata writeback
# Note that every rule can have its own elasticsearch host
es_host: 192.168.101.94
# The elasticsearch port
es_port: 9200
# This is the folder that contains the rule yaml files
# Any .yaml file will be loaded as a rule
rules_folder: rules
# How often ElastAlert will query elasticsearch
# The unit can be anything from weeks to seconds
run_every:
seconds: 5
# ElastAlert will buffer results from the most recent
# period of time, in case some log sources are not in real time
buffer_time:
minutes: 1
# Optional URL prefix for elasticsearch
#es_url_prefix: elasticsearch
# Connect with TLS to elasticsearch
#use_ssl: True
# Verify TLS certificates
#verify_certs: True
# GET request with body is the default option for Elasticsearch.
# If it fails for some reason, you can pass 'GET', 'POST' or 'source'.
# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport
# for details
#es_send_get_body_as: GET
# Option basic-auth username and password for elasticsearch
#es_username: someusername
#es_password: somepassword
# The index on es_host which is used for metadata storage
# This can be a unmapped index, but it is recommended that you run
# elastalert-create-index to set a mapping
writeback_index: elastalert_status
# If an alert fails for some reason, ElastAlert will retry
# sending the alert until this time period has elapsed
alert_time_limit:
days: 2
3.配置Docker ElastAlert的启动参数config.json
[root@nacos-host elastalert]# touch config.json
[root@nacos-host elastalert]# chmod 777 config.json
[root@nacos-host elastalert]# vim config.json
{
"appName": "elastalert-server",
"port": 3030,
"wsport": 3333,
"elastalertPath": "/opt/elastalert",
"verbose": false,
"es_debug": false,
"debug": false,
"rulesPath": {
"relative": true,
"path": "/rules"
},
"templatesPath": {
"relative": true,
"path": "/rule_templates"
},
"es_host": "192.168.101.94",
"es_port": 9200,
"writeback_index": "elastalert_status"
}
4.创建路由规则文件夹 rules
[root@nacos-host ~]# mkdir -p /depy/docker_data/elastalert/rules
5.创建路由模板规则文件夹 rule_templates
[root@nacos-host ~]# mkdir -p /depy/docker_data/elastalert/rule_templates
6.创建邮件认证文件smtp_auth_file.yaml
[root@nacos-host elastalert]# touch smtp_auth_file.yaml
[root@nacos-host elastalert]# chmod 777 smtp_auth_file.yaml
[root@nacos-host elastalert]# vim smtp_auth_file.yaml
#邮箱用户名:
user: [email protected]
##不是邮箱密码，是设置的SMTP密码
password: TSQVPDVKKEMEYWTX
7.创建elastalert启动脚本docker-compose-elastalert.yml
https://hub.docker.com/r/bitsensor/elastalert/tags
[root@nacos-host elastalert]# vim docker-compose-elastalert.yml
[root@nacos-host elastalert]# chmod 777 docker-compose-elastalert.yml
version: '3'
services:
elastalert:
image: bitsensor/elastalert:3.0.0-beta.1
container_name: elastalert
environment:
- ELASTICSEARCH_HOST=192.168.101.94
- ELASTICSEARCH_PORT=9200
- TZ=Asia/Shanghai
volumes:
- /depy/docker_data/elastalert/config/elastalert.yaml:/opt/elastalert/config.yaml
- /depy/docker_data/elastalert/config/config.json:/opt/elastalert-server/config/config.json
- /depy/docker_data/elastalert/rules:/opt/elastalert/rules
- /depy/docker_data/elastalert/rule_templates:/opt/elastalert/rule_templates
- /depy/docker_data/elastalert/smtp_auth_file.yaml:/opt/elastalert/smtp_auth.yaml
# - /depy/docker_data/elastalert/elastalert_modules/:/opt/elastalert/elastalert_modules/
ports:
- 3030:3030
- 3333:3333
networks:
default:
external:
name: depy_default
8.启动elastalert脚本
[root@nacos-host elastalert]# docker-compose -f docker-compose-elastalert.yml up -d
9.Docker离线安装Kibana插件elastalert
https://github.com/bitsensor/elastalert-kibana-plugin/releases/tag/1.0.4
下载 Kibana插件elastalert-kibana-plugin-1.0.4-7.1.0.zip，或者从192.168.101.94服务复制过去
进去kibana容器，离线安装插件
[root@nacos-host opt]# wget https://github.com/bitsensor/elastalert-kibana-plugin/releases/tag/1.0.4/elastalert-kibana-plugin-1.0.4-7.1.0.zip
[root@nacos-host ~]# docker cp ./elastalert-kibana-plugin-1.0.4-7.1.0.zip：/opt/kibana/
[root@nacos-host ~]# docker exec -it kibana /bin/bash
bash-4.2$ /usr/share/kibana/bin/kibana-plugin install file:opt/kibana/elastalert-kibana-plugin-1.0.4-7.1.0.zip
bash-4.2$ cd /usr/share/kibana/config
bash-4.2$ vim kibana.yml
# 在末尾添加告警的服务配置
elastalert-kibana-plugin.serverHost: 192.168.101.94
elastalert-kibana-plugin.serverPort: 3030
# 退出
bash-4.2$ exit
10.重启kibana
[root@nacos-host ~]# docker restart kibana
11.到kibana界面添加报警配置
再kibana左侧找到 Elastalert 图表，然后点击 Create Rule
命名为 dashu_park_log_elastalert_mail
es_host: 192.168.101.94
es_port: 9200
name: dashu_park_error_log
use_strftine_index: true
type: frequency
index: dashu-park-log #匹配的索引名
num_events: 1
timeframe:
minutes: 1
filter:
- term:
level: "ERROR"
realert:
minutes: 1 #出现次数，1方便测试
alert:
- "email"
email:
- "[email protected]" # 接收邮件
# Email格式
email_format: html
smtp_host: "smtp.163.com"
smtp_port: 465
# https 证书
smtp_ssl: true
smtp_auth_file: /opt/elastalert/smtp_auth.yaml
from_addr: "[email protected]" # 发送邮件
# 标题
alert_subject: "日志报警通知!"
alert_text_type: alert_text_only
# 网页格式
alert_text: "日志报警通知!
立马前往Kibana查看
告警详情
@timestamp:
{} ES主键是:_id:
{} ES索引是:_index:
{} ES类型是:_type:
{} 发生异常的项目:project
{} 发生异常的类:javaClass
{} 发生异常时间:createTime
{} 发生异常的线程:threadName
{} 异常消息:message:
{}
"
# 邮箱模板参数
alert_text_args:
- "@timestamp"
- _id
- _index
- _type
- fields.project
- javaClass
- createTime
- threadName
- message