ELK搭建

目标

搭建依赖于ELK kafka Spring-boot项目的日志收集服务

组件

• Spring-boot项目
• Elasticsearch
• Kibana
• Logstash
  目录结构：
  

过程

注意所用到的kafka topic均为：dev-log-test

Spring-boot项目

pom

   <dependencies>
        <dependency>
            <groupId>org.springframework.bootgroupId>
            <artifactId>spring-boot-starter-webartifactId>
            <exclusions>
                <exclusion>
                    <artifactId>spring-boot-starter-loggingartifactId>
                    <groupId>org.springframework.bootgroupId>
                exclusion>
                <exclusion>
                    <artifactId>logback-classicartifactId>
                    <groupId>ch.qos.logbackgroupId>
                exclusion>
            exclusions>
        dependency>

        <dependency>
            <groupId>org.springframework.bootgroupId>
            <artifactId>spring-boot-starter-log4j2artifactId>
        dependency>

        <dependency>
            <groupId>org.apache.kafkagroupId>
            <artifactId>kafka-clientsartifactId>
            <version>2.2.0version>
        dependency>

        <dependency>
            <groupId>org.projectlombokgroupId>
            <artifactId>lombokartifactId>
            <optional>trueoptional>
        dependency>
        <dependency>
            <groupId>org.springframework.bootgroupId>
            <artifactId>spring-boot-starter-testartifactId>
            <scope>testscope>
        dependency>
    dependencies>

log4j2.xml

<Configuration status="warn" monitorInterval="30" strict="true"	schema="Log4J-V2.2.xsd">
    <Appenders>
        
        <Console name="Console" target="SYSTEM_OUT">
            
            <ThresholdFilter level="trace" onMatch="ACCEPT" onMismatch="DENY" />
            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss,SSS}:%4p %t (%F:%L) - %m%n" />
        Console>
        
        <Kafka name="KAFKA" topic="dev-log-test">
            <ThresholdFilter level="info" onMatch="ACCEPT" onMismatch="DENY" />
            <PatternLayout pattern="%date %message"/>
            <Property name="bootstrap.servers">127.0.0.1:9092Property>
        Kafka>
    Appenders>

    <Loggers>
        <Root level="info"> 
            <AppenderRef ref="Console" />
            <AppenderRef ref="KAFKA"/>
        Root>

        <Logger name="com.csii" level="trace" additivity="false">
            <AppenderRef ref="Console" />
            <AppenderRef ref="KAFKA"/>
        Logger>

        <Logger name="org.apache.kafka" level="INFO" /> 
    Loggers>
Configuration>

application.properties

logging.config=classpath:log4j2.xml

Kafka

此处下载：kafka

Step 1 下载解压

tar -xzf kafka_2.12-2.5.0.tgz
cd kafka_2.12-2.5.0

运行Zookeeper、启动Kafka、创建Topic

//traverse to extracted directory of Apache Kafka
./bin/zookeeper-server-start.sh config/zookeeper.properties  //start the zookeeper first
./bin/kafka-server-start.sh config/server.properties  //start Kafka
./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic dev-log-test  //create a topic

LogStash

下载

LogStash 下载

解压配置

进入LogStash的解压文件夹

vim logstash-kafka.conf

输入下面内容，保存。

input {
    kafka {
            bootstrap_servers => "localhost:9092"
            topics => ["dev-log-test"]
    }
}

output {
   elasticsearch {
      hosts => ["localhost:9200"]
      index => "dev-log-test"
      workers => 1
    }
}

启动LogStash

./bin/logstash -f logstash-kafka.conf

Elasticsearch

下载
启动

.bin/elasticsearch

Kibana

下载安装启动

下载

.bin/kibana

配置

配置index
url: http://localhost:5601/app/kibana#/management/kibana/index_patterns?_g=()

Enjoy

localhost:5061

参考
检索参考
apache-kafka/kafka-elasticsearch-logstash-example
kafka
elk download