Flink Kafka 实时日志处理

一、 环境准备

kafka安装

参考Mac springboot 集成 kafka

flink安装

参考Mac上安装flink及快速搭建demo

二、spring 集成

pom配置

        <dependency>
            <groupId>org.apache.flinkgroupId>
            <artifactId>flink-javaartifactId>
            <version>1.10.0version>
        dependency>

        
        <dependency>
            <groupId>org.apache.flinkgroupId>
            <artifactId>flink-clients_2.12artifactId>
            <version>1.10.0version>
        dependency>

        
        <dependency>
            <groupId>org.apache.flinkgroupId>
            <artifactId>flink-streaming-java_2.12artifactId>
            <version>1.10.0version>
            <scope>providedscope>
        dependency>

        
        <dependency>
            <groupId>org.apache.flinkgroupId>
            <artifactId>flink-connector-kafka-0.11_2.12artifactId>
            <version>1.10.0version>
        dependency>

flink job

package com.example.demo.flink;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import org.apache.flink.util.Collector;
import org.springframework.stereotype.Component;

import java.util.Properties;

@Component
public class KafkaConsumerStreamingJob {
    public void flinkJob() throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "localhost:9092");
        properties.setProperty("group.id", "myGroup");
        DataStream<Tuple2<String, String>> stream = env
                .addSource(new FlinkKafkaConsumer011<>("abc123", new SimpleStringSchema(), properties))
                .flatMap(new ServerLogFormat());
        stream.print();
        env.execute("kafka test");
    }

    public static class ServerLogFormat implements FlatMapFunction<String, Tuple2<String, String>> {
        @Override
        public void flatMap(String value, Collector<Tuple2<String, String>> out) throws Exception {
            String name = value.split(" ")[3];
            System.out.println("--收到消息,flink处理job--");
            out.collect(new Tuple2<>(name, value));
        }
    }
}

test

package com.example.demo;

import com.example.demo.flink.KafkaConsumerStreamingJob;
import com.example.demo.kafka.KafkaSender;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.ConfigurableApplicationContext;

@SpringBootApplication
@EnableCaching
public class DemoApplication {

    public static void main(String[] args) {
//        SpringApplication.run(DemoApplication.class, args);
        ConfigurableApplicationContext context = SpringApplication.run(DemoApplication.class, args);
        KafkaConsumerStreamingJob streamingJob = context.getBean(KafkaConsumerStreamingJob.class);
        try {
            streamingJob.flinkJob();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}

结果

--收到消息,flink处理job--
1> (CCProfileMonitor::setStreamEventHandler,{"@timestamp":"2020-04-03T01:31:00.602Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.2.4","topic":"abc123"},"beat":{"name":"MyMac-MB2","hostname":"MyMac-MB2","version":"6.2.4"},"source":"/var/log/corecaptured.log","offset":3497932,"message":"Apr 03 09:30:55 CCProfileMonitor::setStreamEventHandler Woken up by notifyd.","prospector":{"type":"log"}})

三、参考文献

Flink Kafka 实时日志处理实例