flume 自定义开发HttpSink

由于业务需求flume 读取数据，然后直接发送到应用系统。所以只好自己开发了一个 Http Sink 进行发送数据，将接收的数据发送到应用系统的restful API。并支持断点续传。

也就是应用程序宕机以后，重新启动，数据还可以继续传送。不丢失数据。数据发送失败也会继续重试。

#好了废话不多说，上代码。

发送Http请求我使用 okHttp ，代码如下：

mport com.alibaba.fastjson.JSONArray;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import okhttp3.*;
import org.apache.flume.*;
import org.apache.flume.conf.Configurable;
import org.apache.flume.sink.AbstractSink;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class LogCollector extends AbstractSink implements Configurable{

    private Logger LOG = LoggerFactory.getLogger(LogCollector.class);
    private String hostname;
    private String port;
    private int batchSize;
    private String postUrl;
    public LogCollector() {
        LOG.info("LogCollector start...");
    }

    @Override
    public void configure(Context context) {
        hostname = context.getString("hostname");
        Preconditions.checkNotNull(hostname, "hostname must be set!!");
        port = context.getString("port");
        Preconditions.checkNotNull(port, "port must be set!!");
        batchSize = context.getInteger("batchSize", 100);
        Preconditions.checkNotNull(batchSize > 0, "batchSize must be a positive number!!");
        postUrl = "http://"+hostname+":"+port+/路径url;
    }

    @Override
    public void start() {
        super.start();
    }

    @Override
    public void stop() {
        super.stop();
    }

    public Status process() throws EventDeliveryException {
        Status result = Status.READY;
        Channel channel = getChannel();
        Transaction transaction = null;
        try {
            transaction = channel.getTransaction();
            transaction.begin();
            Event event = null;
            String content = null;
            List contents=new ArrayList<>();
            for (int i = 0; i < batchSize; i++) {
                event = channel.take();
                if (event != null) {//对事件进行处理
                    content = new String(event.getBody());
                    contents.add(content);
                } else {
                    result = Status.BACKOFF;
                    break;
                }
            }
            if (contents.size() > 0) {
                Response response= postJson(postUrl, JSONArray.toJSON(contents).toString());
                if(response!=null && response.isSuccessful()){
                    transaction.commit();//通过 commit 机制确保数据不丢失
                }
            }else
            {
                transaction.commit();
            }
        } catch (Exception e) {
            try {
                transaction.rollback();
            } catch (Exception e2) {
                LOG.error("Exception in rollback. Rollback might not have been" +
                        "successful.", e2);
            }
            LOG.error("Failed to commit transaction." +
                    "Transaction rolled back.", e);
            Throwables.propagate(e);
        }finally {
            if (transaction != null) {
                transaction.close();
                LOG.debug("close Transaction");
            }
        }
        return result;
    }

    /**
     * post请求，json数据为body
     *
     * @param url
     * @param json
     */
    public Response postJson(String url, String json) {
        OkHttpClient client = new OkHttpClient();
        RequestBody body = RequestBody.create(MediaType.parse("application/json"), json);
        Request request = new Request.Builder()
                .url(url)
                .post(body)
                .build();
        Response response = null;
        try {
            response = client.newCall(request).execute();
            if (!response.isSuccessful()){
                LOG.info("request was error");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return response;
    }
}

所需依赖如下：

  
        
            com.squareup.okhttp3
            okhttp
            3.6.0
        

        
        
            com.alibaba
            fastjson
            1.2.5
        

配置如下：

a1.sinks = epp

##use epp sink
a1.sinks.epp.type = 包名.LogCollector
a1.sinks.epp.channel = c1
a1.sinks.epp.hostname = localhost
a1.sinks.epp.port = 8080
a1.sinks.epp.batchSize = 100

源码传送门：https://github.com/527515025/bigdata/tree/master/flume/flumeMysqlSink/src/main/java/com/us/flume