电话日志分析callLog（二）

一、前端实现 -- 按照时间段查询通话记录
-----------------------------------------
    1.完善calllog的dao类calllog.class
    ----------------------------------------------
       

 package com.it18zhang.ssm.domain;

        /**
         * calllog的domain类 -- 标准javabean
         */
        public class Calllog {

            private String caller;
            private String callee;
            private String callTime;
            private String callDuration;
            //是否是主叫
            private boolean flag;

            public boolean isFlag() {
                return flag;
            }

            public void setFlag(boolean flag) {
                this.flag = flag;
            }

            public String getCaller() {
                return caller;
            }

            public void setCaller(String caller) {
                this.caller = caller;
            }

            public String getCallee() {
                return callee;
            }

            public void setCallee(String callee) {
                this.callee = callee;
            }

            public String getCallTime() {
                return callTime;
            }

            public void setCallTime(String callTime) {
                this.callTime = callTime;
            }

            public String getCallDuration() {
                return callDuration;
            }

            public void setCallDuration(String callDuration) {
                this.callDuration = callDuration;
            }
        }

    2.通话日志时间查询dao类CalllogRange.class
    -----------------------------------------------
       

 package com.it18zhang.ssm.domain;

        /**
         */
        public class CalllogRange {

            private String startPoint ;
            private String endPoint ;

            public String getStartPoint() {
                return startPoint;
            }

            public void setStartPoint(String startPoint) {
                this.startPoint = startPoint;
            }

            public String getEndPoint() {
                return endPoint;
            }

            public void setEndPoint(String endPoint) {
                this.endPoint = endPoint;
            }

            public String toString() {
                return startPoint + " - " + endPoint ;
            }
        }

    3.完善Service类 CalllogService.class
    ------------------------------------------------------
         

 package com.it18zhang.ssm.service;
          import com.it18zhang.ssm.domain.Calllog;
          import com.it18zhang.ssm.domain.CalllogRange;

          import java.util.List;

          /**
           * Calllog的服务类 -- 用于定制与服务器交互的规则
           */
          public interface CalllogService {

              //查询所有的calllog
              public List findAll();
              /**
               * 按照范围查询通话记录
               */
              public List findCallogs(String call,List list);
          }

    4.完善CalllogServiceImpl类
    -----------------------------------------------
     

   package com.it18zhang.ssm.service.impl;

        import com.it18zhang.ssm.domain.Calllog;
        import com.it18zhang.ssm.domain.CalllogRange;
        import com.it18zhang.ssm.service.CalllogService;
        import com.it18zhang.ssm.util.CalllogUtil;
        import org.apache.hadoop.conf.Configuration;
        import org.apache.hadoop.hbase.HBaseConfiguration;
        import org.apache.hadoop.hbase.TableName;
        import org.apache.hadoop.hbase.client.*;
        import org.apache.hadoop.hbase.util.Bytes;
        import org.springframework.stereotype.Service;

        import java.io.IOException;
        import java.util.*;

        /**
         * CalllogService的实现类
         */
        @Service("calllogService")
        public class CalllogServiceImpl implements CalllogService {

            private Table table;
            public CalllogServiceImpl()
            {
                try {
                    //获取配置文件
                    Configuration conf = HBaseConfiguration.create();
                    //工厂类创建连接
                    Connection conn = ConnectionFactory.createConnection(conf);
                    //get table
                    TableName tbName = TableName.valueOf("call:calllogs");
                    table = conn.getTable(tbName);

                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            /**
             * 查询所有的calllog
             * 全表扫描
             * @return
             */
            public List findAll() {
                List list = new ArrayList();
                try {
                    //扫描
                    Scan scan = new Scan();
                    ResultScanner rs = table.getScanner(scan);
                    Iterator it = rs.iterator();
                    byte[] famliy = Bytes.toBytes("f1");
                    byte[] callerf = Bytes.toBytes("caller");
                    byte[] calleef = Bytes.toBytes("callee");
                    byte[] callTimef = Bytes.toBytes("callTime");
                    byte[] callDurationf = Bytes.toBytes("callDuration");
                    Calllog calllog = null;

                    while (it.hasNext()) {

                        Result next = it.next();
                        String caller = Bytes.toString(next.getValue(famliy, callerf));
                        String callee = Bytes.toString(next.getValue(famliy, calleef));
                        String callTime = Bytes.toString(next.getValue(famliy, callTimef));
                        String callDuration = Bytes.toString(next.getValue(famliy, callDurationf));

                        calllog = new Calllog();
                        calllog.setCaller(caller);
                        calllog.setCallee(callee);
                        calllog.setCallTime(callTime);
                        calllog.setCallDuration(callDuration);

                        list.add(calllog);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
                return list;
            }


            /**
             * 按照范围查询通话记录
             */
            public List findCallogs(String call , List ranges){
                List logs = new ArrayList();
                try {
                    for(CalllogRange range : ranges){
                        Scan scan = new Scan();
                        //设置扫描起始行
                        scan.setStartRow(Bytes.toBytes(CalllogUtil.getStartRowkey(call, range.getStartPoint(),100)));
                        //设置扫描结束行
                        scan.setStopRow(Bytes.toBytes(CalllogUtil.getStopRowkey(call, range.getStartPoint(), range.getEndPoint(),100)));

                        ResultScanner rs = table.getScanner(scan);
                        Iterator it = rs.iterator();
                        byte[] f = Bytes.toBytes("f1");

                        byte[] caller = Bytes.toBytes("caller");
                        byte[] callee = Bytes.toBytes("callee");
                        byte[] callTime = Bytes.toBytes("callTime");
                        byte[] callDuration = Bytes.toBytes("callDuration");
                        Calllog log = null;
                        while (it.hasNext()) {
                            log = new Calllog();
                            Result r = it.next();
                            //rowkey
                            String rowkey = Bytes.toString(r.getRow());
                            String flag = rowkey.split(",")[3] ;
                            log.setFlag(flag.equals("0")?true:false);
                            //caller
                            log.setCaller(Bytes.toString(r.getValue(f, caller)));
                            //callee
                            log.setCallee(Bytes.toString(r.getValue(f, callee)));
                            //callTime
                            log.setCallTime(Bytes.toString(r.getValue(f, callTime)));
                            //callDuration
                            log.setCallDuration(Bytes.toString(r.getValue(f, callDuration)));
                            logs.add(log);
                        }
                    }
                    return logs;
                } catch (Exception e) {
                    e.printStackTrace();
                }
                return null;
            }
        }

    5.编写Callog工具类
    ------------------------------------------
       

 package com.it18zhang.ssm.util;

        import com.it18zhang.ssm.domain.CalllogRange;

        import java.text.DecimalFormat;
        import java.text.SimpleDateFormat;
        import java.util.ArrayList;
        import java.util.Calendar;
        import java.util.List;

        /**
         * calllog工具类
         */
        public class CalllogUtil {


            private static SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
            private static SimpleDateFormat sdfFriend = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");

            //格式化
            private static DecimalFormat df = new DecimalFormat();



            /**
             * 获取hash值,默认分区数100
             */
            public static String getHashcode(String caller, String callTime, int partitions) {
                int len = caller.length();
                //取出后四位电话号码
                String last4Code = caller.substring(len - 4);
                //取出时间单位,年份和月份.
                String mon = callTime.substring(0, 6);
                //
                int hashcode = (Integer.parseInt(mon) ^ Integer.parseInt(last4Code)) % partitions;
                return df.format(hashcode);
            }

            /**
             * 起始时间
             */
            public static String getStartRowkey(String caller, String startTime, int partitions){
                String hashcode = getHashcode(caller, startTime,partitions);
                return hashcode + "," + caller + "," + startTime ;
            }

            /**
             * 结束时间
             */
            public static String getStopRowkey(String caller, String startTime,String endTime, int partitions){
                String hashcode = getHashcode(caller, startTime,partitions);
                return hashcode + "," + caller + "," + endTime ;
            }

            /**
             * 计算查询时间范围
             */
            public static List getCallLogRanges(String startStr ,String endStr){
                try{
                    SimpleDateFormat sdfYMD = new SimpleDateFormat("yyyyMMdd");
                    SimpleDateFormat sdfYM = new SimpleDateFormat("yyyyMM");
                    DecimalFormat df00 = new DecimalFormat("00");

                    //
                    List list = new ArrayList();
                    //字符串时间
                    String startPrefix = startStr.substring(0, 6);

                    String endPrefix = endStr.substring(0, 6);
                    int endDay = Integer.parseInt(endStr.substring(6, 8));
                    //结束点
                    String endPoint = endPrefix + df00.format(endDay + 1);

                    //日历对象
                    Calendar c = Calendar.getInstance();

                    //同年月
                    if (startPrefix.equals(endPrefix)) {
                        CalllogRange range = new CalllogRange();
                        range.setStartPoint(startStr);          //设置起始点

                        range.setEndPoint(endPoint);            //设置结束点
                        list.add(range);
                    } else {
                        //1.起始月
                        CalllogRange range = new CalllogRange();
                        range.setStartPoint(startStr);

                        //设置日历的时间对象
                        c.setTime(sdfYMD.parse(startStr));
                        c.add(Calendar.MONTH, 1);
                        range.setEndPoint(sdfYM.format(c.getTime()));
                        list.add(range);

                        //是否是最后一月
                        while (true) {
                            //到了结束月份
                            if (endStr.startsWith(sdfYM.format(c.getTime()))) {
                                range = new CalllogRange();
                                range.setStartPoint(sdfYM.format(c.getTime()));
                                range.setEndPoint(endPoint);
                                list.add(range);
                                break;
                            } else {
                                range = new CalllogRange();
                                //起始时间
                                range.setStartPoint(sdfYM.format(c.getTime()));

                                //增加月份
                                c.add(Calendar.MONTH, 1);
                                range.setEndPoint(sdfYM.format(c.getTime()));
                                list.add(range);
                            }
                        }
                    }
                    return list ;
                }
                catch(Exception e){
                    e.printStackTrace();
                }
                return null ;
            }

            /**
             * 对时间进行格式化
             */
            public static String formatDate(String timeStr){
                try {
                    return sdfFriend.format(sdf.parse(timeStr));
                } catch (Exception e) {
                    e.printStackTrace();
                }
                return null ;
            }


        }

    6.编写控制器CalllogController.class
    -----------------------------------------
     

   package com.it18zhang.ssm.web.controller;

        import com.it18zhang.ssm.domain.Calllog;
        import com.it18zhang.ssm.domain.CalllogRange;
        import com.it18zhang.ssm.service.CalllogService;
        import com.it18zhang.ssm.service.impl.CalllogServiceImpl;
        import com.it18zhang.ssm.util.CalllogUtil;
        import org.springframework.stereotype.Controller;
        import org.springframework.ui.Model;
        import org.springframework.web.bind.annotation.RequestMapping;
        import org.springframework.web.bind.annotation.RequestMethod;
        import org.springframework.web.bind.annotation.RequestParam;

        import javax.annotation.Resource;
        import java.util.Calendar;
        import java.util.List;

        @Controller
        public class CalllogController {

            @Resource(name="calllogService")
            private CalllogService cs;

            /**
             * 发送参数
             * @param model
             * @return
             */
            @RequestMapping("calllog/findAll")
            public String findAll(Model model)
            {
                List list = cs.findAll();
                model.addAttribute("calllogs", list);
                return "calllog/calllogList";
            }

            /**
             * 跳转到查询界面
             * @return
             */
            @RequestMapping("calllog/toFindCalllogPage")
            public String toFindCalllog()
            {
                return "calllog/findCalllog";
            }

            /**
             * 接受参数
             * @param m
             * @return
             */
            @RequestMapping(value = "calllog/findCalllog", method = RequestMethod.POST)
            public String findCalllog(Model m, @RequestParam("caller") String caller, @RequestParam("startTime") String startTime, @RequestParam("endTime") String endTime)
            {
                List list = CalllogUtil.getCallLogRanges(startTime, endTime);
                List logs = cs.findCallogs(caller,list);
                m.addAttribute("calllogs", logs);
                return "callLog/calllogList" ;
            }
        }

二、编写协处理器模块CalllogCoprossorModel，实现主叫发生的同时即插入被叫记录
--------------------------------------------------------
    1.说明
        因为HbaseCustomer消费者，得到的数据都是主叫数据，没有被叫数据
        所以，需要做一个协处理器，来处理当主叫发生的时候，同时写入被叫的记录

    2.创建协处理器模块CalllogCoprpssorModel，并添加maven依赖
        

        
            4.0.0

            calllog.kafka
            CalllogCustomerModel
            1.0-SNAPSHOT

            

                
                    org.apache.kafka
                    kafka_2.11
                    0.10.0.1
                
                
                    org.apache.hbase
                    hbase-client
                    1.2.4
                
                
                    org.apache.hbase
                    hbase-server
                    1.2.4
                

            

        

    3.新建协处理器区域观察者类com.calllog.coprossor.CalllogRegion
    -------------------------------------
       

 package com.calllog.coprossor;
        import org.apache.hadoop.hbase.Cell;
        import org.apache.hadoop.hbase.CellUtil;
        import org.apache.hadoop.hbase.TableName;
        import org.apache.hadoop.hbase.client.*;
        import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
        import org.apache.hadoop.hbase.coprocessor.ObserverContext;
        import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
        import org.apache.hadoop.hbase.regionserver.InternalScanner;
        import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
        import org.apache.hadoop.hbase.util.Bytes;

        import java.io.IOException;
        import java.util.ArrayList;
        import java.util.List;

        /**
         * 协处理器
         */
        public class CalllogRegion extends BaseRegionObserver {

            //被叫引用id
            private static final String REF_ROW_ID = "refrowid" ;
            //通话记录表名
            private static final String CALL_LOG_TABLE_NAME = "call:calllogs" ;

            /**
             * 每次put数据之后调用
             */
            @Override
            public void postPut(ObserverContext e, Put put, WALEdit edit, Durability durability) throws IOException {
                super.postPut(e, put, edit, durability);

                TableName tName = TableName.valueOf("call:calllogs");

                TableName tName1 = e.getEnvironment().getRegion().getRegionInfo().getTable();

                if (tName.equals(tName1)) {
                    //得到rowkey
                    String rowKey = Bytes.toString(put.getRow());

                    String[] strs = rowKey.split(",");
                    //如果是被叫，直接返回
                    if (strs[3].equals("1")) {
                        return;
                    }
                    //66,15733218888,20181002071335,0,18332561111,063
                    //取出相应的值
                    String caller = strs[1];
                    String time = strs[2];
                    String callee = strs[4];
                    String duration = strs[5];
                    //计算区域号
                    String hash = CalllogUtil.getHashcode(callee, time, 100);
                    String newRowKey = hash + "," + callee + "," + time + "," + "1" + "," + caller + "," + duration;

                    //开始put数据
                    Put p = new Put(Bytes.toBytes(newRowKey));
                    p.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("refrowid"), Bytes.toBytes(rowKey));

                    Table tb = e.getEnvironment().getTable(tName);
                    tb.put(p);
                    System.out.println("put over");
                }
            }

            /**
             * get之后调用 -- 实现被叫查询时，直接返回主叫的记录
             * 将之前get的结果替换
             */
            @Override
            public void postGetOp(ObserverContext e, Get get, List results) throws IOException {
                //获得表名
                String tableName = e.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();

                //判断表名是否是ns1:calllogs
                if(!tableName.equals("call:calllogs")){
                    super.preGetOp(e, get, results);
                }
                else{
                    //得到rowkey
                    String rowkey = Bytes.toString(get.getRow());
                    //
                    String[] arr = rowkey.split(",");
                    //主叫
                    if(arr[3].equals("0")){
                        super.postGetOp(e, get, results);
                    }
                    //被叫
                    else{
                        //得到主叫方的rowkey
                        String refrowid = Bytes.toString(CellUtil.cloneValue(results.get(0)));
                        //
                        Table tt = e.getEnvironment().getTable(TableName.valueOf("call:calllogs"));
                        Get g = new Get(Bytes.toBytes(refrowid));
                        Result r = tt.get(g);
                        List newList = r.listCells();
                        results.clear();
                        results.addAll(newList);
                    }
                }
            }


            /**
             * 扫描之后调用
             */
            @Override
            public boolean postScannerNext(ObserverContext e, InternalScanner s, List results, int limit, boolean hasMore) throws IOException {

                boolean b = super.postScannerNext(e, s, results, limit, hasMore);

                //新集合
                List newList = new ArrayList();

                //获得表名
                String tableName = e.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();

                //判断表名是否是ns1:calllogs
                if (tableName.equals(CALL_LOG_TABLE_NAME)) {
                    Table tt = e.getEnvironment().getTable(TableName.valueOf(CALL_LOG_TABLE_NAME));
                    for(Result r : results){
                        //rowkey
                        String rowkey = Bytes.toString(r.getRow());
                        String flag = rowkey.split(",")[3] ;
                        //主叫
                        if(flag.equals("0")){
                            newList.add(r) ;
                        }
                        //被叫
                        else{
                            //取出主叫号码
                            byte[] refrowkey = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes(REF_ROW_ID)) ;
                            Get newGet = new Get(refrowkey);
                            newList.add(tt.get(newGet));
                        }
                    }
                    results.clear();
                    results.addAll(newList);
                }
                return b ;
            }
        }

    4.新建工具类CoprossorUtil
    ----------------------------------------------
      

  package com.calllog.coprossor;

        import java.text.DecimalFormat;
        import java.text.SimpleDateFormat;
        import java.util.ArrayList;
        import java.util.Calendar;
        import java.util.List;

        /**
         * Created by Administrator on 2017/4/13.
         */
        public class CalllogUtil {

            private static SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
            private static SimpleDateFormat sdfFriend = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");

            //格式化
            private static DecimalFormat df = new DecimalFormat();

            /**
             * 获取hash值,默认分区数100
             */
            public static String getHashcode(String caller, String callTime,int partitions) {
                int len = caller.length();
                //取出后四位电话号码
                String last4Code = caller.substring(len - 4);
                //取出时间单位,年份和月份.
                String mon = callTime.substring(0, 6);
                //
                int hashcode = (Integer.parseInt(mon) ^ Integer.parseInt(last4Code)) % partitions;
                return df.format(hashcode);
            }

            /**
             * 起始时间
             */
            public static String getStartRowkey(String caller, String startTime, int partitions){
                String hashcode = getHashcode(caller, startTime,partitions);
                return hashcode + "," + caller + "," + startTime ;
            }

            /**
             * 结束时间
             */
            public static String getStopRowkey(String caller, String startTime,String endTime, int partitions){
                String hashcode = getHashcode(caller, startTime,partitions);
                return hashcode + "," + caller + "," + endTime ;
            }

            /**
             * 对时间进行格式化
             */
            public static String formatDate(String timeStr){
                try {
                    return sdfFriend.format(sdf.parse(timeStr));
                } catch (Exception e) {
                    e.printStackTrace();
                }
                return null ;
            }
        }

    5.打包部署
      a.注册协处理器,并分发到所有hbase节点
          [hbase-site.xml]
            
              hbase.coprocessor.region.classes
              com.calllog.coprossor.CalllogRegion
            

      b.将打好的jar包分发到所有节点的/hbase/lib目录下

      c.重启hbase集群

      d.进入hbase shell,重建表"call:calllogs" "f1" "f2"
        $hbase> create 'call:calllogs','f1','f2';

三、生成日志，收集日志

    1.开启kafka集群
        [s200 s300 s400]
        $> /soft/kafka/bin/kafka-server-start.sh -daemon /soft/kafka/config/server.properties

    2.在s100和s200上启动flume，开始收集日志
        $s100> flume-ng agent -f /soft/flume/conf/calllog.conf -n a1 &
        $s200> flume-ng agent -f /soft/flume/conf/calllog.conf -n a1 &

    3.开启kafka的消费者--hbase[进入share/calllog目录下，找到CalllogCustomerModel.jar包]
        $> java -cp CalllogCustomerModel.jar calllog.kafka.hbase.customer.HbaseCustomer

    4.开启日志生成工具
        $calllog> ./calllog.sh

    5.运行ssm模块，查看webapp界面显示