kafka

1、producer

-- coding:utf-8 --

"""
Author:Yjx
Time:2019-8-17 14:22
"""

from kafka import KafkaProducer
from kafka import KafkaConsumer
from kafka.errors import KafkaError
from threading import Thread
import time

class kafkaProducer(object):

def __init__(self, kafkatopic, kafkahost, kafkaport):
    self.kafkatopic = kafkatopic
    self.kafkahost = kafkahost
    self.kafkaport = kafkaport
    bootstrap_server = []
    for host in self.kafkahost:
        """auto_offset_reset='latest',# 消费kafka中最近的数据，如果设置为earliest则消费最早的数据，不管这些数据是否消费； consumer_timeout_ms:设置自动退出时间（毫秒）；"""
        bootstrap_server.append('{kafka_host}:{kafka_port}'.format(kafka_host=host, kafka_port=self.kafkaport))
    self.producer = KafkaProducer(bootstrap_servers=bootstrap_server)


def sendMsg(self):
    try:
        for i in range(100):
            startTime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            if i % 2 == 0:
                message_key = 'a'
            elif i % 3 == 0:
                message_key = 'b'
            else:
                message_key = 'c'
            # message = 'send_message_' + str(i+1) + ':' + startTime
            message = 'message_' + str(i)
            self.producer.send(self.kafkatopic, key=message_key.encode('utf-8'), value=message.encode('utf-8'),
                               timestamp_ms=None)
            print(message_key, message, startTime)
            self.producer.flush()
            time.sleep(0.5)
        self.producer.close()
        print('****over****')
    except KafkaError as e:
        print('KafkaError:', e)

if __name__=='__main__':

para_host = ['xxx', 'xxx', 'xxx']
para_port = 9092
producer = kafkaProducer("topic-1", para_host, para_port)
producer.sendMsg()

2、consumer

from redis_try import *
from kafka import KafkaConsumer
from kafka.errors import KafkaError
from threading import Thread
import time

class kafkaConsumer(object):

def __init__(self, kafkatopic, kafkahost, kafkaport):
    self.kafkatopic = kafkatopic
    self.kafkahost = kafkahost
    self.kafkaport = kafkaport
    bootstrap_server = []
    for host in self.kafkahost:
        bootstrap_server.append('{kafka_host}:{kafka_port}'.format(kafka_host=host, kafka_port=self.kafkaport))
    self.consumer = KafkaConsumer(self.kafkatopic, bootstrap_servers=bootstrap_server,
                                  auto_offset_reset='latest', consumer_timeout_ms=10000)

def getMsg(self):
    try:
        for msg in self.consumer:
            # print('get_key_>>>>', msg.key.decode('utf-8'),
            #       ';get_message_>>>>', msg.value.decode('utf-8'))
            msg_value = msg.value.decode('utf-8')
            user, num, label = r.user_count(msg.key.decode('utf-8'))
            print("user:{} click:{} label:{} value:{}".format(user, num, label, msg_value))
        print('****done****')
    except KafkaError as e:
        print('KafkaError:', e)

if name == '__main__':

para_host = ['xxx', 'xxx', 'xxx']
para_port = 9092
r = redis_server(host='xxx', ports=[a, b, c], password='zzzz')# 调redis
consumer = kafkaConsumer("topic-1", para_host, para_port)
messages = consumer.getMsg()