python多线程批量插入redis数据

多线程+线程池+pipeline

import redis
from concurrent.futures import ThreadPoolExecutor, as_completed
import time

# 数据库连接配置
REDIS_HOST = 'localhost'
REDIS_PORT = 6379

# 创建Redis连接池
pool = redis.ConnectionPool(host=REDIS_HOST, port=REDIS_PORT)

# 插入数据的函数
def insert_data_to_db(db_number, data_count):
    # 使用连接池创建Redis连接
    r = redis.Redis(connection_pool=pool, db=db_number)
    
    # 创建pipeline实例
    pipeline = r.pipeline()

    start_time = time.time()  # 记录开始时间
    for i in range(data_count):
        key = f"key_{db_number}_{i}"
        value = f"value_{db_number}_{i}"
        pipeline.set(key, value)
    
    # 执行pipeline中的命令
    pipeline.execute()
    
    end_time = time.time()  # 记录结束时间

    # 计算耗时并返回结果
    elapsed_time = end_time - start_time
    return f"Database {db_number}: Inserted {data_count} items in {elapsed_time:.2f} seconds."

# 插入数据的数量
DATA_COUNT = 3000

# 线程池大小
POOL_SIZE = 32

# 使用线程池
with ThreadPoolExecutor(max_workers=POOL_SIZE) as executor:
    # 提交任务到线程池
    futures = [executor.submit(insert_data_to_db, db_number, DATA_COUNT) for db_number in range(32)]

    # 等待并获取结果
    for future in as_completed(futures):
        print(future.result())

print("Data insertion completed for all databases.")