redis系列之-更新问题解决方案(数据一致性校正)

redis系列之-更新问题解决方案

---

redis做缓存减轻mysql数据库压力的同时,更新会产生双库数据不一致的情况,我这里采取的方案是:

mysql更新数据后,删除redis缓存(这也是大多数企业采用的),用户下次访问时没有命中缓存,会去访问mysql,并产生新的redis缓存.

虽然在大多数情况下是安全的,但并不是绝对安全,也会产生不一致情况,所以要进行定时数据校正.
策略:

先到mysql中查询数据
删除redis中的数据
将mysql中的数据写入redis

1.1设置定时任务
这里为了方便大家理解,我放的项目初始化的代码,你只需要看定时任务注册就好

from apscheduler.executors.pool import ThreadPoolExecutor
from apscheduler.schedulers.background import BackgroundScheduler
from flask import Flask

from schedule.statistic import fix_statistic


def create_flask_app(config, enable_config_file=False):
    """
    创建Flask应用
    """
    app = Flask(__name__)
    app.config.from_object(config)
    if enable_config_file:
        from utils import constants
        # 加载隐私配置
        app.config.from_envvar(constants.GLOBAL_SETTING_ENV_NAME, silent=True)

    return app


def create_app(config, enable_config_file=False):
    """
    创建flask应用 并 初始化各组件

    :param config: 配置类
    :param enable_config_file: 是否允许运行环境中的配置文件覆盖已加载的配置信息
    :return: flask应用
    """
    app = create_flask_app(config, enable_config_file)

    # 添加自定义正则转换器
    from utils.converters import register_converters
    register_converters(app)

    # 创建redis哨兵
    from redis.sentinel import Sentinel
    _sentinel = Sentinel(app.config['REDIS_SENTINELS'])
    # 获取redis主从连接对象
    app.redis_master = _sentinel.master_for(app.config['REDIS_SENTINEL_SERVICE_NAME'])
    app.redis_slave = _sentinel.slave_for(app.config['REDIS_SENTINEL_SERVICE_NAME'])

    # 创建redis集群
    from rediscluster import StrictRedisCluster
    app.redis_cluster = StrictRedisCluster(startup_nodes=app.config['REDIS_CLUSTER'])

    # 配置myql数据库
    from models import db
    db.init_app(app)

    # 配置日志
    from utils.logging import create_logger
    create_logger(app)

    # 限流器
    from utils.limiter import limiter as lmt
    lmt.init_app(app)

    # 创建Snowflake ID worker
    from utils.snowflake.id_worker import IdWorker
    app.id_worker = IdWorker(app.config['DATACENTER_ID'],
                             app.config['WORKER_ID'],
                             app.config['SEQUENCE'])

    # 添加定时任务
    executor = ThreadPoolExecutor(max_workers=3)
    executors = {
        'default':executor
    }
    app.scheduler = BackgroundScheduler(executors=executors)
    # 添加任务  每天3点校正数据
    # app.scheduler.add_job(fix_statistic, 'cron', hour=3)
    # date只用于测试
    app.scheduler.add_job(fix_statistic, 'date')
    app.scheduler.start()


    # 添加请求钩子
    from utils.middlewares import jwt_authentication
    app.before_request(jwt_authentication)

    # 注册用户模块蓝图
    from .resources.user import user_bp
    app.register_blueprint(user_bp)

    # 注册新闻模块蓝图
    from .resources.news import news_bp
    app.register_blueprint(news_bp)

    # 注册搜索模块蓝图
    from .resources.search import search_bp
    app.register_blueprint(search_bp)

    return app

1.2 定义任务函数

from flask import current_app
from sqlalchemy import func

from models import db
from models.news import Article


def fix_statistic(app):
    """修正统计数据"""

    with app.app_context():  # 当通过init_app来关联应用时, 如果没有在视图中使用上下文变量/db, 则必须手动创建应用上下文
        # 需求: 修改用户的作品数量
        # 先从mysql中查询所有用户的作品数量  select user_id, count(id) from t_article group by user_id where status = 2
        ret = db.session.query(Article.user_id, func.count(Article.id)).filter(Article.status == Article.STATUS.APPROVED).group_by(Article.user_id).all()

        # 删除redis中的数据
        pipe = current_app.redis_master.pipeline(transaction=False)
        pipe.delete('count:user:arts')

        # 将mysql中的数据写入redis
        for user_id, count in ret:
            pipe.zadd('count:user:arts', count, user_id)

        pipe.execute()  # 批量发送给redis

为了能够代码复用,对以上代码进行改造

from flask import current_app
from sqlalchemy import func

from cache.statistic import UserArticleCountStorage, UserFollowingsCountStorage, UserFansCountStorage
from models import db
from models.news import Article


def __fix_statistic(cls):
    try:
        # 先到mysql中查询数据
        ret = cls.db_query()
        # 校正redis数据
        cls.reset(ret)
    except BaseException as e:
        current_app.logger.error(e)
        raise e


def fix_statistic(app):
    """修正统计数据"""

    with app.app_context():

        # 校正所有用户的作品数量
        __fix_statistic(UserArticleCountStorage)

        # 校正所有用户的关注数量
        __fix_statistic(UserFollowingsCountStorage)

        # 校正所有用户的分析数量
        __fix_statistic(UserFansCountStorage)

把数据操作抽离封装到对应类里,定义为类方法

from flask import current_app
from redis import StrictRedis, RedisError
from sqlalchemy import func

from models import db
from models.news import Article


class BaseCountStorage:
    """统计基类"""
    key = ''

    @classmethod
    def get(cls, user_id):
        """
        获取指定用户的统计数

        :param user_id: 指定的用户
        :return: 统计数量
        """
        redis = current_app.redis_slave  # type: StrictRedis
        try:
            # 取指定用户的分值
            count = redis.zscore(cls.key, user_id)
        except RedisError as e:
            current_app.logger.error(e)
            raise e

        if count:
            return int(count) if int(count) > 0 else 0
        else:
            return 0

    @classmethod
    def incr(cls, user_id):
        """给指定的用户的统计数量+1"""
        redis = current_app.redis_master  # type: StrictRedis
        try:
            redis.zincrby(cls.key, user_id)
        except RedisError as e:
            current_app.logger.error(e)
            raise e
#   核心逻辑封装到这里(提示1)
    @classmethod
    def reset(cls, db_query_ret):
        """重置数据"""

        # 删除redis中的数据
        pipe = current_app.redis_master.pipeline(transaction=False)
        pipe.delete(cls.key)

        # 将mysql中的数据写入redis
        for user_id, count in db_query_ret:
            pipe.zadd(cls.key, count, user_id)

        pipe.execute()  # 批量发送给redis


class UserArticleCountStorage(BaseCountStorage):
    """用户作品数量统计类

    count:user:arts  zset  [{value: 用户id, score: 作品数}, {}]
    """
    key = 'count:user:arts'  # 设置键
    
# mysql的查询封装到这里(提示2)
    @classmethod
    def db_query(cls):
        return db.session.query(Article.user_id, func.count(Article.id)).filter(
            Article.status == Article.STATUS.APPROVED).group_by(Article.user_id).all()


class UserFollowingsCountStorage(BaseCountStorage):
    """用户关注数量统计类

    count:user:followings  zset  [{value: 用户id, score: 关注数}, {}]
    """
    key = 'count:user:followings'  # 设置键
# mysql的查询封装到这里(提示3)
    @classmethod
    def db_query(cls):
        return db.session.query(Article.user_id, func.count(Article.id)).filter(
            Article.status == Article.STATUS.APPROVED).group_by(Article.user_id).all()


class UserFansCountStorage(BaseCountStorage):
    """用户粉丝数量统计类
    count:user:fans  zset  [{value: 用户id, score: 粉丝数}, {}]
    """
    key = 'count:user:fans'  # 设置键
# mysql的查询封装到这里(提示4)
    @classmethod
    def db_query(cls):
        return db.session.query(Article.user_id, func.count(Article.id)).filter(
            Article.status == Article.STATUS.APPROVED).group_by(Article.user_id).all()