本地布隆过滤器同步至redis

本地服务器定时从redis拉取最新的bitmap值并更新本地的bitmap值，如果本地服务器有新增值，则将新增的值同步到redis服务器上

 

实现代码如下：

public class BloomFilterHelper {
    private int numHashFunctions;
    private Funnel funnel;
    private int bitSize;
    private BitArray bitArray;
    private String key;

    private volatile static Map LOCALBLOOM= new ConcurrentHashMap<>();

    private static Object Lock=new Object();

    public static ThreadLocal CurrentBloomKey=new ThreadLocal<>();

    /**
 * 生成布隆过滤器
 * @param key
 * @param funnel
 * @param expectedInsertions 布隆过滤器容量
 * @param fpp 布隆过滤器误差率
 * @param 
 * @return
 */
 public static  BloomFilterHelper create(String key,Funnel funnel, int expectedInsertions, double fpp){
        if(LOCALBLOOM.containsKey(key)){
            return LOCALBLOOM.get(key);
        }else{
            synchronized (Lock){
                if(LOCALBLOOM.containsKey(key)){
                    return LOCALBLOOM.get(key);
                }else{
                    RedisTemplate redisTemplate = getRedisTemplate();
                    if(redisTemplate==null){
                        return null;
                    }
                    BloomFilterHelper bloomFilterHelper=new BloomFilterHelper(key,funnel,expectedInsertions,fpp);
                    LOCALBLOOM.put(key,bloomFilterHelper);
                    CurrentBloomKey.set(key);
                    BitArray bitArray =(BitArray) redisTemplate.opsForValue().get(key);
                    return bloomFilterHelper;
                }
            }
        }
    }

    /**
 * 刷新布隆过滤器
 */
 public static void refreshBloom(){
        RedisTemplate redisTemplate = getRedisTemplate();
        if(redisTemplate==null){
            return ;
        }
        for (Map.Entry bloomFilterHelperEntry : LOCALBLOOM.entrySet()) {
            String key=bloomFilterHelperEntry.getKey();
            redisTemplate.opsForValue().get(key);
        }
    }

    public static  BloomFilterHelper getBloomFilterHelper(String key){
        if(LOCALBLOOM.containsKey(key)){
            return LOCALBLOOM.get(key);
        }
        return null;
    }

    private BloomFilterHelper(String key, Funnel funnel, int expectedInsertions, double fpp) {
        this.key=key;
        Preconditions.checkArgument(funnel != null, "funnel不能为空");
        this.funnel = funnel;
        // 计算bit数组长度
 bitSize = optimalNumOfBits(expectedInsertions, fpp);
        // 计算hash方法执行次数
 numHashFunctions = optimalNumOfHashFunctions(expectedInsertions, bitSize);
        bitArray=new BitArray(bitSize);
    }

    /**
 * 添加记录到布隆过滤器中
 * @param object
 * @return
 */
 public boolean addValue(T object){
        long hash64 = Hashing.murmur3_128().hashObject(object, funnel).asLong();
        int hash1 = (int)hash64;
        int hash2 = (int)(hash64 >>> 32);
        boolean bitsChanged = false;
        RedisTemplate redisTemplate = getRedisTemplate();
        for(int i = 1; i <= numHashFunctions; ++i) {
            int combinedHash = hash1 + i * hash2;
            if (combinedHash < 0) {
                combinedHash = ~combinedHash;
            }
            long index=(long)combinedHash % bitSize;
            bitsChanged |= bitArray.set(index);
            if(redisTemplate!=null){
                redisTemplate.opsForValue().setBit(this.key,index,true);
            }
        }
        return bitsChanged;
    }

    /**
 * 判断值是否在布隆过滤器中存在
 * @param object
 * @return
 */
 public boolean containsValue(T object){
        long hash64 = Hashing.murmur3_128().hashObject(object, funnel).asLong();
        int hash1 = (int)hash64;
        int hash2 = (int)(hash64 >>> 32);

        for(int i = 1; i <= numHashFunctions; ++i) {
            int combinedHash = hash1 + i * hash2;
            if (combinedHash < 0) {
                combinedHash = ~combinedHash;
            }
            if (!bitArray.get((long)combinedHash % bitSize)) {
                return false;
            }
        }
        return true;
    }

    /**
 * 计算bit数组长度
 */
 private int optimalNumOfBits(long n, double p) {
        if (p == 0) {
            // 设定最小期望长度
 p = Double.MIN_VALUE;
        }
        int sizeOfBitArray = (int) (-n * Math.log(p) / (Math.log(2) * Math.log(2)));
        return sizeOfBitArray;
    }

    /**
 * 计算hash方法执行次数
 */
 private int optimalNumOfHashFunctions(long n, long m) {
        int countOfHash = Math.max(1, (int) Math.round((double) m / n * Math.log(2)));
        return countOfHash;
    }

    private static RedisTemplate getRedisTemplate(){
        RedisTemplate redisTemplate = (RedisTemplate) SpringUtils.getBean("bitmapRedisTemplate",RedisTemplate.class);
        return redisTemplate;
    }

    public BitArray getBitArray() {
        return bitArray;
    }

    public static final class BitArray {
        final long[] data;
        long bitCount;

        BitArray(long bits) {
            this(new long[Ints.checkedCast(LongMath.divide(bits, 64, RoundingMode.CEILING))]);
        }

        // Used by serialization
 BitArray(long[] data) {
            checkArgument(data.length > 0, "data length is zero!");
            this.data = data;
            long bitCount = 0;
            for (long value : data) {
                bitCount += Long.bitCount(value);
            }
            this.bitCount = bitCount;
        }

        /** Returns true if the bit changed value. */
 public boolean set(long index) {
            if (!get(index)) {
                data[(int) (index >>> 6)] |= (1L << index);
                bitCount++;
                return true;
            }
            return false;
        }

        public boolean get(long index) {
            return (data[(int) (index >>> 6)] & (1L << index)) != 0;
        }

        /** Number of bits */
 long bitSize() {
            return (long) data.length * Long.SIZE;
        }

        /** Number of set bits (1s) */
 long bitCount() {
            return bitCount;
        }

        BloomFilterHelper.BitArray copy() {
            return new BloomFilterHelper.BitArray(data.clone());
        }

        /** Combines the two BitArrays using bitwise OR. */
 void putAll(BloomFilterHelper.BitArray array) {
            checkArgument(data.length == array.data.length,
                    "BitArrays must be of equal length (%s != %s)", data.length, array.data.length);
            bitCount = 0;
            for (int i = 0; i < data.length; i++) {
                data[i] |= array.data[i];
                bitCount += Long.bitCount(data[i]);
            }
        }

        @Override public boolean equals(Object o) {
            if (o instanceof BloomFilterHelper.BitArray) {
                BloomFilterHelper.BitArray bitArray = (BloomFilterHelper.BitArray) o;
                return Arrays.equals(data, bitArray.data);
            }
            return false;
        }

        @Override public int hashCode() {
            return Arrays.hashCode(data);
        }
    }
}

@Configuration
public class RedisConfig  {

    @Bean("bitmapRedisTemplate")
    public RedisTemplate stringValueRedisTemplate(@Autowired RedisConnectionFactory redisConnectionFactory){
        RedisTemplate redisTemplate=new RedisTemplate();
        redisTemplate.setValueSerializer(new BitmapSerialize());
        redisTemplate.setConnectionFactory(redisConnectionFactory);
        return redisTemplate;
    }
}

public class BitmapSerialize implements RedisSerializer {
    @Override
 public byte[] serialize(Object o) throws SerializationException {
        return new byte[0];
    }

    @Override
 public BloomFilterHelper.BitArray deserialize(byte[] bytes) throws SerializationException {
        System.out.println("反序列化");
        if(bytes==null){
            return null;
        }
        String key = BloomFilterHelper.CurrentBloomKey.get();
        if(ObjectUtils.isEmpty(key)){
            return null;
        }
        BloomFilterHelper bloomFilterHelper = BloomFilterHelper.getBloomFilterHelper(key);
        long index=0;
        byte a= (byte) (0x1<<7);  //二进制10000000
 for (byte b : bytes) {
            for(int i=0;i<8;i++) {
                byte leftB = (byte) (b< 
  
 
  
@Configuration
@EnableScheduling
public class BloomRefreshTask {
    /**
 * 每隔5分钟从redis拉取最新布隆过滤器值
 */
 @Scheduled(cron = "0 0/5 * * * ? ")
    public void refresh(){
        BloomFilterHelper.refreshBloom();
    }
}
 
  
使用例子：
 
  
BloomFilterHelper bloomFilterHelper=BloomFilterHelper.create("aaa", Funnels.stringFunnel(Charsets.UTF_8),100,0.01);
bloomFilterHelper.addValue("a");
boolean exit1 = bloomFilterHelper.containsValue("a");