Redisson 的「布隆过滤器」需要将当前的元素经过事先设计构建好的 K 个哈希函数计算出 K 个哈希值，并将预先已经构建好的「位数组」的相关下标取值置为 1 。当某个元素需要判断是否已存在时，则同样是先经过 K 个哈希函数求取 K 个哈希值，并判断「位数组」相应的 K 个下标的取值是否都为 1 。如果是，则代表元素是「大概率」是存在的；否则，表示该元素一定不存在。
由于项目中需要将查找的数据进行布隆过滤器进行过滤,使用的原因如下:
由于前端会向后台请求数据,数据库中不存在该数据,会先向对应的redis中查询,如果redis中没有该数据则会向数据库去查询,当数据库没有该数据,那么多次请求后会损耗数据库的性能,
解决方案:
在redis中存储查询的空数据返回给前端
后续问题:
如果前端随机id进行查询的话,redis可能存储过多的无用数据占用内存
这个时候就需要在redis查询之前做一个布隆过滤器进行数据判断
项目上使用思路:
① 启动时思路:定义一个需要创建布隆过滤器的注解,将注解标注到mapper头上,通过实现ApplicationListener接口扫描整个项目上被标注的该注解的类,由于本项目使用的mybatis plus,则将注解标注到mapper类上,通过反射调用selectList方法获取对应的数据集合,再通过反射获取baseMapper接口上的泛型参数,再通过泛型参数获取getId方法,将id集合存储到布隆过滤其中;
② 更新思路 :由于布隆过滤器数据不能删除,则在晚上再次扫描一次注解,重新生成新的布隆过滤器,保证数据的准确性;
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-data-redisartifactId>
<exclusions>
<exclusion>
<groupId>io.lettucegroupId>
<artifactId>lettuce-coreartifactId>
exclusion>
exclusions>
dependency>
<dependency>
<groupId>redis.clientsgroupId>
<artifactId>jedisartifactId>
dependency>
<dependency>
<groupId>org.redissongroupId>
<artifactId>redissonartifactId>
<version>3.15.6version>
dependency>
使用spring框架时继承ApplicationListenter 后会在项目启动后运行这个方法从起到扫描的作用
@Component
@Slf4j
public class InjectionScan implements ApplicationListener<ContextRefreshedEvent> {
@Resource
private AuthServiceClient authServiceClient;
@Resource
private RedissonClient redissonClient;
@Value("${spring.application.name}")
private String serviceName;
@Resource
private ApplicationContext applicationContext;
@Override
public void onApplicationEvent(ContextRefreshedEvent event) {
// generatePermission(event);
generateBloomFilterKey(event);
}
public void generateBloomFilterKey(ContextRefreshedEvent event){
Long bloomNum = 100_000L;
Map<String, Object> beansWithAnnotation = event.getApplicationContext().getBeansWithAnnotation(BloomFilterScan.class);
for (String s : beansWithAnnotation.keySet()) {
Object controller = beansWithAnnotation.get(s);
//获取代理类对象对象
Class<?> aClass = controller.getClass();
//获取接口对象
Class<?> anInterface= aClass.getInterfaces()[0];
//从ioc容器中获取mysqlDao对象
Object mysqlDao = applicationContext.getBean(anInterface);
//获取baseMapper的泛型类型
Class entityClass = null;
Method getId = null;
try {
ParameterizedType parameterizedType = (ParameterizedType) anInterface.getGenericInterfaces()[0];
entityClass = (Class) parameterizedType.getActualTypeArguments()[0];
//找到对应getId方法
getId = Arrays.stream(entityClass.getDeclaredMethods()).filter(method -> Objects.equals("getId", method.getName())).collect(Collectors.toList()).get(0);
} catch (Exception e) {
e.printStackTrace();
//如果找不到方法,就跳出该循环
continue;
}
//存储获取的id集合
List<Object> idList = new ArrayList<>();
for (Method declaredMethod : aClass.getDeclaredMethods()) {
if ("selectList".equals(declaredMethod.getName())){
try {
//获取数量的集合
List<Object> objectList = (List<Object>) declaredMethod.invoke(mysqlDao, new QueryWrapper<>());
//判断获取的集合数量是否大于设定的布隆过滤器数据量大小
if (objectList.size()>bloomNum){
//大于的话就扩大10倍
bloomNum = bloomNum*10L;
}
for (Object po : objectList) {
//运行getId方法获取对应id值
Object invoke = getId.invoke(po);
//存入到对应集合中
idList.add(invoke);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
//设置布隆过滤器的key,目前为po类的名字
RBloomFilter<Object> bloomFilter = redissonClient.getBloomFilter(entityClass.getSimpleName());
//给当前布隆过滤器的设置一个0秒的过期时间
bloomFilter.expire(0, TimeUnit.SECONDS);
//清除过去数据
bloomFilter.clearExpire();
//创建一个5%误差率,数量为自定义变量的空间
bloomFilter.tryInit(bloomNum,0.005);
//循环遍历id并存入到布隆过滤器中
for (Object id : idList) {
bloomFilter.add(id);
}
log.info("表 "+entityClass.getSimpleName()+" 数据的id已存入布隆过滤器中");
}
}
//扫描controller类生产权限表
private void generatePermission(ContextRefreshedEvent event) {
Map<String, Object> beansWithAnnotation = event.getApplicationContext().getBeansWithAnnotation(RestController.class);
for (String s : beansWithAnnotation.keySet()) {
Object o = beansWithAnnotation.get(s);
String path1 = "";
String path2 = "";
String methodType = "";
String roleName = "";
String rightsName = "";
String describe = "";
Class<?> aClass = o.getClass();
path1 = aClass.getAnnotation(RequestMapping.class).value()[0];
for (Method method : aClass.getDeclaredMethods()) {
if (method.getAnnotation(PermissionInjection.class) != null) {
PermissionInjection permissionInjection = method.getAnnotation(PermissionInjection.class);
GetMapping getMapping = method.getAnnotation(GetMapping.class);
PostMapping postMapping = method.getAnnotation(PostMapping.class);
RequestMapping requestMapping = method.getAnnotation(RequestMapping.class);
if (getMapping != null) {
path2 = getMapping.value()[0];
methodType = "GET";
} else if (postMapping != null) {
path2 = postMapping.value()[0];
methodType = "POST";
} else if (requestMapping != null) {
path2 = requestMapping.value()[0];
}
rightsName = permissionInjection.rightsName();
roleName = permissionInjection.roleName();
describe = permissionInjection.describer();
//如果第一个路径不为空则加斜杠
// if (!"".equals(path1)) path1 = path1 +"/";
//拼接权限信息
String authority = roleName;
if (!"".equals(rightsName)) authority = roleName + "," + rightsName;
//拼接路径信息
String uri = "/" + serviceName + path1 + path2;
String machiningUri = machiningUri(uri);
System.out.println("当前路径为 " + machiningUri + "\t角色名为 " + roleName + "\t权限名为 " + rightsName + "\t描述" + describe);
authServiceClient.deleteUriByUri(machiningUri);
authServiceClient.addUri(new ServiceUriAuthorityDto(methodType, machiningUri, authority, describe));
}
}
}
}
private String machiningUri(String uri) {
String[] strings = StringUtils.delimitedListToStringArray(uri, "/");
return Arrays.stream(strings).map(new Function<String, String>() {
@Override
public String apply(String s) {
if (s.contains("{")) {
return "*";
}
return s;
}
}).collect(Collectors.joining("/"));
}
}
注意:由于布隆过滤器只能增加不能删除,所以需要每天或者几天进行一次数据更新
通过spring定时器每天2点进行数据库刷新
@Slf4j
@Component
@EnableScheduling
public class BloomFilterRefresh {
@Resource
private RedissonClient redissonClient;
@Resource
private ApplicationContext applicationContext;
//设置每天凌晨2点进行更新
@Scheduled(cron = "0 0 2 * * ?")
public void bloomFilterValueUpdate(){
//设置扫描包的路径
List<String> list = scanClasses(this, "com.example");
//设置布隆过滤器的原始大小
Long bloomNum = 100_000L;
for (String s : list) {
try {
//获取扫描下的类的类对象
Class<?> aClass = Class.forName(s);
//剔除掉没有注解的类的对象
if (aClass.getAnnotation(BloomFilterScan.class)==null){
continue;
}
//更新布隆过滤器参数
updateValue(bloomNum,aClass);
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
}
}
private void updateValue(Long bloomNum, Class aClass) {
//获取接口对象BaseMapper
Class<?> anInterface= aClass.getInterfaces()[0];
//从ioc容器中获取mysqlDao对象
Object mysqlDao = applicationContext.getBean(aClass);
Class entityClass = null;
Method getId = null;
try {
//获取baseMapper的泛型类型
ParameterizedType parameterizedType = (ParameterizedType) aClass.getGenericInterfaces()[0];
entityClass = (Class) parameterizedType.getActualTypeArguments()[0];
//找到对应getId方法
getId = Arrays.stream(entityClass.getDeclaredMethods()).filter(method -> Objects.equals("getId", method.getName())).collect(Collectors.toList()).get(0);
} catch (Exception e) {
e.printStackTrace();
//如果找不到方法,就跳出该循环
return;
}
//存储获取的id集合
List<Object> idList = new ArrayList<>();
for (Method declaredMethod : anInterface.getDeclaredMethods()) {
if ("selectList".equals(declaredMethod.getName())){
try {
//获取数量的集合
List<Object> objectList = (List<Object>) declaredMethod.invoke(mysqlDao, new QueryWrapper<>());
//判断获取的集合数量是否大于设定的布隆过滤器数据量大小
if (objectList.size()> bloomNum){
//大于的话就扩大3倍
bloomNum = bloomNum *3L;
}
for (Object po : objectList) {
//运行getId方法获取对应id值
Object invoke = getId.invoke(po);
//存入到对应集合中
idList.add(invoke);
}
} catch (Exception e) {
log.error(e.getMessage());
}
}
}
//设置布隆过滤器的key,目前为po类的名字
RBloomFilter<Object> bloomFilter = redissonClient.getBloomFilter(entityClass.getSimpleName());
//给当前布隆过滤器的设置一个0秒的过期时间
bloomFilter.expire(0, TimeUnit.SECONDS);
//清除过去数据
bloomFilter.clearExpire();
//创建一个4%误差率,数量为自定义变量的空间
bloomFilter.tryInit(bloomNum,0.005);
//循环遍历id并存入到布隆过滤器中
for (Object id : idList) {
bloomFilter.add(id);
}
log.info("表 "+entityClass.getSimpleName()+" 数据的id已存入布隆过滤器中");
}
/**
* 根据传入的根包名，扫描该包下所有类
*
* @param thiz this
* @param rootPackageName 包名
*/
public static List<String> scanClasses(Object thiz, String rootPackageName) {
return scanClasses(thiz.getClass(), rootPackageName);
}
/**
* 根据传入的根包名，扫描该包下所有类
*
* @param thisClass 所在类
* @param rootPackageName 包名
*/
public static List<String> scanClasses(Class<?> thisClass, String rootPackageName) {
return scanClasses(Objects.requireNonNull(thisClass.getClassLoader()), rootPackageName);
}
/**
* 根据传入的根包名和对应classloader，扫描该包下所有类
*/
public static List<String> scanClasses(ClassLoader classLoader, String packageName) {
try {
String packageResource = packageName.replace(".", "/");
URL url = classLoader.getResource(packageResource);
File root = new File(url.toURI());
List<String> classList = new ArrayList<>();
scanClassesInner(root, packageName, classList);
return classList;
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* 遍历文件夹下所有.class文件，并转换成包名字符串的形式保存在结果List中。
*/
private static void scanClassesInner(File root, String packageName, List<String> result) {
for (File child : Objects.requireNonNull(root.listFiles())) {
String name = child.getName();
if (child.isDirectory()) {
scanClassesInner(child, packageName + "." + name, result);
} else if (name.endsWith(".class")) {
String className = packageName + "." + name.replace(".class", "");
result.add(className);
}
}
}
}