• ES使用游标查询数据


    1. // 统计查询
    2. CountRequest countRequest = new CountRequest();
    3. countRequest.types("类型,按照版本进行使用");
    4. countRequest.indices("索引");
    5. // 构建查询参数
    6. SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    7. // 设置统计查询参数
    8. countRequest.source(searchSourceBuilder);
    9. // 统计数量
    10. CountResponse countResponse = restHighLevelClient.count(countRequest,RequestOptions.DEFAULT);
    11. // 获取数量
    12. long count = countResponse.getCount();
    13. if(count > 500000){
    14. throw new RumtimeException("下载数量不允许超过50W");
    15. }
    16. // 数量大于1W条时使用scroll进行下载数据
    17. if (count > 10000) {
    18. searchSourceBuilder.size(10000);
    19. SearchRequest searchRequest = new SearchRequest("索引").types("类型,按版本判断是否使用");
    20. searchRequest.source(searchSourceBuilder);
    21. // 创建游标并设置过期时间
    22. Scroll scroll = new Scroll(TimeValue.timeValueMinutes(5L));
    23. // 设定游标
    24. searchRequest.scroll(scroll);
    25. // 执行查询,获取响应数据和游标ID
    26. SearchResponse searchResponse = restHighLevelClient.search(request, RequestOptions.DEFAULT);
    27. String scrollId = searchResponse.getScrollId();
    28. SearchHit[] searchHits = searchResponse.getHits().getHits();
    29. // 收集数据集合
    30. List searchHitList = new ArrayList<>(Arrays.asList(searchHits));
    31. while (true) {
    32. if (searchHits.length == 0) {
    33. break;
    34. }
    35. // 创建游标查询请求
    36. SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
    37. // 设定游标时长
    38. scrollRequest.scroll(scroll);
    39. // 使用游标进行查询
    40. searchResponse = restHighLevelClient.scroll(searchScrollRequest, RequestOptions.DEFAULT);
    41. // 获取游标ID
    42. scrollId = searchResponse.getScrollId();
    43. // 获取数据
    44. searchHits = searchResponse.getHits().getHits();
    45. searchHitList.addAll(Arrays.asList(searchHits));
    46. }
    47. // 清除游标ID
    48. ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
    49. clearScrollRequest.addScrollId(scrollId);
    50. restHighLevelClient.clearScroll(clearScrollRequest,RequestOptions.DEFAULT);
    51. // TODO process 处理逻辑
    52. return;
    53. }

  • 相关阅读:
    Unity HDRP 录制透明通道序列帧
    Android实现在activity启动时传递对象和字符串参数
    Modbus CRC
    【gl-transitions配置】原项目dockerfile修改,为视频添加转场效果
    【BUG】第一次创建vue3+vite项目启动报错Error: Cannot find module ‘worker_threads‘
    spring之AOP
    Java 如何经行异常处理?常见的运行异常的编译时异常
    计算机毕业论文java毕业设计选题基于springboot的社区服务管理包运行成功]
    学习鸿蒙基础(11)
    Redis
  • 原文地址:https://blog.csdn.net/now19930616/article/details/134526700