From 3d634f432dee3a554b7255bf8483abb5b4075369 Mon Sep 17 00:00:00 2001 From: xiaoCJ <406612557@qq.com> Date: Thu, 4 Jul 2024 09:28:38 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9ID=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/application-druid.yml | 25 +- .../ruoyi/biemo/business/domain/DocInfo.java | 2 +- .../business/service/DocInfoService.java | 5 +- .../biemo/elasticsearch/util/EsService.java | 319 +++++++++--------- 4 files changed, 171 insertions(+), 180 deletions(-) diff --git a/ruoyi-admin/src/main/resources/application-druid.yml b/ruoyi-admin/src/main/resources/application-druid.yml index 55f1f5a..a5956d0 100644 --- a/ruoyi-admin/src/main/resources/application-druid.yml +++ b/ruoyi-admin/src/main/resources/application-druid.yml @@ -3,19 +3,10 @@ # data: # mongodb: # database: makesoft -## host: 39.108.144.227 -## port: 1471 # host: 120.78.220.29 # port: 27017 # username: makesoft # password: makesoft -## host: 118.31.7.2 -## port: 27017 -## username: root -## password: root -# #elasticsearch: -# #rest: -# #uris: http://39.108.144.227:9200 # # datasource: # type: com.alibaba.druid.pool.DruidDataSource @@ -26,12 +17,6 @@ # url: jdbc:mysql://120.78.220.29:3306/makesoft?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8 # username: root # password: Biemo123 -## url: jdbc:mysql://39.108.144.227:1637/makesoft?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8 -## username: root -## password: Biemo123 -## url: jdbc:mysql://118.31.7.2:3306/makesoft?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8 -## username: root -## password: sztzjy2017 # # 从库数据源 # slave: # # 从数据源开关/默认关闭 @@ -82,15 +67,11 @@ #elasticsearch: # schema: http # address: 120.78.220.29:9200 -## address: 39.108.144.227:3135 -## address: 118.31.7.2:9200 # connectTimeout: 10000 # socketTimeout: 10000 # connectionRequestTimeout: 10000 # maxConnectNum: 100 # maxConnectPerRoute: 100 -## username: elastic -## password: es@test001 # #sso: # apiUrl: http://test.vsp.cqooc.com/open/api/v2/oauth/token?appId={APPID}&signature={SIGNATURE}&ticket={TICKET} @@ -105,9 +86,9 @@ # getReportListUrl: http://test.vsp.cqooc.com/open/api/v2/query/data?appId=APPID&accessToken=ACCESS_TOKEN&signature=SIGNATURE -##################重庆二师配置 -# 项目相关配置 -# 数据源配置 +#################重庆二师配置 + 项目相关配置 + 数据源配置 spring: data: mongodb: diff --git a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java index a629949..7638bb5 100644 --- a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java +++ b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java @@ -15,7 +15,7 @@ public class DocInfo extends BiemoEntity { private static final long serialVersionUID = 1L; @EsId - private String id; + private Object id; @FieldInfo(type = "string",participle = 3) private String title; //文档类别 1:新闻、2:公众号 diff --git a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/service/DocInfoService.java b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/service/DocInfoService.java index c561f95..bfae79d 100644 --- a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/service/DocInfoService.java +++ b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/service/DocInfoService.java @@ -58,11 +58,10 @@ public class DocInfoService extends EsService { @Override protected DocInfo loadData(SearchSourceBuilder context, SearchHit hit) { String id = hit.getId(); - System.out.println("id:"+id); - System.out.println("TTTT:"+id+"TTTT"); if (Objects.isNull(id) || StringUtil.isEmpty(id)) return null; + System.out.println("hit.getId()===========>:"+id); DocInfo byId = mongoHelper.findById(id, DocInfo.class); - System.out.println("mongo:"+byId.getId()); + System.out.println("mongo.FindById结果======》:"+byId.getId()); return mongoHelper.findById(id, DocInfo.class); } diff --git a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/elasticsearch/util/EsService.java b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/elasticsearch/util/EsService.java index 171ebf3..0903e62 100644 --- a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/elasticsearch/util/EsService.java +++ b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/elasticsearch/util/EsService.java @@ -10,14 +10,9 @@ import com.ruoyi.biemo.elasticsearch.entity.FieldMapping; import com.ruoyi.biemo.elasticsearch.function.GFunction; import com.ruoyi.biemo.mongodb.config.Constant; import com.ruoyi.common.exception.CustomException; -import com.ruoyi.common.exception.ServiceException; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.apache.http.util.EntityUtils; -import org.apache.poi.ss.formula.functions.T; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteRequest; @@ -26,11 +21,13 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.*; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery; import org.elasticsearch.common.settings.Settings; @@ -50,14 +47,14 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.elasticsearch.annotations.Document; -import javax.print.Doc; import java.io.IOException; import java.lang.reflect.*; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -67,7 +64,7 @@ import java.util.stream.Collectors; */ public abstract class EsService { - protected Logger logger = LoggerFactory.getLogger(getClass()); + protected Logger logger = LoggerFactory.getLogger(getClass()); // ES 默认最大返回 10000 与 scroll查询有关 int MAX_RESULT_WINDOW = 10000; @@ -76,7 +73,7 @@ public abstract class EsService { // 加载数据超过200 时用批量加载 减少数据库连接 int BATCH_LOAD_SIZE = 200; // ES 滚动查询 有效时间 与 scroll查询有关 - TimeValue SCROLL_TIME = TimeValue.timeValueMinutes(1); + TimeValue SCROLL_TIME = TimeValue.timeValueMinutes(1); private String index; @@ -85,21 +82,22 @@ public abstract class EsService { private Method getId; - protected abstract T loadData(SearchSourceBuilder context,SearchHit hit); + protected abstract T loadData(SearchSourceBuilder context, SearchHit hit); /** * 批量加载数据 + * * @param context * @param hitArr * @return */ - public abstract List batchLoadData(SearchSourceBuilder context,SearchHit[] hitArr); + public abstract List batchLoadData(SearchSourceBuilder context, SearchHit[] hitArr); /** * 索引设置 */ - protected Settings.Builder createSettings(int shards, int replicas){ + protected Settings.Builder createSettings(int shards, int replicas) { // 设置分片数和副本数 return Settings.builder().put("index.number_of_shards", shards).put("index.number_of_replicas", replicas); } @@ -128,35 +126,37 @@ public abstract class EsService { } } } + /** * 创建mapping + * * @param * @param - * @param clazz 索引类型 + * @param clazz 索引类型 */ - public boolean createIndexAndCreateMapping( Class clazz, boolean rebuild, int number_of_shards, int number_of_replicas) { + public boolean createIndexAndCreateMapping(Class clazz, boolean rebuild, int number_of_shards, int number_of_replicas) { RestHighLevelClient client = null; - try{ + try { client = ElasticSearchPoolUtil.getClient(); if (indexExist()) { - if(rebuild){ + if (rebuild) { AcknowledgedResponse deleteIndexResponse = client.indices().delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT); - if(deleteIndexResponse.isAcknowledged()){ - return createIndexAndCreateMapping(index,ElasticSearchUtils.getFieldInfo(clazz), number_of_shards, number_of_replicas); - }else{ + if (deleteIndexResponse.isAcknowledged()) { + return createIndexAndCreateMapping(index, ElasticSearchUtils.getFieldInfo(clazz), number_of_shards, number_of_replicas); + } else { logger.error("删除旧索引数据失败，创建索引mapping失败"); return false; } - }else{ + } else { logger.info("不需要删除旧的索引，没有进一步创建索引结构哦~"); return true; } - }else{ - return createIndexAndCreateMapping(index,ElasticSearchUtils.getFieldInfo(clazz), number_of_shards, number_of_replicas); + } else { + return createIndexAndCreateMapping(index, ElasticSearchUtils.getFieldInfo(clazz), number_of_shards, number_of_replicas); } - }catch (Exception e){ + } catch (Exception e) { throw new RuntimeException(); - }finally { + } finally { ElasticSearchPoolUtil.returnClient(client); } } @@ -164,11 +164,12 @@ public abstract class EsService { /** * 根据信息自动创建索引与mapping * 构建mapping描述 - * @param index 索引名称 + * + * @param index 索引名称 * @param - * @param fieldMappingList 字段信息 + * @param fieldMappingList 字段信息 * @param - * @param number_of_shards 分片数 + * @param number_of_shards 分片数 * @param number_of_replicas 副本数 * @return */ @@ -178,78 +179,78 @@ public abstract class EsService { try { mapping = XContentFactory.jsonBuilder() .startObject() - .startObject("_source").field("enabled","false").endObject() + .startObject("_source").field("enabled", "false").endObject() .startObject("properties"); //设置自定义字段 - for(FieldMapping info : fieldMappingList){ + for (FieldMapping info : fieldMappingList) { String field = info.getField(); String dateType = info.getType(); - if(dateType == null || "".equals(dateType.trim())){ + if (dateType == null || "".equals(dateType.trim())) { dateType = "string"; } dateType = dateType.toLowerCase(); int participle = info.getParticiple(); - if("string".equals(dateType)){ - if(participle == 0){ + if ("string".equals(dateType)) { + if (participle == 0) { mapping.startObject(field) - .field("type","keyword") + .field("type", "keyword") .field("index", false) .field("ignore_above", info.getIgnoreAbove()) .endObject(); - } else if(participle == 1) { + } else if (participle == 1) { mapping.startObject(field) - .field("type","text") - .field("analyzer","hanlp") + .field("type", "text") + .field("analyzer", "hanlp") .endObject(); - }else if(participle == 2){ + } else if (participle == 2) { mapping.startObject(field) - .field("type","text") - .field("analyzer","hanlp_standard") + .field("type", "text") + .field("analyzer", "hanlp_standard") .endObject(); - }else if(participle == 3){ + } else if (participle == 3) { mapping.startObject(field) - .field("type","text") - .field("analyzer","hanlp_index") + .field("type", "text") + .field("analyzer", "hanlp_index") .endObject(); - }else if(participle == 4){ + } else if (participle == 4) { mapping.startObject(field) - .field("type","text") - .field("analyzer","hanlp_nlp") + .field("type", "text") + .field("analyzer", "hanlp_nlp") .endObject(); - }else if(participle == 5){ + } else if (participle == 5) { mapping.startObject(field) - .field("type","text") - .field("analyzer","hanlp_n_short") + .field("type", "text") + .field("analyzer", "hanlp_n_short") .endObject(); - }else if(participle == 6){ + } else if (participle == 6) { mapping.startObject(field) - .field("type","text") - .field("analyzer","hanlp_crf") + .field("type", "text") + .field("analyzer", "hanlp_crf") .endObject(); - }else if(participle == 7){ + } else if (participle == 7) { mapping.startObject(field) - .field("type","text") - .field("analyzer","hanlp_speed") + .field("type", "text") + .field("analyzer", "hanlp_speed") .endObject(); } - }else if("datetime".equals(dateType)){ + } else if ("datetime".equals(dateType)) { mapping.startObject(field) - .field("type","date") - .field("format","yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis") + .field("type", "date") + .field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis") .endObject(); - }else if ("timestamp".equals(dateType)){ + } else if ("timestamp".equals(dateType)) { mapping.startObject(field) - .field("type","date") - .field("format","strict_date_optional_time||epoch_millis") + .field("type", "date") + .field("format", "strict_date_optional_time||epoch_millis") .endObject(); - }else if("float".equals(dateType)||"double".equals(dateType)){ + } else if ("float".equals(dateType) || "double".equals(dateType)) { mapping.startObject(field) - .field("type","scaled_float") - .field("scaling_factor",100) + .field("type", "scaled_float") + .field("scaling_factor", 100) .endObject(); - }else { + } else { mapping.startObject(field) - .field("type",dateType) - .field("index",true) + .field("type", dateType) + .field("index", true) .endObject(); } } @@ -257,24 +258,23 @@ public abstract class EsService { .endObject(); CreateIndexRequest createIndexRequest = new CreateIndexRequest(index); // 设置分片数和副本数 - createIndexRequest.settings(createSettings(number_of_shards,number_of_replicas)); + createIndexRequest.settings(createSettings(number_of_shards, number_of_replicas)); // 构建对应index的mapping createIndexRequest.mapping(mapping); client = ElasticSearchPoolUtil.getClient(); // 发送创建index的请求 - CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest,RequestOptions.DEFAULT); + CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT); return createIndexResponse.isAcknowledged(); } catch (Exception e) { - logger.error("根据信息自动创建索引与mapping创建失败，失败信息为:"+ e.getMessage()); + logger.error("根据信息自动创建索引与mapping创建失败，失败信息为:" + e.getMessage()); throw new RuntimeException(); - }finally { + } finally { ElasticSearchPoolUtil.returnClient(client); } } - - public boolean indexExist() throws Exception{ + public boolean indexExist() throws Exception { RestHighLevelClient client = null; GetIndexRequest request = new GetIndexRequest(index); request.local(false); @@ -302,20 +302,20 @@ public abstract class EsService { client.index(request, RequestOptions.DEFAULT); } catch (Exception e) { throw new RuntimeException(e); - }finally { + } finally { ElasticSearchPoolUtil.returnClient(client); } } public void insertBatch(List list) { - if(list==null||list.size()==0){ + if (list == null || list.size() == 0) { return; } RestHighLevelClient client = null; try { client = ElasticSearchPoolUtil.getClient(); - if(!indexExist()){ - createIndexAndCreateMapping(list.get(0).getClass(),true,1,0); + if (!indexExist()) { + createIndexAndCreateMapping(list.get(0).getClass(), true, 1, 0); } BulkRequest request = new BulkRequest(); list.forEach(item -> { @@ -331,7 +331,7 @@ public abstract class EsService { client.bulk(request, RequestOptions.DEFAULT); } catch (Exception e) { throw new RuntimeException(e); - }finally { + } finally { ElasticSearchPoolUtil.returnClient(client); } } @@ -345,10 +345,11 @@ public abstract class EsService { client.bulk(request, RequestOptions.DEFAULT); } catch (Exception e) { throw new RuntimeException(e); - }finally { + } finally { ElasticSearchPoolUtil.returnClient(client); } } + public List search(SearchSourceBuilder builder) { SearchRequest request = new SearchRequest(index); request.source(builder); @@ -360,30 +361,30 @@ public abstract class EsService { List res = new ArrayList<>(hits.length); for (SearchHit hit : hits) { //res.add(JSON.parseObject(hit.getSourceAsString(), entity)); - res.add(loadData(builder,hit)); + res.add(loadData(builder, hit)); } return res; } catch (Exception e) { throw new RuntimeException(e); - }finally { + } finally { ElasticSearchPoolUtil.returnClient(client); } } //分页查询 根据需要返回自定义加载数据 比如从mongodb mysql 等 - public Page searchPage(SearchSourceBuilder builder,boolean searchFrom) { + public Page searchPage(SearchSourceBuilder builder, boolean searchFrom) { try { List content = new ArrayList<>(); - Long total = search(content,builder); - System.out.println("*****:"+content.get(0).getId()); + Long total = search(content, builder); + System.out.println("*****:" + content.get(0).getId()); List res = new ArrayList<>(); - if(searchFrom){ + if (searchFrom) { res = sequenceLoadData(content); - }else{ + } else { res = esLoadData(content); } Integer pageNo = builder.from(); - Page page=new Page<>(pageNo<=0?1:pageNo/builder.size()+1,builder.size()); + Page page = new Page<>(pageNo <= 0 ? 1 : pageNo / builder.size() + 1, builder.size()); page.setRows(res); page.setTotal(total); return page; @@ -392,13 +393,15 @@ public abstract class EsService { } } - protected List sequenceLoadData(List content){ + protected List sequenceLoadData(List content) { return content.stream().map(x -> loadData(null, x)).filter(x -> x != null).collect(Collectors.toList()); } - protected List esLoadData(List content){ + + protected List esLoadData(List content) { return content.stream().map(x -> JSON.parseObject(x.getSourceAsString(), entity)).filter(x -> x != null).collect(Collectors.toList()); } - protected long search(List content, SearchSourceBuilder searchSourceBuilder){ + + protected long search(List content, SearchSourceBuilder searchSourceBuilder) { // 用于判断 是否 返回所有数据 boolean flag = true; // 总数量 @@ -410,7 +413,7 @@ public abstract class EsService { // 已经处理过了 int from = pageNo; // 待查询页面内第一条记录的下标 - int firstRowNum = from + 1 ; + int firstRowNum = from + 1; // 最后一行 int lastRowNum = from + pageSize; // 创建查询请求 @@ -418,12 +421,12 @@ public abstract class EsService { // 构建SearchSourceBuilder searchRequest.source(searchSourceBuilder); // 不分页 最多返回10000条 - if(pageNo==0 && pageSize==0){ + if (pageNo == 0 && pageSize == 0) { searchSourceBuilder.from(0).size(MAX_RESULT_WINDOW); - }else{ + } else { // 分页 // 小于 MAX_RESULT_WINDOW 10000 - if(lastRowNum<=MAX_RESULT_WINDOW){ + if (lastRowNum <= MAX_RESULT_WINDOW) { searchSourceBuilder.from(from).size(pageSize); flag = false; } @@ -435,25 +438,27 @@ public abstract class EsService { // 查询数据超过第10000 if (lastRowNum > MAX_RESULT_WINDOW) { // 滚动搜索 - total = searchScroll(content,firstRowNum,lastRowNum,searchSourceBuilder,searchRequest); - }else{ + total = searchScroll(content, firstRowNum, lastRowNum, searchSourceBuilder, searchRequest); + } else { searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); SearchHits hits = searchResponse.getHits(); System.out.println(index); SearchHit[] hits1 = hits.getHits(); - System.out.println(hits1[0].getId()); - total = hits.getTotalHits().value; + if (hits1[0] != null) { + System.out.println(hits1[0].getId()); + } + total = hits.getTotalHits().value; // 不分页情况下 总条数超过10000 - if(total>MAX_RESULT_WINDOW&&flag){ - searchScroll(content,firstRowNum,(int)total,searchSourceBuilder,searchRequest); - }else{ + if (total > MAX_RESULT_WINDOW && flag) { + searchScroll(content, firstRowNum, (int) total, searchSourceBuilder, searchRequest); + } else { content = loadDataByResponse(content, searchResponse); } } } catch (Exception e) { - logger.error("查询索引错误，错误如下：{}",e); - throw new CustomException("查询索引出错",e); - }finally { + logger.error("查询索引错误，错误如下：{}", e); + throw new CustomException("查询索引出错", e); + } finally { ElasticSearchPoolUtil.returnClient(client); } return total; @@ -462,6 +467,7 @@ public abstract class EsService { /** * 通过响应加载hit数据 + * * @param searchHitList * @param searchResponse * @return @@ -470,14 +476,14 @@ public abstract class EsService { SearchHits hits = searchResponse.getHits(); // 条数过多 批量加载 减少数据库连接 SearchHit[] hitArr = hits.getHits(); - if(Objects.nonNull(hitArr)&&hitArr.length>=BATCH_LOAD_SIZE){ + if (Objects.nonNull(hitArr) && hitArr.length >= BATCH_LOAD_SIZE) { searchHitList.addAll(Arrays.asList(hitArr)); - }else{ + } else { //逐条加载 for (SearchHit hit : hitArr) { - String id=hit.getId(); - if (StringUtils.isBlank(id)){ + String id = hit.getId(); + if (StringUtils.isBlank(id)) { continue; } searchHitList.add(hit); @@ -489,6 +495,7 @@ public abstract class EsService { /** * scroll 查询 返回总条数 + * * @param searchHitList * @param firstRowNum * @param lastRowNum @@ -496,7 +503,7 @@ public abstract class EsService { * @param searchRequest * @return */ - private long searchScroll(List searchHitList,int firstRowNum,int lastRowNum,SearchSourceBuilder searchSourceBuilder,SearchRequest searchRequest){ + private long searchScroll(List searchHitList, int firstRowNum, int lastRowNum, SearchSourceBuilder searchSourceBuilder, SearchRequest searchRequest) { long total = 0L; SearchResponse searchResponse; // 初始位置 @@ -504,11 +511,11 @@ public abstract class EsService { // 结束位置 int endPosition; // 页面大小 - int pageSize = lastRowNum-firstRowNum+1; + int pageSize = lastRowNum - firstRowNum + 1; // 第一条数据所在滚动指针 - int firstScrollCursor = firstRowNum/MAX_RESULT_WINDOW; + int firstScrollCursor = firstRowNum / MAX_RESULT_WINDOW; // 最后一条数据所在指针 - int lastScrollCursor = lastRowNum/MAX_RESULT_WINDOW; + int lastScrollCursor = lastRowNum / MAX_RESULT_WINDOW; // 滚动次数 int scrollCount = 0; // 滚动id @@ -521,37 +528,37 @@ public abstract class EsService { // scroll 查询 获取前10000 条数据和scrollId searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); SearchHits hits = searchResponse.getHits(); - total = hits.getTotalHits().value; + total = hits.getTotalHits().value; // 1分钟 内的scrollId 第一次 scrollId = searchResponse.getScrollId(); // 部分数据在10000 以内 先获取一部分 - if(firstRowNum0?firstRowNum - scrollCount*MAX_RESULT_WINDOW:0; - endPosition = lastRowNum-scrollCount*MAX_RESULT_WINDOW>MAX_RESULT_WINDOW?MAX_RESULT_WINDOW:lastRowNum-scrollCount*MAX_RESULT_WINDOW; + if (firstRowNum < MAX_RESULT_WINDOW) { + startPosition = firstRowNum - scrollCount * MAX_RESULT_WINDOW > 0 ? firstRowNum - scrollCount * MAX_RESULT_WINDOW : 0; + endPosition = lastRowNum - scrollCount * MAX_RESULT_WINDOW > MAX_RESULT_WINDOW ? MAX_RESULT_WINDOW : lastRowNum - scrollCount * MAX_RESULT_WINDOW; loopLoadDataByPosition(searchHitList, startPosition, endPosition, hits); } - if(StringUtils.isNotEmpty(scrollId)){ + if (StringUtils.isNotEmpty(scrollId)) { // 指针还没有到最后的指针 - while (scrollCount=firstScrollCursor){ + if (scrollCount >= firstScrollCursor) { // 从hits取对应数据 剩余容量 - int capacity = pageSize-searchHitList.size(); - if(capacity>0){ + int capacity = pageSize - searchHitList.size(); + if (capacity > 0) { hits = searchResponse.getHits(); // 命中数量 int hitCount = hits.getHits().length; - startPosition = firstRowNum - scrollCount*MAX_RESULT_WINDOW>0?firstRowNum - scrollCount*MAX_RESULT_WINDOW:0; - endPosition = lastRowNum-scrollCount*MAX_RESULT_WINDOW>MAX_RESULT_WINDOW?MAX_RESULT_WINDOW:lastRowNum-scrollCount*MAX_RESULT_WINDOW; + startPosition = firstRowNum - scrollCount * MAX_RESULT_WINDOW > 0 ? firstRowNum - scrollCount * MAX_RESULT_WINDOW : 0; + endPosition = lastRowNum - scrollCount * MAX_RESULT_WINDOW > MAX_RESULT_WINDOW ? MAX_RESULT_WINDOW : lastRowNum - scrollCount * MAX_RESULT_WINDOW; // 超过命中次数 取命中次数 - endPosition = endPosition>hitCount?hitCount:endPosition; - if(scrollCount==firstScrollCursor) + endPosition = endPosition > hitCount ? hitCount : endPosition; + if (scrollCount == firstScrollCursor) startPosition--; - if(firstScrollCursor!=lastScrollCursor&&scrollCount==lastScrollCursor) - endPosition=capacity>MAX_RESULT_WINDOW?MAX_RESULT_WINDOW:capacity-startPosition; + if (firstScrollCursor != lastScrollCursor && scrollCount == lastScrollCursor) + endPosition = capacity > MAX_RESULT_WINDOW ? MAX_RESULT_WINDOW : capacity - startPosition; loopLoadDataByPosition(searchHitList, startPosition, endPosition, hits); } } @@ -560,7 +567,7 @@ public abstract class EsService { } } catch (Exception e) { e.printStackTrace(); - }finally { + } finally { ElasticSearchPoolUtil.returnClient(client); } return total; @@ -569,43 +576,46 @@ public abstract class EsService { /** * 滚动搜索 + * * @param scrollId 滚动Id * @return * @throws IOException */ - protected SearchResponse searchScroll(String scrollId) throws Exception{ + protected SearchResponse searchScroll(String scrollId) throws Exception { RestHighLevelClient client = null; client = ElasticSearchPoolUtil.getClient(); SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollId); searchScrollRequest.scroll(SCROLL_TIME); SearchResponse response = client.scroll(searchScrollRequest, RequestOptions.DEFAULT); ElasticSearchPoolUtil.returnClient(client); - return response; + return response; } /** * 循环批量加载数据 + * * @param context * @param startPosition * @param endPosition * @param hits */ - private List loopLoadDataByPosition( SearchSourceBuilder context, int startPosition, int endPosition, SearchHits hits) { + private List loopLoadDataByPosition(SearchSourceBuilder context, int startPosition, int endPosition, SearchHits hits) { int length = hits.getHits().length; - int destLength = endPosition-startPosition; - destLength = destLength>length?length:destLength; + int destLength = endPosition - startPosition; + destLength = destLength > length ? length : destLength; SearchHit[] hitArr = new SearchHit[destLength]; - System.arraycopy(hits.getHits(),startPosition,hitArr,0,hitArr.length); - return batchLoadData(context,hitArr); + System.arraycopy(hits.getHits(), startPosition, hitArr, 0, hitArr.length); + return batchLoadData(context, hitArr); } + private List loopLoadDataByPosition(List searchHitList, int startPosition, int endPosition, SearchHits hits) { int length = hits.getHits().length; - int destLength = endPosition-startPosition; - destLength = destLength>length?length:destLength; + int destLength = endPosition - startPosition; + destLength = destLength > length ? length : destLength; SearchHit[] hitArr = new SearchHit[destLength]; - System.arraycopy(hits.getHits(),startPosition,hitArr,0,hitArr.length); + System.arraycopy(hits.getHits(), startPosition, hitArr, 0, hitArr.length); searchHitList.addAll(Arrays.asList(hitArr)); return searchHitList; } @@ -617,7 +627,7 @@ public abstract class EsService { client.indices().delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT); } catch (Exception e) { throw new RuntimeException(e); - }finally { + } finally { ElasticSearchPoolUtil.returnClient(client); } } @@ -633,7 +643,7 @@ public abstract class EsService { client.deleteByQuery(request, RequestOptions.DEFAULT); } catch (Exception e) { throw new RuntimeException(e); - }finally { + } finally { ElasticSearchPoolUtil.returnClient(client); } } @@ -668,7 +678,7 @@ public abstract class EsService { public EsQueryBuilder getDoc_InfoBy_Id(GFunction gFunction, Object value) { String field = gFunction.field(); - field="_"+field; + field = "_" + field; this.searchSourceBuilder.postFilter(QueryBuilders.termQuery(field, value)); this.boolQueryBuilder.filter(QueryBuilders.termQuery(field, value)); return this; @@ -677,17 +687,17 @@ public abstract class EsService { public EsQueryBuilder eqAll(T t) { Field[] fields = t.getClass().getDeclaredFields(); - if(fields!=null&&fields.length>0){ + if (fields != null && fields.length > 0) { BoolQueryBuilder funScoreBuilders = QueryBuilders.boolQuery(); - for(Field field : fields){ + for (Field field : fields) { field.setAccessible(true); String name = field.getName(); try { Object value = field.get(t); - if(value==null||name==null||Constant.SERIALVERSIONUID.equals(name)){ + if (value == null || name == null || Constant.SERIALVERSIONUID.equals(name)) { continue; } - funScoreBuilders = boolTermQueryBuild( funScoreBuilders,name,value); + funScoreBuilders = boolTermQueryBuild(funScoreBuilders, name, value); } catch (IllegalAccessException e) { e.printStackTrace(); } @@ -701,14 +711,15 @@ public abstract class EsService { private BoolQueryBuilder boolTermQueryBuild(BoolQueryBuilder boolQueryBuilder, String key, Object value) { return boolQueryBuilder.must( QueryBuilders.functionScoreQuery( - QueryBuilders.termQuery(key, value), ScoreFunctionBuilders.weightFactorFunction(1) - ) + QueryBuilders.termQuery(key, value), ScoreFunctionBuilders.weightFactorFunction(1) + ) .setMinScore(0f) .maxBoost(50f) .scoreMode(FunctionScoreQuery.ScoreMode.MAX) .boostMode(CombineFunction.REPLACE) ); } + public EsQueryBuilder notEq(GFunction gFunction, Object value) { String field = gFunction.field(); this.searchSourceBuilder.postFilter(QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(field, value))); @@ -865,8 +876,8 @@ public abstract class EsService { return search(this.searchSourceBuilder); } - public Page queryPage(boolean searchfrom){ - return searchPage(this.searchSourceBuilder,searchfrom); + public Page queryPage(boolean searchfrom) { + return searchPage(this.searchSourceBuilder, searchfrom); } public void delete() { @@ -891,7 +902,7 @@ public abstract class EsService { } //分词 返回分词结果 - public List analyzeTitle(DocInfo docInfo){ + public List analyzeTitle(DocInfo docInfo) { try { RestHighLevelClient client = ElasticSearchPoolUtil.getClient(); @@ -903,7 +914,7 @@ public abstract class EsService { Response response = client.getLowLevelClient().performRequest(request); JSONObject tokens = JSONObject.parseObject(EntityUtils.toString(response.getEntity())); JSONArray arrays = tokens.getJSONArray("tokens"); - List list=new ArrayList<>(); + List list = new ArrayList<>(); for (int i = 0; i < arrays.size(); i++) { JSONObject obj = JSON.parseObject(arrays.getString(i)); list.add(obj.getString("token"));