|
|
|
@ -10,14 +10,9 @@ import com.ruoyi.biemo.elasticsearch.entity.FieldMapping;
|
|
|
|
|
import com.ruoyi.biemo.elasticsearch.function.GFunction;
|
|
|
|
|
import com.ruoyi.biemo.mongodb.config.Constant;
|
|
|
|
|
import com.ruoyi.common.exception.CustomException;
|
|
|
|
|
import com.ruoyi.common.exception.ServiceException;
|
|
|
|
|
import lombok.Data;
|
|
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
|
|
import org.apache.http.util.EntityUtils;
|
|
|
|
|
import org.apache.poi.ss.formula.functions.T;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
|
|
|
|
|
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
|
|
|
|
|
import org.elasticsearch.action.bulk.BulkRequest;
|
|
|
|
|
import org.elasticsearch.action.delete.DeleteRequest;
|
|
|
|
@ -26,11 +21,13 @@ import org.elasticsearch.action.search.SearchRequest;
|
|
|
|
|
import org.elasticsearch.action.search.SearchResponse;
|
|
|
|
|
import org.elasticsearch.action.search.SearchScrollRequest;
|
|
|
|
|
import org.elasticsearch.action.support.master.AcknowledgedResponse;
|
|
|
|
|
import org.elasticsearch.client.*;
|
|
|
|
|
import org.elasticsearch.client.Request;
|
|
|
|
|
import org.elasticsearch.client.RequestOptions;
|
|
|
|
|
import org.elasticsearch.client.Response;
|
|
|
|
|
import org.elasticsearch.client.RestHighLevelClient;
|
|
|
|
|
import org.elasticsearch.client.indices.CreateIndexRequest;
|
|
|
|
|
import org.elasticsearch.client.indices.CreateIndexResponse;
|
|
|
|
|
import org.elasticsearch.client.indices.GetIndexRequest;
|
|
|
|
|
import org.elasticsearch.client.transport.TransportClient;
|
|
|
|
|
import org.elasticsearch.common.lucene.search.function.CombineFunction;
|
|
|
|
|
import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
|
|
|
|
|
import org.elasticsearch.common.settings.Settings;
|
|
|
|
@ -50,14 +47,14 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
|
|
|
|
|
import org.elasticsearch.search.sort.SortOrder;
|
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
|
import org.springframework.beans.factory.annotation.Qualifier;
|
|
|
|
|
import org.springframework.data.elasticsearch.annotations.Document;
|
|
|
|
|
|
|
|
|
|
import javax.print.Doc;
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.lang.reflect.*;
|
|
|
|
|
import java.util.*;
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.Arrays;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.Objects;
|
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
@ -85,21 +82,22 @@ public abstract class EsService<T> {
|
|
|
|
|
|
|
|
|
|
private Method getId;
|
|
|
|
|
|
|
|
|
|
protected abstract T loadData(SearchSourceBuilder context,SearchHit hit);
|
|
|
|
|
protected abstract T loadData(SearchSourceBuilder context, SearchHit hit);
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 批量加载数据
|
|
|
|
|
*
|
|
|
|
|
* @param context
|
|
|
|
|
* @param hitArr
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
public abstract List<T> batchLoadData(SearchSourceBuilder context,SearchHit[] hitArr);
|
|
|
|
|
public abstract List<T> batchLoadData(SearchSourceBuilder context, SearchHit[] hitArr);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 索引设置
|
|
|
|
|
*/
|
|
|
|
|
protected Settings.Builder createSettings(int shards, int replicas){
|
|
|
|
|
protected Settings.Builder createSettings(int shards, int replicas) {
|
|
|
|
|
// 设置分片数和副本数
|
|
|
|
|
return Settings.builder().put("index.number_of_shards", shards).put("index.number_of_replicas", replicas);
|
|
|
|
|
}
|
|
|
|
@ -128,35 +126,37 @@ public abstract class EsService<T> {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 创建mapping
|
|
|
|
|
*
|
|
|
|
|
* @param
|
|
|
|
|
* @param
|
|
|
|
|
* @param clazz 索引类型
|
|
|
|
|
*/
|
|
|
|
|
public boolean createIndexAndCreateMapping( Class clazz, boolean rebuild, int number_of_shards, int number_of_replicas) {
|
|
|
|
|
public boolean createIndexAndCreateMapping(Class clazz, boolean rebuild, int number_of_shards, int number_of_replicas) {
|
|
|
|
|
RestHighLevelClient client = null;
|
|
|
|
|
try{
|
|
|
|
|
try {
|
|
|
|
|
client = ElasticSearchPoolUtil.getClient();
|
|
|
|
|
if (indexExist()) {
|
|
|
|
|
if(rebuild){
|
|
|
|
|
if (rebuild) {
|
|
|
|
|
AcknowledgedResponse deleteIndexResponse = client.indices().delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT);
|
|
|
|
|
if(deleteIndexResponse.isAcknowledged()){
|
|
|
|
|
return createIndexAndCreateMapping(index,ElasticSearchUtils.getFieldInfo(clazz), number_of_shards, number_of_replicas);
|
|
|
|
|
}else{
|
|
|
|
|
if (deleteIndexResponse.isAcknowledged()) {
|
|
|
|
|
return createIndexAndCreateMapping(index, ElasticSearchUtils.getFieldInfo(clazz), number_of_shards, number_of_replicas);
|
|
|
|
|
} else {
|
|
|
|
|
logger.error("删除旧索引数据失败,创建索引mapping失败");
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}else{
|
|
|
|
|
} else {
|
|
|
|
|
logger.info("不需要删除旧的索引,没有进一步创建索引结构哦~");
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}else{
|
|
|
|
|
return createIndexAndCreateMapping(index,ElasticSearchUtils.getFieldInfo(clazz), number_of_shards, number_of_replicas);
|
|
|
|
|
} else {
|
|
|
|
|
return createIndexAndCreateMapping(index, ElasticSearchUtils.getFieldInfo(clazz), number_of_shards, number_of_replicas);
|
|
|
|
|
}
|
|
|
|
|
}catch (Exception e){
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
throw new RuntimeException();
|
|
|
|
|
}finally {
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -164,6 +164,7 @@ public abstract class EsService<T> {
|
|
|
|
|
/**
|
|
|
|
|
* 根据信息自动创建索引与mapping
|
|
|
|
|
* 构建mapping描述
|
|
|
|
|
*
|
|
|
|
|
* @param index 索引名称
|
|
|
|
|
* @param
|
|
|
|
|
* @param fieldMappingList 字段信息
|
|
|
|
@ -178,78 +179,78 @@ public abstract class EsService<T> {
|
|
|
|
|
try {
|
|
|
|
|
mapping = XContentFactory.jsonBuilder()
|
|
|
|
|
.startObject()
|
|
|
|
|
.startObject("_source").field("enabled","false").endObject()
|
|
|
|
|
.startObject("_source").field("enabled", "false").endObject()
|
|
|
|
|
.startObject("properties"); //设置自定义字段
|
|
|
|
|
for(FieldMapping info : fieldMappingList){
|
|
|
|
|
for (FieldMapping info : fieldMappingList) {
|
|
|
|
|
String field = info.getField();
|
|
|
|
|
String dateType = info.getType();
|
|
|
|
|
if(dateType == null || "".equals(dateType.trim())){
|
|
|
|
|
if (dateType == null || "".equals(dateType.trim())) {
|
|
|
|
|
dateType = "string";
|
|
|
|
|
}
|
|
|
|
|
dateType = dateType.toLowerCase();
|
|
|
|
|
int participle = info.getParticiple();
|
|
|
|
|
if("string".equals(dateType)){
|
|
|
|
|
if(participle == 0){
|
|
|
|
|
if ("string".equals(dateType)) {
|
|
|
|
|
if (participle == 0) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","keyword")
|
|
|
|
|
.field("type", "keyword")
|
|
|
|
|
.field("index", false)
|
|
|
|
|
.field("ignore_above", info.getIgnoreAbove())
|
|
|
|
|
.endObject();
|
|
|
|
|
} else if(participle == 1) {
|
|
|
|
|
} else if (participle == 1) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","text")
|
|
|
|
|
.field("analyzer","hanlp")
|
|
|
|
|
.field("type", "text")
|
|
|
|
|
.field("analyzer", "hanlp")
|
|
|
|
|
.endObject();
|
|
|
|
|
}else if(participle == 2){
|
|
|
|
|
} else if (participle == 2) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","text")
|
|
|
|
|
.field("analyzer","hanlp_standard")
|
|
|
|
|
.field("type", "text")
|
|
|
|
|
.field("analyzer", "hanlp_standard")
|
|
|
|
|
.endObject();
|
|
|
|
|
}else if(participle == 3){
|
|
|
|
|
} else if (participle == 3) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","text")
|
|
|
|
|
.field("analyzer","hanlp_index")
|
|
|
|
|
.field("type", "text")
|
|
|
|
|
.field("analyzer", "hanlp_index")
|
|
|
|
|
.endObject();
|
|
|
|
|
}else if(participle == 4){
|
|
|
|
|
} else if (participle == 4) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","text")
|
|
|
|
|
.field("analyzer","hanlp_nlp")
|
|
|
|
|
.field("type", "text")
|
|
|
|
|
.field("analyzer", "hanlp_nlp")
|
|
|
|
|
.endObject();
|
|
|
|
|
}else if(participle == 5){
|
|
|
|
|
} else if (participle == 5) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","text")
|
|
|
|
|
.field("analyzer","hanlp_n_short")
|
|
|
|
|
.field("type", "text")
|
|
|
|
|
.field("analyzer", "hanlp_n_short")
|
|
|
|
|
.endObject();
|
|
|
|
|
}else if(participle == 6){
|
|
|
|
|
} else if (participle == 6) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","text")
|
|
|
|
|
.field("analyzer","hanlp_crf")
|
|
|
|
|
.field("type", "text")
|
|
|
|
|
.field("analyzer", "hanlp_crf")
|
|
|
|
|
.endObject();
|
|
|
|
|
}else if(participle == 7){
|
|
|
|
|
} else if (participle == 7) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","text")
|
|
|
|
|
.field("analyzer","hanlp_speed")
|
|
|
|
|
.field("type", "text")
|
|
|
|
|
.field("analyzer", "hanlp_speed")
|
|
|
|
|
.endObject();
|
|
|
|
|
}
|
|
|
|
|
}else if("datetime".equals(dateType)){
|
|
|
|
|
} else if ("datetime".equals(dateType)) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","date")
|
|
|
|
|
.field("format","yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis")
|
|
|
|
|
.field("type", "date")
|
|
|
|
|
.field("format", "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis")
|
|
|
|
|
.endObject();
|
|
|
|
|
}else if ("timestamp".equals(dateType)){
|
|
|
|
|
} else if ("timestamp".equals(dateType)) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","date")
|
|
|
|
|
.field("format","strict_date_optional_time||epoch_millis")
|
|
|
|
|
.field("type", "date")
|
|
|
|
|
.field("format", "strict_date_optional_time||epoch_millis")
|
|
|
|
|
.endObject();
|
|
|
|
|
}else if("float".equals(dateType)||"double".equals(dateType)){
|
|
|
|
|
} else if ("float".equals(dateType) || "double".equals(dateType)) {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type","scaled_float")
|
|
|
|
|
.field("scaling_factor",100)
|
|
|
|
|
.field("type", "scaled_float")
|
|
|
|
|
.field("scaling_factor", 100)
|
|
|
|
|
.endObject();
|
|
|
|
|
}else {
|
|
|
|
|
} else {
|
|
|
|
|
mapping.startObject(field)
|
|
|
|
|
.field("type",dateType)
|
|
|
|
|
.field("index",true)
|
|
|
|
|
.field("type", dateType)
|
|
|
|
|
.field("index", true)
|
|
|
|
|
.endObject();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -257,24 +258,23 @@ public abstract class EsService<T> {
|
|
|
|
|
.endObject();
|
|
|
|
|
CreateIndexRequest createIndexRequest = new CreateIndexRequest(index);
|
|
|
|
|
// 设置分片数和副本数
|
|
|
|
|
createIndexRequest.settings(createSettings(number_of_shards,number_of_replicas));
|
|
|
|
|
createIndexRequest.settings(createSettings(number_of_shards, number_of_replicas));
|
|
|
|
|
// 构建对应index的mapping
|
|
|
|
|
createIndexRequest.mapping(mapping);
|
|
|
|
|
client = ElasticSearchPoolUtil.getClient();
|
|
|
|
|
// 发送创建index的请求
|
|
|
|
|
CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest,RequestOptions.DEFAULT);
|
|
|
|
|
CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
|
|
|
|
|
return createIndexResponse.isAcknowledged();
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
logger.error("根据信息自动创建索引与mapping创建失败,失败信息为:"+ e.getMessage());
|
|
|
|
|
logger.error("根据信息自动创建索引与mapping创建失败,失败信息为:" + e.getMessage());
|
|
|
|
|
throw new RuntimeException();
|
|
|
|
|
}finally {
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public boolean indexExist() throws Exception{
|
|
|
|
|
public boolean indexExist() throws Exception {
|
|
|
|
|
RestHighLevelClient client = null;
|
|
|
|
|
GetIndexRequest request = new GetIndexRequest(index);
|
|
|
|
|
request.local(false);
|
|
|
|
@ -302,20 +302,20 @@ public abstract class EsService<T> {
|
|
|
|
|
client.index(request, RequestOptions.DEFAULT);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
throw new RuntimeException(e);
|
|
|
|
|
}finally {
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void insertBatch(List<T> list) {
|
|
|
|
|
if(list==null||list.size()==0){
|
|
|
|
|
if (list == null || list.size() == 0) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
RestHighLevelClient client = null;
|
|
|
|
|
try {
|
|
|
|
|
client = ElasticSearchPoolUtil.getClient();
|
|
|
|
|
if(!indexExist()){
|
|
|
|
|
createIndexAndCreateMapping(list.get(0).getClass(),true,1,0);
|
|
|
|
|
if (!indexExist()) {
|
|
|
|
|
createIndexAndCreateMapping(list.get(0).getClass(), true, 1, 0);
|
|
|
|
|
}
|
|
|
|
|
BulkRequest request = new BulkRequest();
|
|
|
|
|
list.forEach(item -> {
|
|
|
|
@ -331,7 +331,7 @@ public abstract class EsService<T> {
|
|
|
|
|
client.bulk(request, RequestOptions.DEFAULT);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
throw new RuntimeException(e);
|
|
|
|
|
}finally {
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -345,10 +345,11 @@ public abstract class EsService<T> {
|
|
|
|
|
client.bulk(request, RequestOptions.DEFAULT);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
throw new RuntimeException(e);
|
|
|
|
|
}finally {
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public List<T> search(SearchSourceBuilder builder) {
|
|
|
|
|
SearchRequest request = new SearchRequest(index);
|
|
|
|
|
request.source(builder);
|
|
|
|
@ -360,30 +361,30 @@ public abstract class EsService<T> {
|
|
|
|
|
List<T> res = new ArrayList<>(hits.length);
|
|
|
|
|
for (SearchHit hit : hits) {
|
|
|
|
|
//res.add(JSON.parseObject(hit.getSourceAsString(), entity));
|
|
|
|
|
res.add(loadData(builder,hit));
|
|
|
|
|
res.add(loadData(builder, hit));
|
|
|
|
|
}
|
|
|
|
|
return res;
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
throw new RuntimeException(e);
|
|
|
|
|
}finally {
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//分页查询 根据需要返回自定义加载数据 比如从mongodb mysql 等
|
|
|
|
|
public Page<T> searchPage(SearchSourceBuilder builder,boolean searchFrom) {
|
|
|
|
|
public Page<T> searchPage(SearchSourceBuilder builder, boolean searchFrom) {
|
|
|
|
|
try {
|
|
|
|
|
List<SearchHit> content = new ArrayList<>();
|
|
|
|
|
Long total = search(content,builder);
|
|
|
|
|
System.out.println("*****:"+content.get(0).getId());
|
|
|
|
|
Long total = search(content, builder);
|
|
|
|
|
System.out.println("*****:" + content.get(0).getId());
|
|
|
|
|
List<T> res = new ArrayList<>();
|
|
|
|
|
if(searchFrom){
|
|
|
|
|
if (searchFrom) {
|
|
|
|
|
res = sequenceLoadData(content);
|
|
|
|
|
}else{
|
|
|
|
|
} else {
|
|
|
|
|
res = esLoadData(content);
|
|
|
|
|
}
|
|
|
|
|
Integer pageNo = builder.from();
|
|
|
|
|
Page<T> page=new Page<>(pageNo<=0?1:pageNo/builder.size()+1,builder.size());
|
|
|
|
|
Page<T> page = new Page<>(pageNo <= 0 ? 1 : pageNo / builder.size() + 1, builder.size());
|
|
|
|
|
page.setRows(res);
|
|
|
|
|
page.setTotal(total);
|
|
|
|
|
return page;
|
|
|
|
@ -392,13 +393,15 @@ public abstract class EsService<T> {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
protected List<T> sequenceLoadData(List<SearchHit> content){
|
|
|
|
|
protected List<T> sequenceLoadData(List<SearchHit> content) {
|
|
|
|
|
return content.stream().map(x -> loadData(null, x)).filter(x -> x != null).collect(Collectors.toList());
|
|
|
|
|
}
|
|
|
|
|
protected List<T> esLoadData(List<SearchHit> content){
|
|
|
|
|
|
|
|
|
|
protected List<T> esLoadData(List<SearchHit> content) {
|
|
|
|
|
return content.stream().map(x -> JSON.parseObject(x.getSourceAsString(), entity)).filter(x -> x != null).collect(Collectors.toList());
|
|
|
|
|
}
|
|
|
|
|
protected long search(List<SearchHit> content, SearchSourceBuilder searchSourceBuilder){
|
|
|
|
|
|
|
|
|
|
protected long search(List<SearchHit> content, SearchSourceBuilder searchSourceBuilder) {
|
|
|
|
|
// 用于判断 是否 返回所有数据
|
|
|
|
|
boolean flag = true;
|
|
|
|
|
// 总数量
|
|
|
|
@ -410,7 +413,7 @@ public abstract class EsService<T> {
|
|
|
|
|
// 已经处理过了
|
|
|
|
|
int from = pageNo;
|
|
|
|
|
// 待查询页面内第一条记录的下标
|
|
|
|
|
int firstRowNum = from + 1 ;
|
|
|
|
|
int firstRowNum = from + 1;
|
|
|
|
|
// 最后一行
|
|
|
|
|
int lastRowNum = from + pageSize;
|
|
|
|
|
// 创建查询请求
|
|
|
|
@ -418,12 +421,12 @@ public abstract class EsService<T> {
|
|
|
|
|
// 构建SearchSourceBuilder
|
|
|
|
|
searchRequest.source(searchSourceBuilder);
|
|
|
|
|
// 不分页 最多返回10000条
|
|
|
|
|
if(pageNo==0 && pageSize==0){
|
|
|
|
|
if (pageNo == 0 && pageSize == 0) {
|
|
|
|
|
searchSourceBuilder.from(0).size(MAX_RESULT_WINDOW);
|
|
|
|
|
}else{
|
|
|
|
|
} else {
|
|
|
|
|
// 分页
|
|
|
|
|
// 小于 MAX_RESULT_WINDOW 10000
|
|
|
|
|
if(lastRowNum<=MAX_RESULT_WINDOW){
|
|
|
|
|
if (lastRowNum <= MAX_RESULT_WINDOW) {
|
|
|
|
|
searchSourceBuilder.from(from).size(pageSize);
|
|
|
|
|
flag = false;
|
|
|
|
|
}
|
|
|
|
@ -435,25 +438,27 @@ public abstract class EsService<T> {
|
|
|
|
|
// 查询数据超过第10000
|
|
|
|
|
if (lastRowNum > MAX_RESULT_WINDOW) {
|
|
|
|
|
// 滚动搜索
|
|
|
|
|
total = searchScroll(content,firstRowNum,lastRowNum,searchSourceBuilder,searchRequest);
|
|
|
|
|
}else{
|
|
|
|
|
total = searchScroll(content, firstRowNum, lastRowNum, searchSourceBuilder, searchRequest);
|
|
|
|
|
} else {
|
|
|
|
|
searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
|
|
|
|
SearchHits hits = searchResponse.getHits();
|
|
|
|
|
System.out.println(index);
|
|
|
|
|
SearchHit[] hits1 = hits.getHits();
|
|
|
|
|
if (hits1[0] != null) {
|
|
|
|
|
System.out.println(hits1[0].getId());
|
|
|
|
|
}
|
|
|
|
|
total = hits.getTotalHits().value;
|
|
|
|
|
// 不分页情况下 总条数超过10000
|
|
|
|
|
if(total>MAX_RESULT_WINDOW&&flag){
|
|
|
|
|
searchScroll(content,firstRowNum,(int)total,searchSourceBuilder,searchRequest);
|
|
|
|
|
}else{
|
|
|
|
|
if (total > MAX_RESULT_WINDOW && flag) {
|
|
|
|
|
searchScroll(content, firstRowNum, (int) total, searchSourceBuilder, searchRequest);
|
|
|
|
|
} else {
|
|
|
|
|
content = loadDataByResponse(content, searchResponse);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
logger.error("查询索引错误,错误如下:{}",e);
|
|
|
|
|
throw new CustomException("查询索引出错",e);
|
|
|
|
|
}finally {
|
|
|
|
|
logger.error("查询索引错误,错误如下:{}", e);
|
|
|
|
|
throw new CustomException("查询索引出错", e);
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
return total;
|
|
|
|
@ -462,6 +467,7 @@ public abstract class EsService<T> {
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 通过响应加载hit数据
|
|
|
|
|
*
|
|
|
|
|
* @param searchHitList
|
|
|
|
|
* @param searchResponse
|
|
|
|
|
* @return
|
|
|
|
@ -470,14 +476,14 @@ public abstract class EsService<T> {
|
|
|
|
|
SearchHits hits = searchResponse.getHits();
|
|
|
|
|
// 条数过多 批量加载 减少数据库连接
|
|
|
|
|
SearchHit[] hitArr = hits.getHits();
|
|
|
|
|
if(Objects.nonNull(hitArr)&&hitArr.length>=BATCH_LOAD_SIZE){
|
|
|
|
|
if (Objects.nonNull(hitArr) && hitArr.length >= BATCH_LOAD_SIZE) {
|
|
|
|
|
searchHitList.addAll(Arrays.asList(hitArr));
|
|
|
|
|
}else{
|
|
|
|
|
} else {
|
|
|
|
|
//逐条加载
|
|
|
|
|
for (SearchHit hit : hitArr) {
|
|
|
|
|
|
|
|
|
|
String id=hit.getId();
|
|
|
|
|
if (StringUtils.isBlank(id)){
|
|
|
|
|
String id = hit.getId();
|
|
|
|
|
if (StringUtils.isBlank(id)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
searchHitList.add(hit);
|
|
|
|
@ -489,6 +495,7 @@ public abstract class EsService<T> {
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* scroll 查询 返回总条数
|
|
|
|
|
*
|
|
|
|
|
* @param searchHitList
|
|
|
|
|
* @param firstRowNum
|
|
|
|
|
* @param lastRowNum
|
|
|
|
@ -496,7 +503,7 @@ public abstract class EsService<T> {
|
|
|
|
|
* @param searchRequest
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
private long searchScroll(List<SearchHit> searchHitList,int firstRowNum,int lastRowNum,SearchSourceBuilder searchSourceBuilder,SearchRequest searchRequest){
|
|
|
|
|
private long searchScroll(List<SearchHit> searchHitList, int firstRowNum, int lastRowNum, SearchSourceBuilder searchSourceBuilder, SearchRequest searchRequest) {
|
|
|
|
|
long total = 0L;
|
|
|
|
|
SearchResponse searchResponse;
|
|
|
|
|
// 初始位置
|
|
|
|
@ -504,11 +511,11 @@ public abstract class EsService<T> {
|
|
|
|
|
// 结束位置
|
|
|
|
|
int endPosition;
|
|
|
|
|
// 页面大小
|
|
|
|
|
int pageSize = lastRowNum-firstRowNum+1;
|
|
|
|
|
int pageSize = lastRowNum - firstRowNum + 1;
|
|
|
|
|
// 第一条数据所在滚动指针
|
|
|
|
|
int firstScrollCursor = firstRowNum/MAX_RESULT_WINDOW;
|
|
|
|
|
int firstScrollCursor = firstRowNum / MAX_RESULT_WINDOW;
|
|
|
|
|
// 最后一条数据所在指针
|
|
|
|
|
int lastScrollCursor = lastRowNum/MAX_RESULT_WINDOW;
|
|
|
|
|
int lastScrollCursor = lastRowNum / MAX_RESULT_WINDOW;
|
|
|
|
|
// 滚动次数
|
|
|
|
|
int scrollCount = 0;
|
|
|
|
|
// 滚动id
|
|
|
|
@ -525,33 +532,33 @@ public abstract class EsService<T> {
|
|
|
|
|
// 1分钟 内的scrollId 第一次
|
|
|
|
|
scrollId = searchResponse.getScrollId();
|
|
|
|
|
// 部分数据在10000 以内 先获取一部分
|
|
|
|
|
if(firstRowNum<MAX_RESULT_WINDOW){
|
|
|
|
|
startPosition = firstRowNum - scrollCount*MAX_RESULT_WINDOW>0?firstRowNum - scrollCount*MAX_RESULT_WINDOW:0;
|
|
|
|
|
endPosition = lastRowNum-scrollCount*MAX_RESULT_WINDOW>MAX_RESULT_WINDOW?MAX_RESULT_WINDOW:lastRowNum-scrollCount*MAX_RESULT_WINDOW;
|
|
|
|
|
if (firstRowNum < MAX_RESULT_WINDOW) {
|
|
|
|
|
startPosition = firstRowNum - scrollCount * MAX_RESULT_WINDOW > 0 ? firstRowNum - scrollCount * MAX_RESULT_WINDOW : 0;
|
|
|
|
|
endPosition = lastRowNum - scrollCount * MAX_RESULT_WINDOW > MAX_RESULT_WINDOW ? MAX_RESULT_WINDOW : lastRowNum - scrollCount * MAX_RESULT_WINDOW;
|
|
|
|
|
loopLoadDataByPosition(searchHitList, startPosition, endPosition, hits);
|
|
|
|
|
}
|
|
|
|
|
if(StringUtils.isNotEmpty(scrollId)){
|
|
|
|
|
if (StringUtils.isNotEmpty(scrollId)) {
|
|
|
|
|
// 指针还没有到最后的指针
|
|
|
|
|
while (scrollCount<lastScrollCursor){
|
|
|
|
|
while (scrollCount < lastScrollCursor) {
|
|
|
|
|
searchResponse = searchScroll(scrollId);
|
|
|
|
|
scrollId = searchResponse.getScrollId();
|
|
|
|
|
scrollCount++;
|
|
|
|
|
//
|
|
|
|
|
if(scrollCount>=firstScrollCursor){
|
|
|
|
|
if (scrollCount >= firstScrollCursor) {
|
|
|
|
|
// 从hits取对应数据 剩余容量
|
|
|
|
|
int capacity = pageSize-searchHitList.size();
|
|
|
|
|
if(capacity>0){
|
|
|
|
|
int capacity = pageSize - searchHitList.size();
|
|
|
|
|
if (capacity > 0) {
|
|
|
|
|
hits = searchResponse.getHits();
|
|
|
|
|
// 命中数量
|
|
|
|
|
int hitCount = hits.getHits().length;
|
|
|
|
|
startPosition = firstRowNum - scrollCount*MAX_RESULT_WINDOW>0?firstRowNum - scrollCount*MAX_RESULT_WINDOW:0;
|
|
|
|
|
endPosition = lastRowNum-scrollCount*MAX_RESULT_WINDOW>MAX_RESULT_WINDOW?MAX_RESULT_WINDOW:lastRowNum-scrollCount*MAX_RESULT_WINDOW;
|
|
|
|
|
startPosition = firstRowNum - scrollCount * MAX_RESULT_WINDOW > 0 ? firstRowNum - scrollCount * MAX_RESULT_WINDOW : 0;
|
|
|
|
|
endPosition = lastRowNum - scrollCount * MAX_RESULT_WINDOW > MAX_RESULT_WINDOW ? MAX_RESULT_WINDOW : lastRowNum - scrollCount * MAX_RESULT_WINDOW;
|
|
|
|
|
// 超过命中次数 取命中次数
|
|
|
|
|
endPosition = endPosition>hitCount?hitCount:endPosition;
|
|
|
|
|
if(scrollCount==firstScrollCursor)
|
|
|
|
|
endPosition = endPosition > hitCount ? hitCount : endPosition;
|
|
|
|
|
if (scrollCount == firstScrollCursor)
|
|
|
|
|
startPosition--;
|
|
|
|
|
if(firstScrollCursor!=lastScrollCursor&&scrollCount==lastScrollCursor)
|
|
|
|
|
endPosition=capacity>MAX_RESULT_WINDOW?MAX_RESULT_WINDOW:capacity-startPosition;
|
|
|
|
|
if (firstScrollCursor != lastScrollCursor && scrollCount == lastScrollCursor)
|
|
|
|
|
endPosition = capacity > MAX_RESULT_WINDOW ? MAX_RESULT_WINDOW : capacity - startPosition;
|
|
|
|
|
loopLoadDataByPosition(searchHitList, startPosition, endPosition, hits);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -560,7 +567,7 @@ public abstract class EsService<T> {
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
}finally {
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
return total;
|
|
|
|
@ -569,11 +576,12 @@ public abstract class EsService<T> {
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 滚动搜索
|
|
|
|
|
*
|
|
|
|
|
* @param scrollId 滚动Id
|
|
|
|
|
* @return
|
|
|
|
|
* @throws IOException
|
|
|
|
|
*/
|
|
|
|
|
protected SearchResponse searchScroll(String scrollId) throws Exception{
|
|
|
|
|
protected SearchResponse searchScroll(String scrollId) throws Exception {
|
|
|
|
|
RestHighLevelClient client = null;
|
|
|
|
|
client = ElasticSearchPoolUtil.getClient();
|
|
|
|
|
SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollId);
|
|
|
|
@ -587,25 +595,27 @@ public abstract class EsService<T> {
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 循环批量加载数据
|
|
|
|
|
*
|
|
|
|
|
* @param context
|
|
|
|
|
* @param startPosition
|
|
|
|
|
* @param endPosition
|
|
|
|
|
* @param hits
|
|
|
|
|
*/
|
|
|
|
|
private List<T> loopLoadDataByPosition( SearchSourceBuilder context, int startPosition, int endPosition, SearchHits hits) {
|
|
|
|
|
private List<T> loopLoadDataByPosition(SearchSourceBuilder context, int startPosition, int endPosition, SearchHits hits) {
|
|
|
|
|
int length = hits.getHits().length;
|
|
|
|
|
int destLength = endPosition-startPosition;
|
|
|
|
|
destLength = destLength>length?length:destLength;
|
|
|
|
|
int destLength = endPosition - startPosition;
|
|
|
|
|
destLength = destLength > length ? length : destLength;
|
|
|
|
|
SearchHit[] hitArr = new SearchHit[destLength];
|
|
|
|
|
System.arraycopy(hits.getHits(),startPosition,hitArr,0,hitArr.length);
|
|
|
|
|
return batchLoadData(context,hitArr);
|
|
|
|
|
System.arraycopy(hits.getHits(), startPosition, hitArr, 0, hitArr.length);
|
|
|
|
|
return batchLoadData(context, hitArr);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private List<SearchHit> loopLoadDataByPosition(List<SearchHit> searchHitList, int startPosition, int endPosition, SearchHits hits) {
|
|
|
|
|
int length = hits.getHits().length;
|
|
|
|
|
int destLength = endPosition-startPosition;
|
|
|
|
|
destLength = destLength>length?length:destLength;
|
|
|
|
|
int destLength = endPosition - startPosition;
|
|
|
|
|
destLength = destLength > length ? length : destLength;
|
|
|
|
|
SearchHit[] hitArr = new SearchHit[destLength];
|
|
|
|
|
System.arraycopy(hits.getHits(),startPosition,hitArr,0,hitArr.length);
|
|
|
|
|
System.arraycopy(hits.getHits(), startPosition, hitArr, 0, hitArr.length);
|
|
|
|
|
searchHitList.addAll(Arrays.asList(hitArr));
|
|
|
|
|
return searchHitList;
|
|
|
|
|
}
|
|
|
|
@ -617,7 +627,7 @@ public abstract class EsService<T> {
|
|
|
|
|
client.indices().delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
throw new RuntimeException(e);
|
|
|
|
|
}finally {
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -633,7 +643,7 @@ public abstract class EsService<T> {
|
|
|
|
|
client.deleteByQuery(request, RequestOptions.DEFAULT);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
throw new RuntimeException(e);
|
|
|
|
|
}finally {
|
|
|
|
|
} finally {
|
|
|
|
|
ElasticSearchPoolUtil.returnClient(client);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -668,7 +678,7 @@ public abstract class EsService<T> {
|
|
|
|
|
|
|
|
|
|
public EsQueryBuilder getDoc_InfoBy_Id(GFunction<? extends T, Object> gFunction, Object value) {
|
|
|
|
|
String field = gFunction.field();
|
|
|
|
|
field="_"+field;
|
|
|
|
|
field = "_" + field;
|
|
|
|
|
this.searchSourceBuilder.postFilter(QueryBuilders.termQuery(field, value));
|
|
|
|
|
this.boolQueryBuilder.filter(QueryBuilders.termQuery(field, value));
|
|
|
|
|
return this;
|
|
|
|
@ -677,17 +687,17 @@ public abstract class EsService<T> {
|
|
|
|
|
|
|
|
|
|
public EsQueryBuilder eqAll(T t) {
|
|
|
|
|
Field[] fields = t.getClass().getDeclaredFields();
|
|
|
|
|
if(fields!=null&&fields.length>0){
|
|
|
|
|
if (fields != null && fields.length > 0) {
|
|
|
|
|
BoolQueryBuilder funScoreBuilders = QueryBuilders.boolQuery();
|
|
|
|
|
for(Field field : fields){
|
|
|
|
|
for (Field field : fields) {
|
|
|
|
|
field.setAccessible(true);
|
|
|
|
|
String name = field.getName();
|
|
|
|
|
try {
|
|
|
|
|
Object value = field.get(t);
|
|
|
|
|
if(value==null||name==null||Constant.SERIALVERSIONUID.equals(name)){
|
|
|
|
|
if (value == null || name == null || Constant.SERIALVERSIONUID.equals(name)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
funScoreBuilders = boolTermQueryBuild( funScoreBuilders,name,value);
|
|
|
|
|
funScoreBuilders = boolTermQueryBuild(funScoreBuilders, name, value);
|
|
|
|
|
} catch (IllegalAccessException e) {
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
}
|
|
|
|
@ -709,6 +719,7 @@ public abstract class EsService<T> {
|
|
|
|
|
.boostMode(CombineFunction.REPLACE)
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public EsQueryBuilder notEq(GFunction<? extends T, Object> gFunction, Object value) {
|
|
|
|
|
String field = gFunction.field();
|
|
|
|
|
this.searchSourceBuilder.postFilter(QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(field, value)));
|
|
|
|
@ -865,8 +876,8 @@ public abstract class EsService<T> {
|
|
|
|
|
return search(this.searchSourceBuilder);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public Page<T> queryPage(boolean searchfrom){
|
|
|
|
|
return searchPage(this.searchSourceBuilder,searchfrom);
|
|
|
|
|
public Page<T> queryPage(boolean searchfrom) {
|
|
|
|
|
return searchPage(this.searchSourceBuilder, searchfrom);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void delete() {
|
|
|
|
@ -891,7 +902,7 @@ public abstract class EsService<T> {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//分词 返回分词结果
|
|
|
|
|
public List<String> analyzeTitle(DocInfo docInfo){
|
|
|
|
|
public List<String> analyzeTitle(DocInfo docInfo) {
|
|
|
|
|
try {
|
|
|
|
|
RestHighLevelClient client = ElasticSearchPoolUtil.getClient();
|
|
|
|
|
|
|
|
|
@ -903,7 +914,7 @@ public abstract class EsService<T> {
|
|
|
|
|
Response response = client.getLowLevelClient().performRequest(request);
|
|
|
|
|
JSONObject tokens = JSONObject.parseObject(EntityUtils.toString(response.getEntity()));
|
|
|
|
|
JSONArray arrays = tokens.getJSONArray("tokens");
|
|
|
|
|
List<String> list=new ArrayList<>();
|
|
|
|
|
List<String> list = new ArrayList<>();
|
|
|
|
|
for (int i = 0; i < arrays.size(); i++) {
|
|
|
|
|
JSONObject obj = JSON.parseObject(arrays.getString(i));
|
|
|
|
|
list.add(obj.getString("token"));
|
|
|
|
|