diff --git a/README.url b/README.url new file mode 100644 index 0000000..e69de29 diff --git a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/controller/DocInfoController.java b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/controller/DocInfoController.java index cb2da75..7aac7f6 100644 --- a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/controller/DocInfoController.java +++ b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/controller/DocInfoController.java @@ -7,6 +7,7 @@ import com.hankcs.hanlp.corpus.document.sentence.word.IWord; import com.hankcs.hanlp.corpus.document.sentence.word.Word; import com.ruoyi.biemo.business.domain.Category; import com.ruoyi.biemo.business.domain.DocInfo; +import com.ruoyi.biemo.mongodb.entity.SplitWordDrill; import com.ruoyi.biemo.business.service.CategoryService; import com.ruoyi.biemo.business.service.DocInfoService; import com.ruoyi.biemo.core.page.Page; @@ -20,14 +21,17 @@ import com.ruoyi.common.core.domain.AjaxResult; import com.ruoyi.common.enums.BusinessType; import com.ruoyi.common.utils.poi.ExcelUtil; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.util.ArrayList; +import java.util.Date; import java.util.List; -import java.util.Map; /** @@ -39,6 +43,8 @@ import java.util.Map; @RequestMapping(value = "/makesoft/docInfo") public class DocInfoController extends BaseController { + @Autowired + private MongoTemplate mongoTemplate; @Autowired private DocInfoService docInfoService; @Autowired @@ -93,14 +99,17 @@ public class DocInfoController extends BaseController { public AjaxResult getInfoDelHtml(@PathVariable("id") String docInfoId) throws IOException { DocInfo docInfo = docInfoService.selectDocInfoById(docInfoId); JSONObject result = new JSONObject(); + if (docInfo == null) { + return AjaxResult.error("未查询到文本"); + } String content = docInfo.getContent(); content = MyObjects.delHTMLTag(content); content = MyObjects.delSpace(content); JSONArray labels = new JSONArray(); - result.put("content",content); - result.put("connections",new JSONArray()); - result.put("connectionCategories",MyObjects.connectionCategories); - result.put("labelCategories",MyObjects.labelCategories); + result.put("content", content); + result.put("connections", new JSONArray()); + result.put("connectionCategories", MyObjects.connectionCategories); + result.put("labelCategories", MyObjects.labelCategories); Sentence sentence = DependencyParserUtils.nerAnalysis(content); List<IWord> wordList = sentence.wordList; if(wordList!=null&&wordList.size()>0){ @@ -219,11 +228,30 @@ public class DocInfoController extends BaseController { public AjaxResult analysis(@PathVariable String id){ return AjaxResult.success(docInfoService.analysis(id,false)); } + + + // @GetMapping("/socialAnalysis") +// public AjaxResult socialAnalysis(@RequestParam String content) { +// return AjaxResult.success(docInfoService.socialAnalysis(content)); +// } + //社会网络分析--用户输入 + @PostMapping("/socialAnalysis") + public AjaxResult socialAnalysis(@RequestBody String content) { + return AjaxResult.success(docInfoService.socialAnalysis(content)); + } + + //文章管理--批量分析 @GetMapping("/batchAnalysis/{ids}") public AjaxResult batchAnalysis(@PathVariable String[] ids){ return AjaxResult.success(docInfoService.batchAnalysis(ids)); } + //社会网络分析 + @PostMapping("/socialNetworkAnalysis") + public AjaxResult batchAnalysisByIds(@RequestBody String [] ids) { + return AjaxResult.success(docInfoService.socialNetworkAnalysis(ids)); + } + @GetMapping("/summary/{id}") public AjaxResult summary(@PathVariable String id){ return AjaxResult.success(docInfoService.summary(id)); @@ -252,7 +280,40 @@ public class DocInfoController extends BaseController { return AjaxResult.success("分词成功",docInfo); } - //分词结果上传 + /** + * 分词结果上传 + * + * @param docInfo + * @return + */ + @PostMapping("/uploadSplitWord") + public AjaxResult uploadsplitWord(@RequestBody DocInfo docInfo) { + Query query = new Query(Criteria.where("_id").is(docInfo.getId())); + Update update = new Update(); + update.set("splitWordResult", docInfo.getSplitWordResult()); + mongoTemplate.upsert(query, update, DocInfo.class); + return AjaxResult.success("上传成功"); + } + + /** + * 分词词训练结果上传 + * + * @param splitWordDrill + * @return + */ + @PostMapping("/uploadSplitWordDrill") + public AjaxResult uploadSplitWordDrill(@RequestBody SplitWordDrill splitWordDrill) { + for (String data : splitWordDrill.getSplitResult()) { + mongoTemplate.updateFirst( + Query.query(Criteria.where("_id").is(splitWordDrill.getId())), // 查询条件 + new Update() + .push("splitResult").each(data) + .currentDate("updateTime"), +// .currentTimestamp("updateTime"), + "splitWordDrill"); + } + return AjaxResult.success("上传成功"); + } } diff --git a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java index c7caac3..cd93863 100644 --- a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java +++ b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java @@ -63,4 +63,7 @@ public class DocInfo extends BiemoEntity { //分词数据 private List<String> analyzeTitle; + //分词结果 + private String splitWordResult; + } diff --git a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/service/DocInfoService.java b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/service/DocInfoService.java index 36f732b..2e0b2fd 100644 --- a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/service/DocInfoService.java +++ b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/service/DocInfoService.java @@ -14,6 +14,7 @@ import com.ruoyi.biemo.mongodb.utils.MongoHelper; import com.ruoyi.biemo.nlp.DependencyParserUtils; import com.ruoyi.biemo.nlp.SummaryUtils; import com.ruoyi.biemo.utils.MyObjects; +import com.ruoyi.common.core.domain.AjaxResult; import com.ruoyi.common.utils.StringUtils; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -47,33 +48,34 @@ public class DocInfoService extends EsService<DocInfo> { protected DocInfo loadData(SearchSourceBuilder context, SearchHit hit) { String id = hit.getId(); if (Objects.isNull(id) || StringUtil.isEmpty(id)) return null; - return mongoHelper.findById(id,DocInfo.class); + return mongoHelper.findById(id, DocInfo.class); } + @Override public List<DocInfo> batchLoadData(SearchSourceBuilder context, SearchHit[] hitArr) { List<String> ids = new ArrayList<>(); - for(SearchHit hit:hitArr){ + for (SearchHit hit : hitArr) { String id = hit.getId(); if (Objects.isNull(id) || StringUtil.isEmpty(id)) continue; ids.add(id); } - return mongoHelper.findListByIds(ids,DocInfo.class); + return mongoHelper.findListByIds(ids, DocInfo.class); } + /** * 查询文章管理 * * @param id 文章管理主键 * @return 文章管理 */ - public DocInfo selectDocInfoById(String id) - { - if(id==null||id.trim().length()==0){ + public DocInfo selectDocInfoById(String id) { + if (id == null || id.trim().length() == 0) { return null; } // List<DocInfo> docInfos = esLambdaQuery().eq(DocInfo::getId,id).query(); - List<DocInfo> docInfos = esLambdaQuery().getDoc_InfoBy_Id(DocInfo::getId,id).query(); + List<DocInfo> docInfos = esLambdaQuery().getDoc_InfoBy_Id(DocInfo::getId, id).query(); - return docInfos!=null&&docInfos.size()>0?docInfos.get(0):null; + return docInfos != null && docInfos.size() > 0 ? docInfos.get(0) : null; } /** @@ -82,8 +84,7 @@ public class DocInfoService extends EsService<DocInfo> { * @param docInfo 文章管理 * @return 文章管理 */ - public List<DocInfo> selectDocInfoList(DocInfo docInfo) - { + public List<DocInfo> selectDocInfoList(DocInfo docInfo) { return esLambdaQuery().eqAll(docInfo).query(); } @@ -93,8 +94,8 @@ public class DocInfoService extends EsService<DocInfo> { * @param docInfo 文章管理 * @return 文章管理 */ - public Page<DocInfo> selectDocInfoPage(DocInfo docInfo, Page<DocInfo> page){ - return esLambdaQuery().eqAll(docInfo).sort(DocInfo::getCreateTime,SortOrder.DESC).page(page.getPageNum(),page.getPageSize()).queryPage(true); + public Page<DocInfo> selectDocInfoPage(DocInfo docInfo, Page<DocInfo> page) { + return esLambdaQuery().eqAll(docInfo).sort(DocInfo::getCreateTime, SortOrder.DESC).page(page.getPageNum(), page.getPageSize()).queryPage(true); } /** @@ -103,18 +104,18 @@ public class DocInfoService extends EsService<DocInfo> { * @param docInfo 文章管理 * @return 结果 */ - public String insertOrUpdateDocInfo(DocInfo docInfo) - { + public String insertOrUpdateDocInfo(DocInfo docInfo) { String id = mongoHelper.insert(docInfo); docInfo.setId(id); - applicationContext.publishEvent(new DocInfoSaveEvent(this,docInfo)); + applicationContext.publishEvent(new DocInfoSaveEvent(this, docInfo)); return id; } - public void insertBatchEs(List<DocInfo> docInfos){ + public void insertBatchEs(List<DocInfo> docInfos) { insertBatch(docInfos); } - public void insertEs(DocInfo docInfo){ + + public void insertEs(DocInfo docInfo) { List<DocInfo> docInfos = new ArrayList<>(); docInfos.add(docInfo); insertBatch(docInfos); @@ -127,11 +128,10 @@ public class DocInfoService extends EsService<DocInfo> { * @param ids 需要删除的文章管理主键 * @return 结果 */ - public void deleteDocInfoByIds(String[] ids) - { - List<String> list = Arrays.stream(ids).collect(Collectors.toList()); + public void deleteDocInfoByIds(String[] ids) { + List<String> list = Arrays.stream(ids).collect(Collectors.toList()); mongoHelper.deleteByIds(list, DocInfo.class); - applicationContext.publishEvent(new DocInfoDeleteEvent(this,list)); + applicationContext.publishEvent(new DocInfoDeleteEvent(this, list)); } @@ -141,22 +141,21 @@ public class DocInfoService extends EsService<DocInfo> { * @param id 文章管理主键 * @return 结果 */ - public void deleteDocInfoById(String id) - { - mongoHelper.deleteById(id,DocInfo.class); - applicationContext.publishEvent(new DocInfoDeleteEvent(this,id)); + public void deleteDocInfoById(String id) { + mongoHelper.deleteById(id, DocInfo.class); + applicationContext.publishEvent(new DocInfoDeleteEvent(this, id)); } @EventListener(classes = {DocInfoSaveEvent.class}) public void saveIndex(ApplicationEvent event) { - DocInfoSaveEvent saveEvent = (DocInfoSaveEvent)event; + DocInfoSaveEvent saveEvent = (DocInfoSaveEvent) event; List<DocInfo> infoList = saveEvent.getInfos(); insertBatch(infoList); } @EventListener(classes = {DocInfoDeleteEvent.class}) public void delIndex(ApplicationEvent event) { - DocInfoDeleteEvent saveEvent = (DocInfoDeleteEvent)event; + DocInfoDeleteEvent saveEvent = (DocInfoDeleteEvent) event; List<String> ids = saveEvent.getIds(); deleteBatch(ids); } @@ -170,7 +169,7 @@ public class DocInfoService extends EsService<DocInfo> { DocInfo docInfo = this.selectDocInfoById(id); String grapData = docInfo.getParserGraphData(); if(StringUtils.isNotBlank(grapData)&&!reAnalysis){ - response = JSONObject.parseObject(grapData,MyResultResponse.class); + response = JSONObject.parseObject(grapData,MyResultResponse.class); if(response!=null){ return response; } @@ -188,35 +187,35 @@ public class DocInfoService extends EsService<DocInfo> { int i = 1; //Map<String,String> nodesMap = getAllNodes(allRelations); - if(allRelations!=null&&allRelations.size()>0){ - for (String relation : allRelations) { - String[] nrn = StringUtils.isNotBlank(relation)?relation.split(","):null; - if(nrn==null||nrn.length<3){ - continue; - } - String nodeName1 = nrn[0]; - String relationName = nrn[1]; - String nodeName2 = nrn[2]; - Node node1 = new Node(); - //node1.setId(nodesMap.get(nodeName1)); - node1.setId(nodeName1); - node1.setLabels(new ArrayList<String>(){{add(nodeName1);}}); - node1.setProperties(new HashMap<String,Object>(){{put("name",nodeName1);put("email","9094908@qq.com");}}); - Node node2 = new Node(); - node2.setId(nodeName2); - node2.setLabels(new ArrayList<String>(){{add(nodeName2);}}); - node2.setProperties(new HashMap<String,Object>(){{put("name",nodeName2);put("email","9094908@qq.com");}}); - nodes.add(node1); - nodes.add(node2); - Relationship relationship = new Relationship(); - relationship.setId(String.valueOf(i++)); - relationship.setType(relationName); - relationship.setStartNode(nodeName1); - relationship.setEndNode(nodeName2); - relationship.setProperties(new HashMap<String,Object>(){{put("source",nodeName1);put("relation",relationName);put("target",nodeName2);}}); - relationships.add(relationship); - } - } + if(allRelations!=null&&allRelations.size()>0){ + for (String relation : allRelations) { + String[] nrn = StringUtils.isNotBlank(relation)?relation.split(","):null; + if(nrn==null||nrn.length<3){ + continue; + } + String nodeName1 = nrn[0]; + String relationName = nrn[1]; + String nodeName2 = nrn[2]; + Node node1 = new Node(); + //node1.setId(nodesMap.get(nodeName1)); + node1.setId(nodeName1); + node1.setLabels(new ArrayList<String>(){{add(nodeName1);}}); + node1.setProperties(new HashMap<String,Object>(){{put("name",nodeName1);put("email","9094908@qq.com");}}); + Node node2 = new Node(); + node2.setId(nodeName2); + node2.setLabels(new ArrayList<String>(){{add(nodeName2);}}); + node2.setProperties(new HashMap<String,Object>(){{put("name",nodeName2);put("email","9094908@qq.com");}}); + nodes.add(node1); + nodes.add(node2); + Relationship relationship = new Relationship(); + relationship.setId(String.valueOf(i++)); + relationship.setType(relationName); + relationship.setStartNode(nodeName1); + relationship.setEndNode(nodeName2); + relationship.setProperties(new HashMap<String,Object>(){{put("source",nodeName1);put("relation",relationName);put("target",nodeName2);}}); + relationships.add(relationship); + } + } //获取命名实体 try { Map<String,Set<String>> nerTagSet = DependencyParserUtils.getMyNERTagSet(strArr); @@ -232,21 +231,22 @@ public class DocInfoService extends EsService<DocInfo> { insertOrUpdateDocInfo(docInfo); return response; } + private Map<String, String> getAllNodes(Set<String> allRelations) { - Map<String,String> result = new HashMap<>(); + Map<String, String> result = new HashMap<>(); Set<String> nodeSets = new HashSet<>(); - if(allRelations!=null&&allRelations.size()>0){ - allRelations.forEach(relation->{ - String[] nrn = StringUtils.isNotBlank(relation)?relation.split(","):null; - if(nrn!=null&&nrn.length==3){ + if (allRelations != null && allRelations.size() > 0) { + allRelations.forEach(relation -> { + String[] nrn = StringUtils.isNotBlank(relation) ? relation.split(",") : null; + if (nrn != null && nrn.length == 3) { nodeSets.add(nrn[0]); nodeSets.add(nrn[2]); } }); } AtomicInteger i = new AtomicInteger(0); - if(nodeSets!=null&&nodeSets.size()>0){ - nodeSets.forEach(set->{ + if (nodeSets != null && nodeSets.size() > 0) { + nodeSets.forEach(set -> { result.put(set, String.valueOf(i.getAndIncrement())); }); } @@ -256,8 +256,8 @@ public class DocInfoService extends EsService<DocInfo> { private Set<String> getAllRelations(String[] strArr) { Set<String> allRelations = new HashSet<>(); - if(strArr!=null&&strArr.length>0){ - for(String text : strArr) { + if (strArr != null && strArr.length > 0) { + for (String text : strArr) { Set<String> relations = DependencyParserUtils.analysis(text); allRelations.addAll(relations); } @@ -265,7 +265,7 @@ public class DocInfoService extends EsService<DocInfo> { return allRelations; } - public MyResultResponse returnResponse(List<Node> nodes,List<Relationship> relationships) { + public MyResultResponse returnResponse(List<Node> nodes, List<Relationship> relationships) { MyResultResponse response = new MyResultResponse(); List<MyResultResponse.Datas> datass = new ArrayList<>(); MyResultResponse.Datas datas = new MyResultResponse.Datas(); @@ -276,7 +276,10 @@ public class DocInfoService extends EsService<DocInfo> { graph.setRelationships(relationships); data.setGraph(graph); dataList.add(data); - datas.setColumns(new ArrayList<String>(){{add("user");add("entity");}}); + datas.setColumns(new ArrayList<String>() {{ + add("user"); + add("entity"); + }}); datas.setData(dataList); datass.add(datas); response.setResults(datass); @@ -292,6 +295,24 @@ public class DocInfoService extends EsService<DocInfo> { return summary; } +// 不用集合耗时:293270952500 不用集合耗时:245180467700 +// } + //社会网络分析 + public List<MyResultResponse> socialNetworkAnalysis(String[] ids) { + List<MyResultResponse> list = new ArrayList<>(); + long stratTime = System.nanoTime(); + if (ids != null && ids.length > 0) { + for (int i = 0; i < ids.length; i++) { +// list.add(analysis(ids[i], true)); + list.add(analysis(ids[i], true)); + } + } + long endTime = System.nanoTime(); + System.out.println("不用集合耗时:" + (endTime - stratTime)); + return list; + } + + //文章管理--批量分析 public String batchAnalysis(String[] ids) { if(ids!=null&&ids.length>0){ for(int i=0;i<ids.length;i++) { @@ -300,4 +321,86 @@ public class DocInfoService extends EsService<DocInfo> { } return "ok"; } + + + public MyResultResponse socialAnalysis(String content) { +// String grapData = docInfo.getParserGraphData(); + MyResultResponse response = null; +// if(StringUtils.isNotBlank(grapData)&&reAnalysis){ +// response = JSONObject.parseObject(grapData,MyResultResponse.class); +// if(response!=null){ +// return response; +// } +// } + List<Node> nodes = new ArrayList<>(); + List<Relationship> relationships = new ArrayList<>(); + //拆句 + if (StringUtils.isNotBlank(content)) { + content = MyObjects.delHTMLTag(content); + content = MyObjects.delSpace(content); + String regx = "\n|!|\\.|\\。|\\;|\\;|\\!|\\,|\\,|\\、|\\@|\\#|\\$|\\¥|\\%|\\&|\\*|\\(|\\)|\\=|\\+|\\-|\\_|\\>|\\<|\\[|\\]|\\【|\\】|\\?|\\?|”|\\\""; + String[] strArr = content.split(regx); + Set<String> allRelations = getAllRelations(strArr); + int i = 1; +// Map<String,String> nodesMap = getAllNodes(allRelations); + + if (allRelations != null && allRelations.size() > 0) { + for (String relation : allRelations) { + String[] nrn = StringUtils.isNotBlank(relation) ? relation.split(",") : null; + if (nrn == null || nrn.length < 3) { + continue; + } + String nodeName1 = nrn[0]; + String relationName = nrn[1]; + String nodeName2 = nrn[2]; + Node node1 = new Node(); + //node1.setId(nodesMap.get(nodeName1)); + node1.setId(nodeName1); + node1.setLabels(new ArrayList<String>() {{ + add(nodeName1); + }}); + node1.setProperties(new HashMap<String, Object>() {{ + put("name", nodeName1); + put("email", "9094908@qq.com"); + }}); + Node node2 = new Node(); + node2.setId(nodeName2); + node2.setLabels(new ArrayList<String>() {{ + add(nodeName2); + }}); + node2.setProperties(new HashMap<String, Object>() {{ + put("name", nodeName2); + put("email", "9094908@qq.com"); + }}); + nodes.add(node1); + nodes.add(node2); + Relationship relationship = new Relationship(); + relationship.setId(String.valueOf(i++)); + relationship.setType(relationName); + relationship.setStartNode(nodeName1); + relationship.setEndNode(nodeName2); + relationship.setProperties(new HashMap<String, Object>() {{ + put("source", nodeName1); + put("relation", relationName); + put("target", nodeName2); + }}); + relationships.add(relationship); + } + } +// //获取命名实体 +// try { +// Map<String,Set<String>> nerTagSet = DependencyParserUtils.getMyNERTagSet(strArr); +// docInfo.setParserNamedEntity(JSONObject.toJSONString(nerTagSet)); +// docInfo.setSummary(SummaryUtils.autoSummary(content)); +// } catch (Exception e) { +// e.printStackTrace(); +// } + } + response = returnResponse(nodes, relationships); +// docInfo.setParserGraphData(JSONObject.toJSONString(response)); +// docInfo.setStatus(1); +// insertOrUpdateDocInfo(docInfo); + return response; + } + } diff --git a/ruoyi-biemo/src/main/java/com/ruoyi/biemo/mongodb/entity/SplitWordDrill.java b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/mongodb/entity/SplitWordDrill.java new file mode 100644 index 0000000..fd2fec0 --- /dev/null +++ b/ruoyi-biemo/src/main/java/com/ruoyi/biemo/mongodb/entity/SplitWordDrill.java @@ -0,0 +1,21 @@ +package com.ruoyi.biemo.mongodb.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.data.mongodb.core.mapping.Field; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Document(collection ="splitWordDrill") +public class SplitWordDrill { + @Id + private final String id = "6466d460f07e000095000787"; + @Field + private String [] splitResult; + @Field + private String updateTime; +} \ No newline at end of file