tianzeOrg
/
TextBigData
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

上传和社会网络分析

Browse Source
master
xiaoCJ 2 years ago
parent b8e3d55a57
commit fb8b905b46
5 changed files with 263 additions and 75 deletions
Show Stats Download Patch File Download Diff File

0
README.url
Unescape Escape View File

75
ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/controller/DocInfoController.java
Unescape Escape View File

@ -7,6 +7,7 @@ import com.hankcs.hanlp.corpus.document.sentence.word.IWord;
import com.hankcs.hanlp.corpus.document.sentence.word.Word;
import com.ruoyi.biemo.business.domain.Category;
import com.ruoyi.biemo.business.domain.DocInfo;
import com.ruoyi.biemo.mongodb.entity.SplitWordDrill;
import com.ruoyi.biemo.business.service.CategoryService;
import com.ruoyi.biemo.business.service.DocInfoService;
import com.ruoyi.biemo.core.page.Page;
@ -20,14 +21,17 @@ import com.ruoyi.common.core.domain.AjaxResult;
import com.ruoyi.common.enums.BusinessType;
import com.ruoyi.common.utils.poi.ExcelUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
@ -39,6 +43,8 @@ import java.util.Map;
@RequestMapping(value = "/makesoft/docInfo")
public class DocInfoController extends BaseController {
@Autowired
private MongoTemplate mongoTemplate;
@Autowired
private DocInfoService docInfoService;
@Autowired
@ -93,14 +99,17 @@ public class DocInfoController extends BaseController {
public AjaxResult getInfoDelHtml(@PathVariable("id") String docInfoId) throws IOException {
DocInfo docInfo = docInfoService.selectDocInfoById(docInfoId);
JSONObject result = new JSONObject();
if (docInfo == null) {
return AjaxResult.error("未查询到文本");
}
String content = docInfo.getContent();
content = MyObjects.delHTMLTag(content);
content = MyObjects.delSpace(content);
JSONArray labels = new JSONArray();
result.put("content",content);
result.put("connections",new JSONArray());
result.put("connectionCategories",MyObjects.connectionCategories);
result.put("labelCategories",MyObjects.labelCategories);
result.put("content", content);
result.put("connections", new JSONArray());
result.put("connectionCategories", MyObjects.connectionCategories);
result.put("labelCategories", MyObjects.labelCategories);
Sentence sentence = DependencyParserUtils.nerAnalysis(content);
List<IWord> wordList = sentence.wordList;
if(wordList!=null&&wordList.size()>0){
@ -219,11 +228,30 @@ public class DocInfoController extends BaseController {
public AjaxResult analysis(@PathVariable String id){
return AjaxResult.success(docInfoService.analysis(id,false));
}
// @GetMapping("/socialAnalysis")
// public AjaxResult socialAnalysis(@RequestParam String content) {
// return AjaxResult.success(docInfoService.socialAnalysis(content));
// }
//社会网络分析--用户输入
@PostMapping("/socialAnalysis")
public AjaxResult socialAnalysis(@RequestBody String content) {
return AjaxResult.success(docInfoService.socialAnalysis(content));
}
//文章管理--批量分析
@GetMapping("/batchAnalysis/{ids}")
public AjaxResult batchAnalysis(@PathVariable String[] ids){
return AjaxResult.success(docInfoService.batchAnalysis(ids));
}
//社会网络分析
@PostMapping("/socialNetworkAnalysis")
public AjaxResult batchAnalysisByIds(@RequestBody String [] ids) {
return AjaxResult.success(docInfoService.socialNetworkAnalysis(ids));
}
@GetMapping("/summary/{id}")
public AjaxResult summary(@PathVariable String id){
return AjaxResult.success(docInfoService.summary(id));
@ -252,7 +280,40 @@ public class DocInfoController extends BaseController {
return AjaxResult.success("分词成功",docInfo);
}
//分词结果上传
/**
* 分词结果上传
*
* @param docInfo
* @return
*/
@PostMapping("/uploadSplitWord")
public AjaxResult uploadsplitWord(@RequestBody DocInfo docInfo) {
Query query = new Query(Criteria.where("_id").is(docInfo.getId()));
Update update = new Update();
update.set("splitWordResult", docInfo.getSplitWordResult());
mongoTemplate.upsert(query, update, DocInfo.class);
return AjaxResult.success("上传成功");
}
/**
* 分词词训练结果上传
*
* @param splitWordDrill
* @return
*/
@PostMapping("/uploadSplitWordDrill")
public AjaxResult uploadSplitWordDrill(@RequestBody SplitWordDrill splitWordDrill) {
for (String data : splitWordDrill.getSplitResult()) {
mongoTemplate.updateFirst(
Query.query(Criteria.where("_id").is(splitWordDrill.getId())), // 查询条件
new Update()
.push("splitResult").each(data)
.currentDate("updateTime"),
// .currentTimestamp("updateTime"),
"splitWordDrill");
}
return AjaxResult.success("上传成功");
}
}

3
ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java
Unescape Escape View File

@ -63,4 +63,7 @@ public class DocInfo extends BiemoEntity {
//分词数据
private List<String> analyzeTitle;
//分词结果
private String splitWordResult;
}

177
ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/service/DocInfoService.java
Unescape Escape View File

@ -14,6 +14,7 @@ import com.ruoyi.biemo.mongodb.utils.MongoHelper;
import com.ruoyi.biemo.nlp.DependencyParserUtils;
import com.ruoyi.biemo.nlp.SummaryUtils;
import com.ruoyi.biemo.utils.MyObjects;
import com.ruoyi.common.core.domain.AjaxResult;
import com.ruoyi.common.utils.StringUtils;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
@ -47,33 +48,34 @@ public class DocInfoService extends EsService<DocInfo> {
protected DocInfo loadData(SearchSourceBuilder context, SearchHit hit) {
String id = hit.getId();
if (Objects.isNull(id) || StringUtil.isEmpty(id)) return null;
return mongoHelper.findById(id,DocInfo.class);
return mongoHelper.findById(id, DocInfo.class);
}
@Override
public List<DocInfo> batchLoadData(SearchSourceBuilder context, SearchHit[] hitArr) {
List<String> ids = new ArrayList<>();
for(SearchHit hit:hitArr){
for (SearchHit hit : hitArr) {
String id = hit.getId();
if (Objects.isNull(id) || StringUtil.isEmpty(id)) continue;
ids.add(id);
}
return mongoHelper.findListByIds(ids,DocInfo.class);
return mongoHelper.findListByIds(ids, DocInfo.class);
}
/**
* 查询文章管理
*
* @param id 文章管理主键
* @return 文章管理
*/
public DocInfo selectDocInfoById(String id)
{
if(id==null||id.trim().length()==0){
public DocInfo selectDocInfoById(String id) {
if (id == null || id.trim().length() == 0) {
return null;
}
// List<DocInfo> docInfos = esLambdaQuery().eq(DocInfo::getId,id).query();
List<DocInfo> docInfos = esLambdaQuery().getDoc_InfoBy_Id(DocInfo::getId,id).query();
List<DocInfo> docInfos = esLambdaQuery().getDoc_InfoBy_Id(DocInfo::getId, id).query();
return docInfos!=null&&docInfos.size()>0?docInfos.get(0):null;
return docInfos != null && docInfos.size() > 0 ? docInfos.get(0) : null;
}
/**
@ -82,8 +84,7 @@ public class DocInfoService extends EsService<DocInfo> {
* @param docInfo 文章管理
* @return 文章管理
*/
public List<DocInfo> selectDocInfoList(DocInfo docInfo)
{
public List<DocInfo> selectDocInfoList(DocInfo docInfo) {
return esLambdaQuery().eqAll(docInfo).query();
}
@ -93,8 +94,8 @@ public class DocInfoService extends EsService<DocInfo> {
* @param docInfo 文章管理
* @return 文章管理
*/
public Page<DocInfo> selectDocInfoPage(DocInfo docInfo, Page<DocInfo> page){
return esLambdaQuery().eqAll(docInfo).sort(DocInfo::getCreateTime,SortOrder.DESC).page(page.getPageNum(),page.getPageSize()).queryPage(true);
public Page<DocInfo> selectDocInfoPage(DocInfo docInfo, Page<DocInfo> page) {
return esLambdaQuery().eqAll(docInfo).sort(DocInfo::getCreateTime, SortOrder.DESC).page(page.getPageNum(), page.getPageSize()).queryPage(true);
}
/**
@ -103,18 +104,18 @@ public class DocInfoService extends EsService<DocInfo> {
* @param docInfo 文章管理
* @return 结果
*/
public String insertOrUpdateDocInfo(DocInfo docInfo)
{
public String insertOrUpdateDocInfo(DocInfo docInfo) {
String id = mongoHelper.insert(docInfo);
docInfo.setId(id);
applicationContext.publishEvent(new DocInfoSaveEvent(this,docInfo));
applicationContext.publishEvent(new DocInfoSaveEvent(this, docInfo));
return id;
}
public void insertBatchEs(List<DocInfo> docInfos){
public void insertBatchEs(List<DocInfo> docInfos) {
insertBatch(docInfos);
}
public void insertEs(DocInfo docInfo){
public void insertEs(DocInfo docInfo) {
List<DocInfo> docInfos = new ArrayList<>();
docInfos.add(docInfo);
insertBatch(docInfos);
@ -127,11 +128,10 @@ public class DocInfoService extends EsService<DocInfo> {
* @param ids 需要删除的文章管理主键
* @return 结果
*/
public void deleteDocInfoByIds(String[] ids)
{
public void deleteDocInfoByIds(String[] ids) {
List<String> list = Arrays.stream(ids).collect(Collectors.toList());
mongoHelper.deleteByIds(list, DocInfo.class);
applicationContext.publishEvent(new DocInfoDeleteEvent(this,list));
applicationContext.publishEvent(new DocInfoDeleteEvent(this, list));
}
@ -141,22 +141,21 @@ public class DocInfoService extends EsService<DocInfo> {
* @param id 文章管理主键
* @return 结果
*/
public void deleteDocInfoById(String id)
{
mongoHelper.deleteById(id,DocInfo.class);
applicationContext.publishEvent(new DocInfoDeleteEvent(this,id));
public void deleteDocInfoById(String id) {
mongoHelper.deleteById(id, DocInfo.class);
applicationContext.publishEvent(new DocInfoDeleteEvent(this, id));
}
@EventListener(classes = {DocInfoSaveEvent.class})
public void saveIndex(ApplicationEvent event) {
DocInfoSaveEvent saveEvent = (DocInfoSaveEvent)event;
DocInfoSaveEvent saveEvent = (DocInfoSaveEvent) event;
List<DocInfo> infoList = saveEvent.getInfos();
insertBatch(infoList);
}
@EventListener(classes = {DocInfoDeleteEvent.class})
public void delIndex(ApplicationEvent event) {
DocInfoDeleteEvent saveEvent = (DocInfoDeleteEvent)event;
DocInfoDeleteEvent saveEvent = (DocInfoDeleteEvent) event;
List<String> ids = saveEvent.getIds();
deleteBatch(ids);
}
@ -232,21 +231,22 @@ public class DocInfoService extends EsService<DocInfo> {
insertOrUpdateDocInfo(docInfo);
return response;
}
private Map<String, String> getAllNodes(Set<String> allRelations) {
Map<String,String> result = new HashMap<>();
Map<String, String> result = new HashMap<>();
Set<String> nodeSets = new HashSet<>();
if(allRelations!=null&&allRelations.size()>0){
allRelations.forEach(relation->{
String[] nrn = StringUtils.isNotBlank(relation)?relation.split(","):null;
if(nrn!=null&&nrn.length==3){
if (allRelations != null && allRelations.size() > 0) {
allRelations.forEach(relation -> {
String[] nrn = StringUtils.isNotBlank(relation) ? relation.split(",") : null;
if (nrn != null && nrn.length == 3) {
nodeSets.add(nrn[0]);
nodeSets.add(nrn[2]);
}
});
}
AtomicInteger i = new AtomicInteger(0);
if(nodeSets!=null&&nodeSets.size()>0){
nodeSets.forEach(set->{
if (nodeSets != null && nodeSets.size() > 0) {
nodeSets.forEach(set -> {
result.put(set, String.valueOf(i.getAndIncrement()));
});
}
@ -256,8 +256,8 @@ public class DocInfoService extends EsService<DocInfo> {
private Set<String> getAllRelations(String[] strArr) {
Set<String> allRelations = new HashSet<>();
if(strArr!=null&&strArr.length>0){
for(String text : strArr) {
if (strArr != null && strArr.length > 0) {
for (String text : strArr) {
Set<String> relations = DependencyParserUtils.analysis(text);
allRelations.addAll(relations);
}
@ -265,7 +265,7 @@ public class DocInfoService extends EsService<DocInfo> {
return allRelations;
}
public MyResultResponse returnResponse(List<Node> nodes,List<Relationship> relationships) {
public MyResultResponse returnResponse(List<Node> nodes, List<Relationship> relationships) {
MyResultResponse response = new MyResultResponse();
List<MyResultResponse.Datas> datass = new ArrayList<>();
MyResultResponse.Datas datas = new MyResultResponse.Datas();
@ -276,7 +276,10 @@ public class DocInfoService extends EsService<DocInfo> {
graph.setRelationships(relationships);
data.setGraph(graph);
dataList.add(data);
datas.setColumns(new ArrayList<String>(){{add("user");add("entity");}});
datas.setColumns(new ArrayList<String>() {{
add("user");
add("entity");
}});
datas.setData(dataList);
datass.add(datas);
response.setResults(datass);
@ -292,6 +295,24 @@ public class DocInfoService extends EsService<DocInfo> {
return summary;
}
// 不用集合耗时：293270952500 不用集合耗时：245180467700
// }
//社会网络分析
public List<MyResultResponse> socialNetworkAnalysis(String[] ids) {
List<MyResultResponse> list = new ArrayList<>();
long stratTime = System.nanoTime();
if (ids != null && ids.length > 0) {
for (int i = 0; i < ids.length; i++) {
// list.add(analysis(ids[i], true));
list.add(analysis(ids[i], true));
}
}
long endTime = System.nanoTime();
System.out.println("不用集合耗时：" + (endTime - stratTime));
return list;
}
//文章管理--批量分析
public String batchAnalysis(String[] ids) {
if(ids!=null&&ids.length>0){
for(int i=0;i<ids.length;i++) {
@ -300,4 +321,86 @@ public class DocInfoService extends EsService<DocInfo> {
}
return "ok";
}
public MyResultResponse socialAnalysis(String content) {
// String grapData = docInfo.getParserGraphData();
MyResultResponse response = null;
// if(StringUtils.isNotBlank(grapData)&&reAnalysis){
// response = JSONObject.parseObject(grapData,MyResultResponse.class);
// if(response!=null){
// return response;
// }
// }
List<Node> nodes = new ArrayList<>();
List<Relationship> relationships = new ArrayList<>();
//拆句
if (StringUtils.isNotBlank(content)) {
content = MyObjects.delHTMLTag(content);
content = MyObjects.delSpace(content);
String regx = "\n|!|\\.|\\。|\\;|\\；|\\！|\\,|\\，|\\、|\\@|\\#|\\$|\\￥|\\%|\\&|\\*|\\(|\\)|\\=|\\+|\\-|\\_|\\>|\\<|\\[|\\]|\\【|\\】|\\？|\\?|”|\\\"";
String[] strArr = content.split(regx);
Set<String> allRelations = getAllRelations(strArr);
int i = 1;
// Map<String,String> nodesMap = getAllNodes(allRelations);
if (allRelations != null && allRelations.size() > 0) {
for (String relation : allRelations) {
String[] nrn = StringUtils.isNotBlank(relation) ? relation.split(",") : null;
if (nrn == null || nrn.length < 3) {
continue;
}
String nodeName1 = nrn[0];
String relationName = nrn[1];
String nodeName2 = nrn[2];
Node node1 = new Node();
//node1.setId(nodesMap.get(nodeName1));
node1.setId(nodeName1);
node1.setLabels(new ArrayList<String>() {{
add(nodeName1);
}});
node1.setProperties(new HashMap<String, Object>() {{
put("name", nodeName1);
put("email", "9094908@qq.com");
}});
Node node2 = new Node();
node2.setId(nodeName2);
node2.setLabels(new ArrayList<String>() {{
add(nodeName2);
}});
node2.setProperties(new HashMap<String, Object>() {{
put("name", nodeName2);
put("email", "9094908@qq.com");
}});
nodes.add(node1);
nodes.add(node2);
Relationship relationship = new Relationship();
relationship.setId(String.valueOf(i++));
relationship.setType(relationName);
relationship.setStartNode(nodeName1);
relationship.setEndNode(nodeName2);
relationship.setProperties(new HashMap<String, Object>() {{
put("source", nodeName1);
put("relation", relationName);
put("target", nodeName2);
}});
relationships.add(relationship);
}
}
// //获取命名实体
// try {
// Map<String,Set<String>> nerTagSet = DependencyParserUtils.getMyNERTagSet(strArr);
// docInfo.setParserNamedEntity(JSONObject.toJSONString(nerTagSet));
// docInfo.setSummary(SummaryUtils.autoSummary(content));
// } catch (Exception e) {
// e.printStackTrace();
// }
}
response = returnResponse(nodes, relationships);
// docInfo.setParserGraphData(JSONObject.toJSONString(response));
// docInfo.setStatus(1);
// insertOrUpdateDocInfo(docInfo);
return response;
}
}

21
ruoyi-biemo/src/main/java/com/ruoyi/biemo/mongodb/entity/SplitWordDrill.java
Unescape Escape View File

@ -0,0 +1,21 @@
package com.ruoyi.biemo.mongodb.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.data.mongodb.core.mapping.Field;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Document(collection ="splitWordDrill")
public class SplitWordDrill {
@Id
private final String id = "6466d460f07e000095000787";
@Field
private String [] splitResult;
@Field
private String updateTime;
}
Write Preview
Loading…
Cancel
Save