tianzeOrg
/
TextBigData
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

词性标注修改

Browse Source
master
yz 2 years ago
parent 51b74c0e96
commit 9b587e3502
6 changed files with 92 additions and 26 deletions
Show Stats Download Patch File Download Diff File

55
ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/controller/DocInfoController.java
Unescape Escape View File

@ -95,6 +95,17 @@ public class DocInfoController extends BaseController {
return AjaxResult.success(docInfoService.selectDocInfoById(id));
}
/**
* 批量标注
*/
@PostMapping(value = "/uploadPosTagIds")
public AjaxResult uploadPosTag(@RequestBody List<String> idList) throws IOException {
for (int i = 0; i < idList.size(); i++) {
getInfoDelHtml(idList.get(i));
}
return AjaxResult.success();
}
/**
* 获取文章管理详细信息 去掉内容标签
*/
@ -110,6 +121,7 @@ public class DocInfoController extends BaseController {
content = MyObjects.delHTMLTag(content);
content = MyObjects.delSpace(content);
JSONArray labels = new JSONArray();
result.put("docInfoId",docInfoId);
result.put("content", content);
result.put("connections", new JSONArray());
result.put("connectionCategories", MyObjects.connectionCategories);
@ -170,6 +182,12 @@ public class DocInfoController extends BaseController {
map.put("adjective",adjective);
map.put("adverb",adverb);
result.put("partData",map);
//点击词性标注后直接上传 并修改状态
Query query = new Query(Criteria.where("_id").is(docInfoId));
Update update = new Update();
update.set("posTagStatus", "已标注");
mongoTemplate.upsert(query, update, DocInfo.class);
return AjaxResult.success("查询成功",result);
}
@ -212,9 +230,9 @@ public class DocInfoController extends BaseController {
@PostMapping("/uploadPosTag")
public AjaxResult uploadPosTag(@RequestBody JSONObject jsonObject) throws IOException {
System.out.println(jsonObject);
String content = jsonObject.getString("content");
JSONArray labels = jsonObject.getJSONArray("labels");
String docInfoId = jsonObject.getString("docInfoId");
List<IWord> wordList = new ArrayList<>();
if(labels!=null&&labels.size()>0){
for(int i=0;i<labels.size();i++){
@ -223,7 +241,6 @@ public class DocInfoController extends BaseController {
Integer startIndex = labelJSONObject.getInteger("startIndex");
Integer endIndex = labelJSONObject.getInteger("endIndex");
Integer categoryId = labelJSONObject.getInteger("categoryId");
String value = content.substring(startIndex,endIndex);
String label = MyObjects.wordsMappingIndex.get(categoryId);
word.setLabel(label);
@ -233,7 +250,11 @@ public class DocInfoController extends BaseController {
Sentence sentence = new Sentence(wordList);
DependencyParserUtils.learn(sentence);
}
return AjaxResult.success();
Query query = new Query(Criteria.where("_id").is(docInfoId));
Update update = new Update();
update.set("posTagStatus", "已标注");
mongoTemplate.upsert(query, update, DocInfo.class);
return AjaxResult.success("标注上传成功");
}
/**
@ -309,8 +330,16 @@ public class DocInfoController extends BaseController {
// docInfoService.insertEs(docInfo);
// }
@PostMapping("/splitWordIds")
public AjaxResult splitWordIds(@RequestBody List<DocInfo> docInfoList){
for (int i = 0; i < docInfoList.size(); i++) {
splitWordDocInfo(docInfoList.get(i));
}
return AjaxResult.success("批量分词成功");
}
@PostMapping ("/splitWordStr")
public AjaxResult splitWord(@RequestBody String str){
public AjaxResult splitWordStr(@RequestBody String str){
Segment segment= HanLP.newSegment();
List<Term> termList = segment.seg(str);
List<String> wordList=new ArrayList<>();
@ -322,8 +351,8 @@ public class DocInfoController extends BaseController {
}
@PostMapping ("/splitWordDocInfo")
public AjaxResult splitWordNew(@RequestBody DocInfo docInfo){
String title = docInfo.getTitle();
public AjaxResult splitWordDocInfo(@RequestBody DocInfo docInfo){
String title = docInfo.getContent();
Segment segment= HanLP.newSegment();
List<Term> termList = segment.seg(title);
List<String> wordList=new ArrayList<>();
@ -333,6 +362,16 @@ public class DocInfoController extends BaseController {
wordList.add(word);
}
docInfo.setAnalyzeTitle(wordList);
List<String> analyzeTitleList = docInfo.getAnalyzeTitle();
for (int i = 0; i < analyzeTitleList.size(); i++) {
if(StringUtils.isNotBlank(analyzeTitleList.get(i))){
CustomDictionary.add(analyzeTitleList.get(i));
}
}
Query query = new Query(Criteria.where("_id").is(docInfo.getId()));
Update update = new Update();
update.set("splitWordStatus", "已分词");
mongoTemplate.upsert(query, update, DocInfo.class);
return AjaxResult.success("分词成功",docInfo);
}
@ -343,7 +382,7 @@ public class DocInfoController extends BaseController {
* @return
*/
@PostMapping("/uploadSplitWordStr")
public AjaxResult uploadSplitWordNew(@RequestBody String splitResult) {
public AjaxResult uploadSplitWordStr(@RequestBody String splitResult) {
try {
String[] split = splitResult.split("/");
for (int i = 0; i < split.length; i++) {
@ -358,7 +397,7 @@ public class DocInfoController extends BaseController {
}
@PostMapping("/uploadSplitWordDocInfo")
public AjaxResult uploadSplitWordNew(@RequestBody DocInfo docInfo) {
public AjaxResult uploadSplitWordDocInfo(@RequestBody DocInfo docInfo) {
try {
List<String> analyzeTitleList = docInfo.getAnalyzeTitle();
for (int i = 0; i < analyzeTitleList.size(); i++) {

10
ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/controller/ReportController.java
Unescape Escape View File

@ -1,5 +1,7 @@
package com.ruoyi.biemo.business.controller;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.ruoyi.biemo.business.domain.Report;
import com.ruoyi.biemo.business.service.ReportService;
import com.ruoyi.biemo.core.page.Page;
@ -126,12 +128,14 @@ public class ReportController
@PostMapping("/similar")
public AjaxResult similar(String text1,String text2){
return AjaxResult.success(WordVectorModelUtils.similarCount(text1,text2));
public AjaxResult similar(@RequestBody JSONObject text1, @RequestBody String text2){
String text11 = text1.get("text1").toString();
String text22 = text1.get("text2").toString();
return AjaxResult.success(WordVectorModelUtils.similarCount(text11,text22));
}
@PostMapping("/sentiment")
public AjaxResult sentiment(String text){
public AjaxResult sentiment(@RequestBody String text){
return AjaxResult.success("查询成功",SentimentAnalysisUtils.analysis(text));
}

3
ruoyi-biemo/src/main/java/com/ruoyi/biemo/business/domain/DocInfo.java
Unescape Escape View File

@ -66,4 +66,7 @@ public class DocInfo extends BiemoEntity {
//分词状态 已分词 和未分词
private String splitWordStatus;
//词性标注状态 已标注 未标注
private String posTagStatus;
}

7
ruoyi-biemo/src/main/java/com/ruoyi/biemo/nlp/DependencyParserUtils.java
Unescape Escape View File

@ -32,6 +32,10 @@ public class DependencyParserUtils {
return analyzer.analyze(text);
}
public static Sentence nerAnalysis(String text,PerceptronLexicalAnalyzer analyzer) throws IOException {
return analyzer.analyze(text);
}
//上传词性
public static Boolean learn(Sentence sentence) throws IOException {
PerceptronLexicalAnalyzer analyzer = new PerceptronLexicalAnalyzer(HanLP.Config.PerceptronCWSModelPath,
@ -57,8 +61,9 @@ public class DependencyParserUtils {
public static Map<String,Set<String>> getMyNERTagSet(String[] strArr) throws Exception{
Map<String,Set<String>> result = new HashMap<>();
if(strArr!=null&&strArr.length>0){
PerceptronLexicalAnalyzer analyzer = new PerceptronLexicalAnalyzer(HanLP.Config.PerceptronCWSModelPath, HanLP.Config.PerceptronPOSModelPath, HanLP.Config.PerceptronNERModelPath);
for(int i=0;i<strArr.length;i++){
Sentence sentence = nerAnalysis(strArr[i]);
Sentence sentence = nerAnalysis(strArr[i],analyzer);
sentence = sentence.translateLabels();
List<IWord> words = sentence.wordList;
words.forEach(word->{

19
ruoyi-biemo/src/main/java/com/ruoyi/biemo/utils/TestUtility.java
Unescape Escape View File

@ -28,7 +28,9 @@ public class TestUtility
public static void ensureFullData()
{
ensureData("data/model/crf", "http://nlp.hankcs.com/download.php?file=data", ".", false);
// ensureData("data/model/crf", "http://nlp.hankcs.com/download.php?file=data", ".", false);
// ensureData("D:\\tianze\\文本大数据\\analysis-hanlp\\data\\model\\crf", "http://nlp.hankcs.com/download.php?file=data", ".", false);
ensureData("/usr/local/textjar/analysis-hanlp/data/model/crf", "http://nlp.hankcs.com/download.php?file=data", ".", false);
}
/**
@ -39,6 +41,12 @@ public class TestUtility
* @return name的绝对路径
*/
public static String ensureData(String name, String url)
{
return null;
// return ensureData(name, url, null, true);
}
public static String ensureData2(String name, String url)
{
return ensureData(name, url, null, true);
}
@ -83,7 +91,14 @@ public class TestUtility
*/
public static String ensureTestData(String name, String url)
{
return ensureData(String.format("data/test/%s", name), url);
// return ensureData(String.format("data/test/%s", name), url);
return ensureData(String.format("/usr/local/textjar/analysis-hanlp/data/test/%s", name), url);
}
public static String ensureTest2Data(String name, String url)
{
// return ensureData(String.format("data/test/%s", name), url);
return ensureData2(String.format("/usr/local/textjar/analysis-hanlp/data/test/%s", name), url);
}
/**

24
ruoyi-system/src/main/resources/mapper/system/SysMenuMapper.xml
Unescape Escape View File

@ -29,7 +29,7 @@
<sql id="selectMenuVo">
select menu_id, menu_name, parent_id, order_num, path, component, `query`, is_frame, is_cache, menu_type, visible, status, ifnull(perms,'') as perms, icon, create_time
from sys_menu
from sys_menu_stu
</sql>
<select id="selectMenuList" parameterType="SysMenu" resultMap="SysMenuResult">
@ -50,13 +50,13 @@
<select id="selectMenuTreeAll" resultMap="SysMenuResult">
select distinct m.menu_id, m.parent_id, m.menu_name, m.path, m.component, m.`query`, m.visible, m.status, ifnull(m.perms,'') as perms, m.is_frame, m.is_cache, m.menu_type, m.icon, m.order_num, m.create_time
from sys_menu m where m.menu_type in ('M', 'C') and m.status = 0
from sys_menu_stu m where m.menu_type in ('M', 'C') and m.status = 0
order by m.parent_id, m.order_num
</select>
<select id="selectMenuListByUserId" parameterType="SysMenu" resultMap="SysMenuResult">
select distinct m.menu_id, m.parent_id, m.menu_name, m.path, m.component, m.`query`, m.visible, m.status, ifnull(m.perms,'') as perms, m.is_frame, m.is_cache, m.menu_type, m.icon, m.order_num, m.create_time
from sys_menu m
from sys_menu_stu m
left join sys_role_menu rm on m.menu_id = rm.menu_id
left join sys_user_role ur on rm.role_id = ur.role_id
left join sys_role ro on ur.role_id = ro.role_id
@ -75,7 +75,7 @@
<select id="selectMenuTreeByUserId" parameterType="Long" resultMap="SysMenuResult">
select distinct m.menu_id, m.parent_id, m.menu_name, m.path, m.component, m.`query`, m.visible, m.status, ifnull(m.perms,'') as perms, m.is_frame, m.is_cache, m.menu_type, m.icon, m.order_num, m.create_time
from sys_menu m
from sys_menu_stu m
left join sys_role_menu rm on m.menu_id = rm.menu_id
left join sys_user_role ur on rm.role_id = ur.role_id
left join sys_role ro on ur.role_id = ro.role_id
@ -86,25 +86,25 @@
<select id="selectMenuListByRoleId" resultType="Long">
select m.menu_id
from sys_menu m
from sys_menu_stu m
left join sys_role_menu rm on m.menu_id = rm.menu_id
where rm.role_id = #{roleId}
<if test="menuCheckStrictly">
and m.menu_id not in (select m.parent_id from sys_menu m inner join sys_role_menu rm on m.menu_id = rm.menu_id and rm.role_id = #{roleId})
and m.menu_id not in (select m.parent_id from sys_menu_stu m inner join sys_role_menu rm on m.menu_id = rm.menu_id and rm.role_id = #{roleId})
</if>
order by m.parent_id, m.order_num
</select>
<select id="selectMenuPerms" resultType="String">
select distinct m.perms
from sys_menu m
from sys_menu_stu m
left join sys_role_menu rm on m.menu_id = rm.menu_id
left join sys_user_role ur on rm.role_id = ur.role_id
</select>
<select id="selectMenuPermsByUserId" parameterType="Long" resultType="String">
select distinct m.perms
from sys_menu m
from sys_menu_stu m
left join sys_role_menu rm on m.menu_id = rm.menu_id
left join sys_user_role ur on rm.role_id = ur.role_id
left join sys_role r on r.role_id = ur.role_id
@ -117,7 +117,7 @@
</select>
<select id="hasChildByMenuId" resultType="Integer">
select count(1) from sys_menu where parent_id = #{menuId}
select count(1) from sys_menu_stu where parent_id = #{menuId}
</select>
<select id="checkMenuNameUnique" parameterType="SysMenu" resultMap="SysMenuResult">
@ -126,7 +126,7 @@
</select>
<update id="updateMenu" parameterType="SysMenu">
update sys_menu
update sys_menu_stu
<set>
<if test="menuName != null and menuName != ''">menu_name = #{menuName},</if>
<if test="parentId != null">parent_id = #{parentId},</if>
@ -149,7 +149,7 @@
</update>
<insert id="insertMenu" parameterType="SysMenu">
insert into sys_menu(
insert into sys_menu_stu(
<if test="menuId != null and menuId != 0">menu_id,</if>
<if test="parentId != null and parentId != 0">parent_id,</if>
<if test="menuName != null and menuName != ''">menu_name,</if>
@ -189,7 +189,7 @@
</insert>
<delete id="deleteMenuById" parameterType="Long">
delete from sys_menu where menu_id = #{menuId}
delete from sys_menu_stu where menu_id = #{menuId}
</delete>
</mapper>
Write Preview
Loading…
Cancel
Save