diff --git a/pom.xml b/pom.xml index 13a6a9e..9d0f048 100644 --- a/pom.xml +++ b/pom.xml @@ -244,14 +244,19 @@ - - - - - + + com.hankcs + hanlp + portable-1.8.4 + + + com.squareup.okhttp3 + okhttp + 4.9.3 + diff --git a/src/main/java/com/sztzjy/linkCommerce/controller/stu/IndustryDemandAnalysisController.java b/src/main/java/com/sztzjy/linkCommerce/controller/stu/IndustryDemandAnalysisController.java new file mode 100644 index 0000000..2213df2 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/controller/stu/IndustryDemandAnalysisController.java @@ -0,0 +1,175 @@ +package com.sztzjy.linkCommerce.controller.stu; + +import cn.hutool.core.util.IdUtil; +import com.alibaba.excel.EasyExcel; +import com.alibaba.fastjson.JSONObject; +import com.nimbusds.jose.shaded.gson.Gson; +import com.sztzjy.linkCommerce.annotation.AnonymousAccess; +import com.sztzjy.linkCommerce.entity.*; +import com.sztzjy.linkCommerce.entity.dto.MonthlyCakeIndex; +import com.sztzjy.linkCommerce.entity.dto.StuIndustryAnalysisDTO; +import com.sztzjy.linkCommerce.entity.dto.YearCakeIndexDto; +import com.sztzjy.linkCommerce.mapper.HyxqfxAnswerMapper; +import com.sztzjy.linkCommerce.mapper.HyxqfxDataMapper; +import com.sztzjy.linkCommerce.mapper.HyxqfxStuAnswerMapper; +import com.sztzjy.linkCommerce.service.IndustryDemandAnalysisService; +import com.sztzjy.linkCommerce.util.ResultEntity; +import com.sztzjy.linkCommerce.util.excel.DataListener; +import com.sztzjy.linkCommerce.util.excel.DemoDataListener; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.checkerframework.checker.units.qual.A; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.util.*; + +/** + * @author 17803 + * @date 2025-04-24 13:30 + */ +@Api(tags = "行业需求分析") +@RestController +@RequestMapping("api/industryAnalysis") +public class IndustryDemandAnalysisController { + + + @Autowired + private IndustryDemandAnalysisService service; + + @Autowired + private HyxqfxDataMapper hyxqfxDataMapper; + + @Autowired + private HyxqfxStuAnswerMapper hyxqfxStuAnswerMapper; + + + + + //数据查询 + @ApiOperation("行业需求分析提交") + @PostMapping("/analysisSubmit") + @AnonymousAccess + @Transactional(rollbackFor = Exception.class) + public ResultEntity uploadExcel(@RequestBody StuIndustryAnalysisDTO analysisDTO) { + + //查询数据 + HyxqfxStuAnswerExample example = new HyxqfxStuAnswerExample(); + example.createCriteria().andUserIdEqualTo(analysisDTO.getUserId()); + + List hyxqfxStuAnswerList = hyxqfxStuAnswerMapper.selectByExample(example); + if (!CollectionUtils.isEmpty(hyxqfxStuAnswerList)) { + return new ResultEntity<>(HttpStatus.OK); + } + + //读取文件内容 批量导入excel中 以文件名区分\ + + Map yearMap = new HashMap<>(); + yearMap.put("2018年",0.28); + yearMap.put("2019年",0.25); + yearMap.put("2020年",0.24); + yearMap.put("2021年",0.22); + yearMap.put("2022年",0.21); + yearMap.put("2023年",0.22); + yearMap.put("2024年",0.20); + + Map monthMap = new HashMap<>(); + monthMap.put("2024年4月",0.21); + monthMap.put("2024年5月",0.20); + monthMap.put("2024年6月",0.20); + monthMap.put("2024年7月",0.22); + monthMap.put("2024年8月",0.33); + monthMap.put("2024年9月",0.30); + monthMap.put("2024年10月",0.33); + + monthMap.put("2024年11月",0.23); + monthMap.put("2024年12月",0.22); + monthMap.put("2025年1月",0.22); + monthMap.put("2025年2月",0.15); + monthMap.put("2025年3月",0.27); + + int count = 0; + + List yearCakeIndexDtoList = analysisDTO.getYearCakeIndexDtoList(); + + for (YearCakeIndexDto yearCakeIndexDto : yearCakeIndexDtoList) { + Double v = yearMap.get(yearCakeIndexDto.getYear()); + if (yearCakeIndexDto.getYearAnswer() == null || v != yearCakeIndexDto.getYearAnswer()) { + count ++; + } + } + + //分别将两个字段转为json格式 + List monthlyCakeIndexList = analysisDTO.getMonthlyCakeIndexList(); + + for (MonthlyCakeIndex yearCakeIndexDto : monthlyCakeIndexList) { + Double v = monthMap.get(yearCakeIndexDto.getYear()); + if (yearCakeIndexDto.getMonthAnswer() == null || v != yearCakeIndexDto.getMonthAnswer()) { + count ++; + } + } + + + String jsonString = JSONObject.toJSONString(analysisDTO); + HyxqfxStuAnswer hyxqfxStuAnswer = new HyxqfxStuAnswer(); + hyxqfxStuAnswer.setId(IdUtil.randomUUID()); + hyxqfxStuAnswer.setUserId(analysisDTO.getUserId()); + hyxqfxStuAnswer.setAnswer(jsonString); + hyxqfxStuAnswer.setErrorNumber(count); + + hyxqfxStuAnswerMapper.insertSelective(hyxqfxStuAnswer); + + //插入主观题 + SubjectiveQuestions subjectiveQuestions = new SubjectiveQuestions(); + subjectiveQuestions.setId(IdUtil.randomUUID()); + subjectiveQuestions.setUserId(analysisDTO.getUserId()); + subjectiveQuestions.setModule("行业需求分析"); + subjectiveQuestions.setProject("市场需求挖掘"); + subjectiveQuestions.setCreateTime(new Date()); + subjectiveQuestions.setSubjectiveContent(analysisDTO.getContext()); + subjectiveQuestions.setSubjectiveContent(analysisDTO.getContext()); + + return new ResultEntity<>(HttpStatus.OK, "上传成功"); + + } + + @ApiOperation("行业需求分析根据用户ID查询") + @GetMapping("/selectByUserId") + @AnonymousAccess + public ResultEntity selectByUserId(@ApiParam("用户ID") String userId) { + //读取文件内容 批量导入excel中 以文件名区分\ + + HyxqfxStuAnswerExample hyxqfxStuAnswer = new HyxqfxStuAnswerExample(); + hyxqfxStuAnswer.createCriteria().andUserIdEqualTo(userId); + + List hyxqfxStuAnswersList = hyxqfxStuAnswerMapper.selectByExampleWithBLOBs(hyxqfxStuAnswer); + if (CollectionUtils.isEmpty(hyxqfxStuAnswersList)) { + return new ResultEntity<>(HttpStatus.OK); + }else { + JSONObject jsonObject = JSONObject.parseObject(hyxqfxStuAnswersList.get(0).getAnswer()); + return new ResultEntity<>(HttpStatus.OK,jsonObject); + } + + } + + + public void jiaoYanCodeSize(String code){ + String[] strings = code.split(","); + if (strings.length>=20) + { + //计分规则 + + + } + + } + + + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/controller/stu/StuDigitalMarketingModelController.java b/src/main/java/com/sztzjy/linkCommerce/controller/stu/StuDigitalMarketingModelController.java new file mode 100644 index 0000000..3b871e3 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/controller/stu/StuDigitalMarketingModelController.java @@ -0,0 +1,317 @@ +package com.sztzjy.linkCommerce.controller.stu; + +import com.sztzjy.linkCommerce.annotation.AnonymousAccess; +import com.sztzjy.linkCommerce.entity.*; +import com.sztzjy.linkCommerce.entity.dto.*; +import com.sztzjy.linkCommerce.mapper.StuUserCommentMapper; +import com.sztzjy.linkCommerce.service.StuDigitalMarketingModelService; +import com.sztzjy.linkCommerce.util.BigDecimalUtils; +import com.sztzjy.linkCommerce.util.ResultEntity; +import com.sztzjy.linkCommerce.util.algorithm.Apriori; +import com.sztzjy.linkCommerce.util.algorithm.KMeans; +import com.sztzjy.linkCommerce.util.algorithm.LogisticRegression; +import com.sztzjy.linkCommerce.util.excel.ImportExcelUtil; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.*; + +import static com.sztzjy.linkCommerce.util.algorithm.KMeans.*; +import static com.sztzjy.linkCommerce.util.algorithm.KMeansResult.*; + +/** + * @author tz + * @date 2024/6/13 10:43 + */ +@Api(tags = "数字营销模型") +@RequestMapping("api/model") +@RestController +public class StuDigitalMarketingModelController { + + @Value("${file.path}") + private String filePath; + + @Resource + StuDigitalMarketingModelService modelService; + @Resource + StuUserCommentMapper stuUserCommentMapper; + + @ApiOperation("选择指标--下拉框") + @PostMapping("/dropdownBox") + @AnonymousAccess + public ResultEntity dropdownBox(String userId) { + List list=modelService.dropdownBox(userId); + return new ResultEntity(HttpStatus.OK,"成功",list); + } + + @ApiOperation("选择指标--查看指标") + @PostMapping("/viewMetrics") + @AnonymousAccess + public ResultEntity viewMetrics(@ApiParam("用户ID") String userId, + @ApiParam("表格名") String tableName, + @ApiParam("算法名称") String algorithmName) { + List list=modelService.viewMetrics(userId,tableName,algorithmName); + return new ResultEntity(HttpStatus.OK,"成功",list); + } + + + @ApiOperation("选择指标--分析数据展示") + @PostMapping("/viewAnalyzeData") + @AnonymousAccess + public ResultEntity viewAnalyzeData(@RequestBody AnalyzeDataDTO analyzeDataDTO) { + + return modelService.viewAnalyzeData(analyzeDataDTO); + } + + @ApiOperation("数据预处理") + @PostMapping("/dataPreprocessing") + @AnonymousAccess + public ResultEntity dataPreprocessing(@RequestBody DataPreprocessingDTO dto) { + + String userId = dto.getUserId(); + String method = dto.getMethod(); + List> mapList = dto.getMapList(); + + return modelService.dataPreprocessing(userId,method,mapList); + } + + + @ApiOperation("描述性统计") + @PostMapping("/descriptiveStatistics") + @AnonymousAccess + public ResultEntity descriptiveStatistics(@ApiParam("数据集、需要计算的方式、用户ID")@RequestBody StatisticsDTO statisticsDTO) { + + Map> map = statisticsDTO.getMap(); + List statistic = statisticsDTO.getStatistic(); + String userId = statisticsDTO.getUserId(); + + return modelService.descriptiveStatistics(map,statistic,userId); + } + + + @ApiOperation("聚类分析--散点图") + @PostMapping("/clusterScatterPlot") + @AnonymousAccess + public ResultEntity clusterScatterPlot(@RequestBody ClusterScatterPlotDTO clusterScatterPlotDTO) { + + Integer t = clusterScatterPlotDTO.getT(); + Integer k = clusterScatterPlotDTO.getK(); + List> deduplicatedDataList = clusterScatterPlotDTO.getDeduplicatedDataList(); + + List strings = convertToDelimitedStringList(deduplicatedDataList); + + List irisData = readIrisData(strings); + + //获取数据集 + KMeans kMeans = new KMeans(k, t, irisData); + List> clusters = kMeans.cluster(); + + //封装成map + //数据集 + Map> dataSetMap=new HashMap<>(); + for (int i = 0; i < clusters.size(); i++) { + dataSetMap.put(i,clusters.get(i)); + } + + //质心 + Map> centroidMap=new HashMap<>(); + List centroids = kMeans.getCentroids(); + for (int i = 0; i < centroids.size(); i++) { + centroidMap.put(i, Collections.singletonList(centroids.get(i))); + } + + ClusterAnalysisDTO clusterAnalysisDTO=new ClusterAnalysisDTO(); + + clusterAnalysisDTO.setDataSet(dataSetMap); + clusterAnalysisDTO.setCentroid(centroidMap); + + return new ResultEntity(HttpStatus.OK,clusterAnalysisDTO); + } + + + @ApiOperation("聚类分析--分析结果") + @PostMapping("/clusterAnalysisResult") + @AnonymousAccess + public ResultEntity clusterAnalysisResult(@RequestBody ClusterScatterPlotDTO clusterScatterPlotDTO) { + + + Integer t = clusterScatterPlotDTO.getT(); + Integer k = clusterScatterPlotDTO.getK(); + List> deduplicatedDataList = clusterScatterPlotDTO.getDeduplicatedDataList(); + + if(k>deduplicatedDataList.size()){ + return new ResultEntity(HttpStatus.ACCEPTED,"簇数K值不能大于数据条数"); + } + + List strings = convertToDelimitedStringList(deduplicatedDataList); + + //初始化数据 + ArrayList> arrayLists = readTable(strings); + + //随机选择 k 个数据点作为初始聚类中心 + ArrayList> centerList = randomList(k, arrayLists); + + //开始迭代 + KmeansDTO kmeansDTO = kmeans(k, t, centerList); + + return new ResultEntity(HttpStatus.OK,kmeansDTO); + } + + + @ApiOperation("关联规则挖掘") + @PostMapping("/apriori") + @AnonymousAccess + public ResultEntity apriori(@RequestBody AprioriDTO aprioriDTO) throws IOException { + + double support = aprioriDTO.getSupport(); + double confidence = aprioriDTO.getConfidence(); + String userId = aprioriDTO.getUserId(); + List> deduplicatedDataList = aprioriDTO.getDeduplicatedDataList(); + + List strings = convertToDelimitedStringList(deduplicatedDataList); + //初始化事务数据库、项目集、候选集再进行剪枝 + Apriori.init(strings,confidence); + + //迭代求出最终的候选频繁集 + Apriori.iteration(Apriori.C,Apriori.L,confidence); + + //根据最终的关联集，根据公式计算出各个关联事件 + List connection = Apriori.connection(support); + + return new ResultEntity(HttpStatus.OK,"成功",connection); + } + +// @ApiOperation("回归分析--线性回归") +// @PostMapping("/regressionAnalysis") +// @AnonymousAccess +// public ResultEntity regressionAnalysis(@RequestBody RegressionAnalysisDTO analysisDTO) { +// double[][] x = analysisDTO.getX(); +// double[] y = analysisDTO.getY(); +// +// // 创建线性回归模型 +// LinearRegression regression=new LinearRegression(); +// regression.fit(x,y); +// +// +// RAnalysisDTO rAnalysisDTO=new RAnalysisDTO(); +// rAnalysisDTO.setIntercept(regression.getIntercept()); +// rAnalysisDTO.setSlopes(regression.getSlopes()); +// rAnalysisDTO.setType("线性回归"); +// +// return new ResultEntity(HttpStatus.OK,"成功",rAnalysisDTO); +// } + + + @ApiOperation("回归分析--逻辑回归") + @PostMapping("/logistic") + @AnonymousAccess + public ResultEntity logistic(@RequestBody LogisticDTO logisticDTO) { + double[] x = logisticDTO.getX(); + double[] y = logisticDTO.getY(); + + LogisticRegression lr = new LogisticRegression(x, y); + double[] coefficients = lr.fit(); + + DecimalFormat df = new DecimalFormat("#.00"); + RAnalysisDTO rAnalysisDTO=new RAnalysisDTO(); + + rAnalysisDTO.setIntercept(Double.parseDouble(df.format(coefficients[0]))); + rAnalysisDTO.setSlope(Double.parseDouble(df.format(coefficients[1]))); + + return new ResultEntity(HttpStatus.OK,"成功",rAnalysisDTO); + } + + @ApiOperation("回归分析--走势图") + @PostMapping("/trendChart") + @AnonymousAccess + public ResultEntity trendChart(@RequestBody LogisticDTO logisticDTO) { + double[] x = logisticDTO.getX(); + double[] y = logisticDTO.getY(); + + LogisticRegression lr = new LogisticRegression(x, y); + double[][] coefficients = lr.getCoordinatesOnLine(); + + return new ResultEntity(HttpStatus.OK,"成功",coefficients); + } + + + @ApiOperation("线性回归/逻辑回归--预测") + @PostMapping("/prediction") + @AnonymousAccess + public ResultEntity prediction(@RequestBody LogisticPredictDTO logisticPredictDTO) { + + DecimalFormat df = new DecimalFormat("#.0"); + BigDecimalUtils bigDecimalUtils=new BigDecimalUtils(); + double predict = bigDecimalUtils.add(bigDecimalUtils.mul(logisticPredictDTO.getSlope(),logisticPredictDTO.getX()),logisticPredictDTO.getIntercept()); + return new ResultEntity(HttpStatus.OK,"成功",df.format(predict)); + + } + + @ApiOperation("情感分析/文本挖掘") + @PostMapping("/emotionalAnalysis") + @AnonymousAccess + public ResultEntity emotionalAnalysis(@RequestBody EmotionalAnalysisDto emotionalAnalysisDto) throws IOException { + String id = emotionalAnalysisDto.getId(); + String modelType = emotionalAnalysisDto.getModelType(); + String content = emotionalAnalysisDto.getContent(); + String userId = emotionalAnalysisDto.getUserId(); + return modelService.emotionalAnalysis(userId,modelType,content,id); + } + + @ApiOperation("批量导入") + @PostMapping("/batchImport") + @AnonymousAccess + public ResultEntity batchImport(@RequestParam(required = false) @RequestPart MultipartFile file) { + + //验证文件类型 + if (!file.getOriginalFilename().substring(file.getOriginalFilename().lastIndexOf(".")).equals(".xls") && !file.getOriginalFilename().substring(file.getOriginalFilename().lastIndexOf(".")).equals(".xlsx")) { + return new ResultEntity<>(HttpStatus.ACCEPTED, "文件类型有误！请上传Excel文件"); + } + + try { + List stuUserComments=new ArrayList<>(); + + //获取数据 + List> listByExcel = ImportExcelUtil.getListByExcel(file.getInputStream(), file.getOriginalFilename()); + //封装数据 + for (int i = 0; i < listByExcel.size(); i++) { + List list = listByExcel.get(i); + if (list.get(0) == "" || ("序号").equals(list.get(0))) { + continue; + } + + StuUserComment stuUserComment=new StuUserComment(); + Integer uuid = UUID.randomUUID().toString().replaceAll("-", "").hashCode(); + uuid = uuid < 0 ? -uuid : uuid;//String.hashCode() 值会为空 + stuUserComment.setId(uuid); + stuUserComment.setContent(String.valueOf(list.get(0))); + stuUserComment.setCreationTime(new Date(String.valueOf(list.get(1)))); + stuUserComment.setNickName(String.valueOf((list.get(2)))); + stuUserComment.setReferenceName(String.valueOf(list.get(3))); + stuUserComment.setContentType(String.valueOf(list.get(4))); + + stuUserComments.add(stuUserComment); + } + stuUserCommentMapper.addList(stuUserComments); + + } catch (Exception e) { + e.printStackTrace(); + }finally { + try { + file.getInputStream().close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return new ResultEntity(HttpStatus.OK,"成功"); + } + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswer.java b/src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswer.java index 3e3923c..3b31be4 100644 --- a/src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswer.java +++ b/src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswer.java @@ -20,6 +20,9 @@ public class HyxqfxStuAnswer { @ApiModelProperty(notes = "用户ID（关联userinfo表）") private String userId; + @ApiModelProperty(notes = "错误次数") + private Integer errorNumber; + @ApiModelProperty(notes = "学生答案内容（富文本格式）") private String answer; @@ -55,6 +58,14 @@ public class HyxqfxStuAnswer { this.userId = userId == null ? null : userId.trim(); } + public Integer getErrorNumber() { + return errorNumber; + } + + public void setErrorNumber(Integer errorNumber) { + this.errorNumber = errorNumber; + } + public String getAnswer() { return answer; } diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswerExample.java b/src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswerExample.java index d570d1b..9554f11 100644 --- a/src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswerExample.java +++ b/src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswerExample.java @@ -383,6 +383,66 @@ public class HyxqfxStuAnswerExample { addCriterion("user_id not between", value1, value2, "userId"); return (Criteria) this; } + + public Criteria andErrorNumberIsNull() { + addCriterion("error_number is null"); + return (Criteria) this; + } + + public Criteria andErrorNumberIsNotNull() { + addCriterion("error_number is not null"); + return (Criteria) this; + } + + public Criteria andErrorNumberEqualTo(Integer value) { + addCriterion("error_number =", value, "errorNumber"); + return (Criteria) this; + } + + public Criteria andErrorNumberNotEqualTo(Integer value) { + addCriterion("error_number <>", value, "errorNumber"); + return (Criteria) this; + } + + public Criteria andErrorNumberGreaterThan(Integer value) { + addCriterion("error_number >", value, "errorNumber"); + return (Criteria) this; + } + + public Criteria andErrorNumberGreaterThanOrEqualTo(Integer value) { + addCriterion("error_number >=", value, "errorNumber"); + return (Criteria) this; + } + + public Criteria andErrorNumberLessThan(Integer value) { + addCriterion("error_number <", value, "errorNumber"); + return (Criteria) this; + } + + public Criteria andErrorNumberLessThanOrEqualTo(Integer value) { + addCriterion("error_number <=", value, "errorNumber"); + return (Criteria) this; + } + + public Criteria andErrorNumberIn(List values) { + addCriterion("error_number in", values, "errorNumber"); + return (Criteria) this; + } + + public Criteria andErrorNumberNotIn(List values) { + addCriterion("error_number not in", values, "errorNumber"); + return (Criteria) this; + } + + public Criteria andErrorNumberBetween(Integer value1, Integer value2) { + addCriterion("error_number between", value1, value2, "errorNumber"); + return (Criteria) this; + } + + public Criteria andErrorNumberNotBetween(Integer value1, Integer value2) { + addCriterion("error_number not between", value1, value2, "errorNumber"); + return (Criteria) this; + } } public static class Criteria extends GeneratedCriteria { diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/AnalyzeDataDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/AnalyzeDataDTO.java new file mode 100644 index 0000000..ef641d7 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/AnalyzeDataDTO.java @@ -0,0 +1,16 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import lombok.Data; + +import java.util.List; + +/** + * @author tz + * @date 2024/8/16 15:26 + */ +@Data +public class AnalyzeDataDTO { + private String userId; + private String tableName; + private List fieldList; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/AprioriDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/AprioriDTO.java new file mode 100644 index 0000000..824ea11 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/AprioriDTO.java @@ -0,0 +1,23 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * @author tz + * @date 2024/8/17 14:16 + */ +@Data +public class AprioriDTO { + @ApiModelProperty("最小支持度阀值") + private double support; + @ApiModelProperty("最小置信度") + private double confidence; + @ApiModelProperty("用户ID") + private String userId; + @ApiModelProperty("数据集") + private List> deduplicatedDataList; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/AssociationRulesDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/AssociationRulesDTO.java new file mode 100644 index 0000000..0e5cebe --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/AssociationRulesDTO.java @@ -0,0 +1,25 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +/** + * @author tz + * @date 2024/6/20 8:51 + */ +@Data +public class AssociationRulesDTO { + @ApiModelProperty("关联") + private List correlation; + + @ApiModelProperty("被关联") + private List associated; + + @ApiModelProperty("置信度") + private double confidenceLevel; + + @ApiModelProperty("规则") + private String rule; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/ClusterAnalysisDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/ClusterAnalysisDTO.java new file mode 100644 index 0000000..d4b5cdd --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/ClusterAnalysisDTO.java @@ -0,0 +1,18 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import com.sztzjy.linkCommerce.util.algorithm.KMeans; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * @author tz + * @date 2024/6/14 10:12 + */ +@Data +public class ClusterAnalysisDTO { + private Map> dataSet; + + private Map> centroid; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/ClusterScatterPlotDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/ClusterScatterPlotDTO.java new file mode 100644 index 0000000..5012bba --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/ClusterScatterPlotDTO.java @@ -0,0 +1,23 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * @author tz + * @date 2024/8/16 16:26 + */ +@Data +public class ClusterScatterPlotDTO { + @ApiModelProperty("簇数") + private Integer k; + @ApiModelProperty("最大迭代次数") + private Integer t; + @ApiModelProperty("用户ID") + private String userId; + @ApiModelProperty("数据集") + private List> deduplicatedDataList; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/DSExportDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/DSExportDTO.java new file mode 100644 index 0000000..2ed9b1d --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/DSExportDTO.java @@ -0,0 +1,18 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +/** + * @author tz + * @date 2024/8/19 10:33 + */ +@Data +@Api(tags = "描述性统计下载表格数据") +public class DSExportDTO extends DescriptiveStatistics{ + + @ApiModelProperty("字段名") + private String field; + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/DataPreprocessingDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/DataPreprocessingDTO.java new file mode 100644 index 0000000..022c6dd --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/DataPreprocessingDTO.java @@ -0,0 +1,19 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * @author tz + * @date 2024/6/25 14:21 + */ +@Data +public class DataPreprocessingDTO { + private String userId; + + private String method; + + private List> mapList; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/DescriptiveStatistics.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/DescriptiveStatistics.java new file mode 100644 index 0000000..c2a163c --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/DescriptiveStatistics.java @@ -0,0 +1,50 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +/** + * @author tz + * @date 2024/6/21 9:51 + */ +@Data +public class DescriptiveStatistics { + @ApiModelProperty("平均数") + private double average; + + @ApiModelProperty("中位数") + private double median; + + @ApiModelProperty("众数") + private double mode; + + @ApiModelProperty("标准差") + private double standardDeviation; + + @ApiModelProperty("方差") + private double variance; + + @ApiModelProperty("标准误差") + private double standardError; + + @ApiModelProperty("峰度") + private double kurtosis; + + @ApiModelProperty("偏度") + private double skewness; + + @ApiModelProperty("最大值") + private double max; + + @ApiModelProperty("最小值") + private double min; + + @ApiModelProperty("求和") + private double summation; + + @ApiModelProperty("观测数") + private Integer observations; + + @ApiModelProperty("字段名") + private String field; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/DescriptiveStatisticsDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/DescriptiveStatisticsDTO.java new file mode 100644 index 0000000..5c82055 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/DescriptiveStatisticsDTO.java @@ -0,0 +1,15 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import lombok.Data; + +/** + * @author tz + * @date 2024/6/21 10:00 + */ +@Data +public class DescriptiveStatisticsDTO { + private String field; + + private DescriptiveStatistics statistics; + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/EmotionalAnalysisDto.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/EmotionalAnalysisDto.java new file mode 100644 index 0000000..04751c0 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/EmotionalAnalysisDto.java @@ -0,0 +1,19 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import lombok.Data; + +import javax.xml.bind.annotation.XmlElement; + +/** + * @author tz + * @date 2024/8/19 16:09 + */ +@Data +public class EmotionalAnalysisDto { + + private String userId; + private String modelType; + private String content; + @XmlElement(name = "optionalField", required = false) + private String id; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/KmeansDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/KmeansDTO.java new file mode 100644 index 0000000..f491098 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/KmeansDTO.java @@ -0,0 +1,17 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import lombok.Data; + +import java.util.ArrayList; + +/** + * @author tz + * @date 2024/8/28 9:09 + */ +@Data +public class KmeansDTO { + + private ArrayList>> clusters; + + private Integer iterations; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/LogisticDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/LogisticDTO.java new file mode 100644 index 0000000..a25e09a --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/LogisticDTO.java @@ -0,0 +1,21 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +/** + * @author tz + * @date 2024/7/23 16:52 + */ +@Data +public class LogisticDTO { + + @ApiModelProperty("自变量x") + private double[] x; + + @ApiModelProperty("因变量y") + private double[] y; + + @ApiModelProperty("用户ID") + private String userId; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/LogisticPredictDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/LogisticPredictDTO.java new file mode 100644 index 0000000..18bf844 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/LogisticPredictDTO.java @@ -0,0 +1,16 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author tz + * @date 2024/8/19 9:38 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class LogisticPredictDTO extends RAnalysisDTO { + + private double x; + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/MonthlyCakeIndex.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/MonthlyCakeIndex.java new file mode 100644 index 0000000..69b98a6 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/MonthlyCakeIndex.java @@ -0,0 +1,19 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +/** + * @author 17803 + * @date 2025-04-24 14:22 + */ +@Data +public class MonthlyCakeIndex { + + + @ApiModelProperty(notes = "月份") + private String year; + + @ApiModelProperty(notes = "月度蛋糕指数") + private Double monthAnswer; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/NewDescriptiveStatisticsDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/NewDescriptiveStatisticsDTO.java new file mode 100644 index 0000000..62735c3 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/NewDescriptiveStatisticsDTO.java @@ -0,0 +1,48 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +/** + * @author tz + * @date 2024/11/26 13:42 + */ +//新版描述性统计分析返回数据格式 +@Data +public class NewDescriptiveStatisticsDTO { + @ApiModelProperty("平均数") + private double average; + + @ApiModelProperty("中位数") + private double median; + + @ApiModelProperty("众数") + private double mode; + + @ApiModelProperty("标准差") + private double standardDeviation; + + @ApiModelProperty("方差") + private double variance; + + @ApiModelProperty("标准误差") + private double standardError; + + @ApiModelProperty("峰度") + private double kurtosis; + + @ApiModelProperty("偏度") + private double skewness; + + @ApiModelProperty("最大值") + private double max; + + @ApiModelProperty("最小值") + private double min; + + @ApiModelProperty("求和") + private double summation; + + @ApiModelProperty("观测数") + private Integer observations; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/RAnalysisDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/RAnalysisDTO.java new file mode 100644 index 0000000..cff9309 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/RAnalysisDTO.java @@ -0,0 +1,16 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +/** + * @author tz + * @date 2024/8/17 16:30 + */ +@Data +public class RAnalysisDTO { + @ApiModelProperty("回归系数值") + private double intercept; + @ApiModelProperty("逻辑回归常数项值") + private double slope; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/RAnalysisExportDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/RAnalysisExportDTO.java new file mode 100644 index 0000000..94dec67 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/RAnalysisExportDTO.java @@ -0,0 +1,16 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author tz + * @date 2024/8/19 14:55 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class RAnalysisExportDTO extends RAnalysisDTO{ + @ApiModelProperty("预测值") + private double prediction; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/RegressionAnalysisDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/RegressionAnalysisDTO.java new file mode 100644 index 0000000..955ed84 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/RegressionAnalysisDTO.java @@ -0,0 +1,20 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +/** + * @author tz + * @date 2024/7/19 13:56 + */ +@Data +public class RegressionAnalysisDTO { + @ApiModelProperty("自变量x") + private double[][] x; + + @ApiModelProperty("因变量y") + private double[] y; + + @ApiModelProperty("用户ID") + private String userId; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/StatisticsDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/StatisticsDTO.java new file mode 100644 index 0000000..f25357a --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/StatisticsDTO.java @@ -0,0 +1,17 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * @author tz + * @date 2024/6/21 10:57 + */ +@Data +public class StatisticsDTO { + private Map> map; + private List statistic; + private String userId; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/StuIndustryAnalysisDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/StuIndustryAnalysisDTO.java new file mode 100644 index 0000000..c7f408a --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/StuIndustryAnalysisDTO.java @@ -0,0 +1,30 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +/** + * @author 17803 + * @date 2025-04-24 14:12 + */ +// + @Data + public class StuIndustryAnalysisDTO { + + @ApiModelProperty(notes = "年度蛋糕指数") + private List yearCakeIndexDtoList; + + @ApiModelProperty(notes = "月度蛋糕指数") + private List monthlyCakeIndexList; + + + @ApiModelProperty(notes = "用户ID") + private String userId; + + @ApiModelProperty(notes = "主观题") + private String context; + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/WordFrequencyDTO.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/WordFrequencyDTO.java new file mode 100644 index 0000000..2ed160f --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/WordFrequencyDTO.java @@ -0,0 +1,14 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import lombok.Data; + +/** + * @author tz + * @date 2024/6/18 16:48 + */ +@Data +public class WordFrequencyDTO { + private String keyword; + + private Integer frequency; +} diff --git a/src/main/java/com/sztzjy/linkCommerce/entity/dto/YearCakeIndexDto.java b/src/main/java/com/sztzjy/linkCommerce/entity/dto/YearCakeIndexDto.java new file mode 100644 index 0000000..ba99f11 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/entity/dto/YearCakeIndexDto.java @@ -0,0 +1,21 @@ +package com.sztzjy.linkCommerce.entity.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +/** + * @author 17803 + * @date 2025-04-24 14:22 + */ +@Data +public class YearCakeIndexDto { + + + + @ApiModelProperty(notes = "月份") + private String year; + + @ApiModelProperty(notes = "童装童鞋类目年度蛋糕指数") + private Double yearAnswer; + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/service/IndustryDemandAnalysisService.java b/src/main/java/com/sztzjy/linkCommerce/service/IndustryDemandAnalysisService.java new file mode 100644 index 0000000..6d638d0 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/service/IndustryDemandAnalysisService.java @@ -0,0 +1,8 @@ +package com.sztzjy.linkCommerce.service; + +/** + * @author 17803 + * @date 2025-04-24 13:35 + */ +public interface IndustryDemandAnalysisService { +} diff --git a/src/main/java/com/sztzjy/linkCommerce/service/StuDigitalMarketingModelService.java b/src/main/java/com/sztzjy/linkCommerce/service/StuDigitalMarketingModelService.java new file mode 100644 index 0000000..fa3ebaa --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/service/StuDigitalMarketingModelService.java @@ -0,0 +1,28 @@ +package com.sztzjy.linkCommerce.service; + +import com.sztzjy.linkCommerce.entity.dto.AnalyzeDataDTO; +import com.sztzjy.linkCommerce.util.ResultEntity; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * @author tz + * @date 2024/6/14 11:05 + */ +public interface StuDigitalMarketingModelService { + List dropdownBox(String userId); + + ResultEntity emotionalAnalysis(String userId, String modelType, String content,String id) throws IOException; + + + ResultEntity descriptiveStatistics(Map> map, List statistic, String userId); + + List viewMetrics(String userId, String tableName,String algorithmName); + + ResultEntity viewAnalyzeData(AnalyzeDataDTO analyzeDataDTO); + + ResultEntity dataPreprocessing(String userId, String method, List> mapList); + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/service/impl/IndustryDemandAnalysisServiceImpl.java b/src/main/java/com/sztzjy/linkCommerce/service/impl/IndustryDemandAnalysisServiceImpl.java new file mode 100644 index 0000000..9c2a960 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/service/impl/IndustryDemandAnalysisServiceImpl.java @@ -0,0 +1,12 @@ +package com.sztzjy.linkCommerce.service.impl; + +import com.sztzjy.linkCommerce.service.IndustryDemandAnalysisService; +import org.springframework.stereotype.Service; + +/** + * @author 17803 + * @date 2025-04-24 13:35 + */ +@Service +public class IndustryDemandAnalysisServiceImpl implements IndustryDemandAnalysisService { +} diff --git a/src/main/java/com/sztzjy/linkCommerce/service/impl/StuDigitalMarketingModelServiceImpl.java b/src/main/java/com/sztzjy/linkCommerce/service/impl/StuDigitalMarketingModelServiceImpl.java new file mode 100644 index 0000000..734a7e2 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/service/impl/StuDigitalMarketingModelServiceImpl.java @@ -0,0 +1,645 @@ +package com.sztzjy.linkCommerce.service.impl; + + +import com.alibaba.fastjson.JSON; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.hankcs.hanlp.HanLP; +import com.hankcs.hanlp.seg.common.Term; +import com.sztzjy.linkCommerce.config.Constant; +import com.sztzjy.linkCommerce.entity.StuTableName; +import com.sztzjy.linkCommerce.entity.dto.AnalyzeDataDTO; +import com.sztzjy.linkCommerce.entity.dto.DescriptiveStatistics; +import com.sztzjy.linkCommerce.entity.dto.DescriptiveStatisticsDTO; +import com.sztzjy.linkCommerce.entity.dto.WordFrequencyDTO; +import com.sztzjy.linkCommerce.mapper.StuTableNameMapper; +import com.sztzjy.linkCommerce.mapper.StuUploadExcelUserMapper; +import com.sztzjy.linkCommerce.service.StuDigitalMarketingModelService; +import com.sztzjy.linkCommerce.util.ResultEntity; +import com.sztzjy.linkCommerce.util.algorithm.DescriptiveStatisticsUtil; +import okhttp3.*; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import javax.annotation.Resource; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.*; + +import static com.sztzjy.linkCommerce.util.StringUtils.convertCamelCaseToUnderscore; +import static com.sztzjy.linkCommerce.util.algorithm.BaiDuZhiNengYun.getAccessToken; + +/** + * @author tz + * @date 2024/6/14 11:05 + */ +@Service +public class StuDigitalMarketingModelServiceImpl implements StuDigitalMarketingModelService { + + private final OkHttpClient HTTP_CLIENT = new OkHttpClient().newBuilder().build(); + + + @Resource + StuTableNameMapper stuTableNameMapper; + + @Resource + StuUploadExcelUserMapper uploadExcelUserMapper; + + + @Override + public List dropdownBox(String userId) { + + List list=stuTableNameMapper.getTableName(userId); + + if(list.isEmpty()){ //设置默认值 + String[] arr={"客户细分数据表","销售预测数据表","购物车数据表"}; + + StuTableName stuTableName=new StuTableName(); + for (int i = 0; i < arr.length; i++) { + Integer uuid = UUID.randomUUID().toString().replaceAll("-", "").hashCode(); + uuid = uuid < 0 ? -uuid : uuid;//String.hashCode() 值会为空 + + stuTableName.setId(uuid); + stuTableName.setTableName(arr[i]); + stuTableName.setUserId(userId); + stuTableName.setCreateTime(new Date()); + + stuTableNameMapper.insert(stuTableName); + list.add(arr[i]); + } + } + return list; + } + + @Override + public ResultEntity emotionalAnalysis(String userId, String modelType, String content,String id) throws IOException { + + + MediaType mediaType = MediaType.parse("application/json"); + + if(modelType.equals(Constant.COMMENT_EXTRACTION)){ //评论观点抽取 + String commentUrl = "https://aip.baidubce.com/rpc/2.0/nlp/v2/comment_tag?charset=UTF-8&access_token="+ getAccessToken(); + + + RequestBody body = RequestBody.create(mediaType, "{\"text\":\""+content+"\",\"type\":\""+id+"\"}"); + + Request request = new Request.Builder() + .url(commentUrl) + .addHeader("Content-Type", "application/json") + .addHeader("Accept", "application/json") + .method("POST", body) + .build(); + Response response = HTTP_CLIENT.newCall(request).execute(); + + String string = response.body().string(); + // Gson对象 + Gson gson = new Gson(); + Map data = gson.fromJson(string, new TypeToken>(){}.getType()); +// Object object = data.get("items"); +// +// +// if(object instanceof List){ +// ArrayList tagList = (ArrayList) object; +// if (tagList.isEmpty()) { +// return new ResultEntity(HttpStatus.ACCEPTED,"请重新选择评论类型"); +// } +// } + if(data.containsKey("error_msg")){ + return new ResultEntity(HttpStatus.ACCEPTED,"评论异常"); + } + if (response.body() != null) { + return new ResultEntity(HttpStatus.OK, "成功", string); + }else { + return new ResultEntity(HttpStatus.ACCEPTED,"评论异常"); + } + + } + if (modelType.equals(Constant.EMOTIONAL_TENDENCIES)) { //情感倾向分析 + String emoUrl = "https://aip.baidubce.com/rpc/2.0/nlp/v1/sentiment_classify?charset=UTF-8&access_token=" + getAccessToken(); + + RequestBody body = RequestBody.create(mediaType, "{\"text\":\"" + content + "\"}"); + + Request request = new Request.Builder() + .url(emoUrl) + .method("POST", body) + .build(); + Response response = HTTP_CLIENT.newCall(request).execute(); + + String string = response.body().string(); + // Gson对象 + Gson gson = new Gson(); + Map data = gson.fromJson(string, new TypeToken>(){}.getType()); + + if(data.containsKey("error_msg")){ + return new ResultEntity(HttpStatus.ACCEPTED,"评论异常"); + } + + if (response.body() != null) { + return new ResultEntity(HttpStatus.OK, "成功", string); + } + + + } + + + if(modelType.equals(Constant.WORD_FREQUENCY)){ //词频分析 + + String string = this.filterPunctuation(content); + + // 使用 HanLP 分词 + List terms = HanLP.segment(string); + // 创建词汇表 + Map wordCount = new HashMap<>(); + // 读取停用词 + Set stopwords = this.loadStopwords("/usr/local/tianzeProject/digitalMarketing/jarAndDockerFile/停用词.txt"); + +// Set stopwords = this.loadStopwords("D:\\project\\digital_marketing\\src\\main\\resources\\words\\停用词.txt"); + + // 统计词频 + int count = 0; + for (Term term : terms) { + String word = term.word.trim(); // 去除空格 + if (!word.isEmpty() && !stopwords.contains(word)) { // 过滤停用词 + count++; + if (wordCount.containsKey(word)) { + wordCount.put(word, wordCount.get(word) + 1); + } else { + wordCount.put(word, 1); + } + } + if (count >= 200) { // 达到200个关键词时结束统计 + break; + } + } + + // 将词汇表转换为列表，便于排序 + List> wordList = new ArrayList<>(wordCount.entrySet()); + // 按词频排序 + wordList.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue())); + + List list = new ArrayList<>(); + + for (Map.Entry entry : wordList) { + WordFrequencyDTO frequencyDTO = new WordFrequencyDTO(); + frequencyDTO.setKeyword(entry.getKey()); + frequencyDTO.setFrequency(entry.getValue()); + list.add(frequencyDTO); + } + + return new ResultEntity<>(HttpStatus.OK, "成功", list); + + } +// if(modelType.equals(Constant.WORD_CLOUD)){ //词云分析 +// +// } + return null; + } + + // 过滤标点符号的函数 + private String filterPunctuation(String content) { + //标点符号 + return content.replaceAll("[，。、？！：;,.?!:'“”’‘]", ""); + } + + @Override + public ResultEntity descriptiveStatistics(Map> map, List statistic, String userId) { + + DescriptiveStatisticsUtil statisticsUtil=new DescriptiveStatisticsUtil(); + + List dtoList=new ArrayList<>(); + + for(String key: map.keySet()){ + DescriptiveStatisticsDTO statisticsDTO=new DescriptiveStatisticsDTO(); + statisticsDTO.setField(key); + + DescriptiveStatistics statistics=new DescriptiveStatistics(); + + DecimalFormat df = new DecimalFormat("#.0"); + + for (int i = 0; i < statistic.size(); i++) { + if(statistic.get(i).equals("平均数")){ + double mean = statisticsUtil.getMean(map.get(key)); + statistics.setAverage(Double.parseDouble(df.format(mean))); + + } + if(statistic.get(i).equals("中位数")){ + double median = statisticsUtil.getMedian(map.get(key)); + statistics.setMedian(Double.parseDouble(df.format(median))); + + } + if(statistic.get(i).equals("众数")){ + Double mode = statisticsUtil.getMode(map.get(key)); + statistics.setMode(Double.parseDouble(df.format(mode))); + + } + if(statistic.get(i).equals("标准差")){ + double standardDeviation = statisticsUtil.getStandardDeviation(map.get(key)); + + statistics.setStandardDeviation(Double.parseDouble(df.format(standardDeviation))); + + } + if(statistic.get(i).equals("方差")){ + double variance = statisticsUtil.getVariance(map.get(key)); + statistics.setVariance(Double.parseDouble(df.format(variance))); + + } + if(statistic.get(i).equals("标准误差")){ + double standardError = statisticsUtil.getStandardError(map.get(key)); + statistics.setStandardError(Double.parseDouble(df.format(standardError))); + + } + if(statistic.get(i).equals("峰度")){ + double kurtosis = statisticsUtil.getKurtosis(map.get(key)); + + if(Double.isNaN(kurtosis)){ + statistics.setKurtosis(0); + }else { + statistics.setKurtosis(Double.parseDouble(df.format(kurtosis))); + + } + + } + if(statistic.get(i).equals("偏度")){ + double skewness = statisticsUtil.getSkewness(map.get(key)); + + if(Double.isNaN(skewness)){ + statistics.setSkewness(0); + }else { + statistics.setSkewness(Double.parseDouble(df.format(skewness))); + } + + + } + if(statistic.get(i).equals("最大值")){ + double max = statisticsUtil.getMax(map.get(key)); + statistics.setMax(Double.parseDouble(df.format(max))); + + } + if(statistic.get(i).equals("最小值")){ + double min = statisticsUtil.getMin(map.get(key)); + statistics.setMin(Double.parseDouble(df.format(min))); + + } + if(statistic.get(i).equals("求和")){ + double sum = statisticsUtil.getSum(map.get(key)); + statistics.setSummation(Double.parseDouble(df.format(sum))); + + } + if(statistic.get(i).equals("观测数")){ + int count = statisticsUtil.getCount(map.get(key)); + statistics.setObservations(count); + } + + } + statisticsDTO.setStatistics(statistics); + dtoList.add(statisticsDTO); + + } + return new ResultEntity(HttpStatus.OK,"成功",dtoList); + } + + @Override + public List viewMetrics(String userId, String tableName,String algorithmName) { + + + List list=new ArrayList<>(); + + if (!StringUtils.hasText(tableName)) + { + tableName = "空表"; + } + + if(tableName.equals(Constant.KEHUXISHUJU)){ +// list=indicatorsMapper.getYHSXB(); +// list.add("id"); + list.add("gender"); + list.add("age"); + list.add("annual_income"); + list.add("spending_score"); + } + else if(tableName.equals(Constant.YONGHUXIAOSHOUNLB)){ +// list=indicatorsMapper.getYHDLHYB(); + list.add("sales_volume"); + list.add("sales_forehead"); + + } + else if(tableName.equals(Constant.GOUWUCHESHUJU)){ +// list=indicatorsMapper.getYHXFNLB(); + if(algorithmName.equals("关联规则挖掘")){ + list.add("consumer_goods"); + } + + } + else if(tableName.equals(Constant.YHPLB)){ +// list=indicatorsMapper.getYHPLB(); + if(algorithmName.equals("情感分析")){ + list.add("content"); + }else { + list.add("id"); + } + }else { + //查询导入的表数据 + + List stringList=new ArrayList<>(); + + stringList=uploadExcelUserMapper.selectByString(userId,tableName); + + //查询默认数据 + if(stringList.isEmpty()){ + stringList=uploadExcelUserMapper.selectByString("999999999",tableName); + } + + + for (String s : stringList) { + + List rightList = JSON.parseArray(s, String.class); + + List strings=new ArrayList<>(); + for (int i = 0; i < rightList.size(); i++) { //转为驼峰命名 + String string = convertCamelCaseToUnderscore(rightList.get(i)); + strings.add(string); + } + if(!list.isEmpty()){ + if(list.size() fieldList = analyzeDataDTO.getFieldList(); + String userId = analyzeDataDTO.getUserId(); + String tableName = analyzeDataDTO.getTableName(); + + List> attributes=new ArrayList<>(); + + attributes = stuTableNameMapper.selectBuildTable(fieldList,table,userId,tableName); + + if(attributes.isEmpty()){ + attributes = stuTableNameMapper.selectBuildTable(fieldList,table,"999999999",tableName); + } + + + for (int i = 0; i < attributes.size(); i++) { + if(attributes.get(i).size()> filteredAttributes = filterAndConvertAttributes(attributes); + + return new ResultEntity(HttpStatus.OK,filteredAttributes); + } + List fieldList = analyzeDataDTO.getFieldList(); + + + List> attributes = stuTableNameMapper.selectByFields(fieldList,table); + + + + for (int i = 0; i < attributes.size(); i++) { + if(attributes.get(i)!=null){ + if(attributes.get(i).size() row : attributes) { +// for (Map.Entry entry : row.entrySet()) { +// if (entry.getValue() instanceof LocalDateTime) { +// LocalDateTime localDateTime = (LocalDateTime) entry.getValue(); +// ZonedDateTime zonedDateTime = localDateTime.atZone(ZoneId.systemDefault()); +// Instant instant = zonedDateTime.toInstant(); +// entry.setValue(instant.toEpochMilli()); +// } +// } +// } + + + return new ResultEntity(HttpStatus.OK,attributes); + } + + @Override + public ResultEntity dataPreprocessing(String userId, String method, List> mapList) { + + if(mapList.isEmpty()){ + return null; + } + + //数据去重 + Set set = new HashSet<>(); + List> deduplicatedDataList = new ArrayList<>(); + + for (Map map : mapList) { + if(map.isEmpty()){ + continue; + } + String mapString = map.toString(); + if (!set.contains(mapString)) { + set.add(mapString); + deduplicatedDataList.add(map); + } + } + + //判断缺失值处理方式 + if(method.equals("缺失值处理--剔除数据")){ + + int a=0; + List list=new ArrayList<>(); + for (int i = 0; i < deduplicatedDataList.size(); i++) { + a = deduplicatedDataList.get(0).size(); + if(deduplicatedDataList.get(i).size() record.containsValue(null)); + + + } + + + + if(method.equals("缺失值处理--均值代替")){ + +// // 均值代替处理 +// for (Map record : deduplicatedDataList) { +// for (Map.Entry entry : record.entrySet()) { +// if (entry.getValue() == null) { +// double mean = calculateMean(deduplicatedDataList, entry.getKey()); +// entry.setValue(mean); +// } +// } +// } + + int a=0; + List list=new ArrayList<>(); + for (int i = 0; i < deduplicatedDataList.size(); i++) { + a = deduplicatedDataList.get(0).size(); + if(deduplicatedDataList.get(i).size() record : deduplicatedDataList) { + for (Map.Entry entry : record.entrySet()) { + if (entry.getValue() == null) { + String mode = calculateMode(deduplicatedDataList, entry.getKey()); + entry.setValue(mode); + } + } + } + } + return new ResultEntity(HttpStatus.OK,"成功",deduplicatedDataList); + } + + //均值代替计算方式 + private static double calculateMean(List> dataList, String key) { + double sum = 0; + int count = 0; + for (Map record : dataList) { + Object value = record.get(key); + if (value != null && value instanceof Number) { + sum += ((Number) value).doubleValue(); + count++; + } + } + return count > 0 ? (sum / count) : 0; + } + + //众数代替计算方式 + private static String calculateMode(List> dataList, String key) { + Map counts = new HashMap<>(); + for (Map record : dataList) { + Object value = record.get(key); + if (value != null && value instanceof String) { + String stringValue = (String) value; + counts.put(stringValue, counts.getOrDefault(stringValue, 0) + 1); + } + } + int maxCount = 0; + String mode = null; + for (Map.Entry entry : counts.entrySet()) { + if (entry.getValue() > maxCount) { + maxCount = entry.getValue(); + mode = entry.getKey(); + } + } + return mode != null ? mode : ""; // 如果存在众数则返回，否则返回空字符串 + } + public static List> filterEmptyValues(List> dataList) { + List> filteredList = new ArrayList<>(); + for (Map map : dataList) { + boolean allValuesEmpty = true; + for (Object value : map.values()) { + if (value != null) { + allValuesEmpty = false; + break; + } + } + if (!allValuesEmpty) { + filteredList.add(map); + } + } + return filteredList; + } + + // 过滤掉包含逗号的字符串值,并将负数转换为正数（对于数字类型） + public static List> filterAndConvertAttributes(List> attributes) { + List> filteredAndConvertedList = new ArrayList<>(); + + for (Map map : attributes) { + Map convertedMap = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + Object value = entry.getValue(); + + // 检查值是否为字符串且包含逗号 + if (value instanceof String) { + // 去除字符串中的逗号 + String strValue = (String) value; + value = strValue.replace(",", "").replace("-",""); + } + + // 检查值是否为数字类型且为负数 + if (value instanceof Number && ((Number) value).doubleValue() < 0) { + // 转换为正数 + value = Math.abs(((Number) value).doubleValue()); + // 注意：这里为了保持类型一致，我们将其转换回原始类型（如果需要） + // 这里简化为double类型，实际中可能需要更复杂的处理来保持原始类型 + } + + convertedMap.put(entry.getKey(), value); + } + filteredAndConvertedList.add(convertedMap); + } + + return filteredAndConvertedList; + } + + // 读取停用词 + Set loadStopwords(String filePath) throws IOException { + Set stopwords = new HashSet<>(); + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + while ((line = br.readLine()) != null) { + stopwords.add(line.trim()); + } + } + return stopwords; + } +} diff --git a/src/main/java/com/sztzjy/linkCommerce/util/StringUtils.java b/src/main/java/com/sztzjy/linkCommerce/util/StringUtils.java new file mode 100644 index 0000000..b83477d --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/StringUtils.java @@ -0,0 +1,24 @@ +package com.sztzjy.linkCommerce.util; + +public class StringUtils { + + /** + * 将驼峰命名法的字符串转换为下划线命名法的字符串。 + * @param camelCaseStr 驼峰命名的字符串。 + * @return 下划线命名法的字符串。 + */ + public static String convertCamelCaseToUnderscore(String camelCaseStr) { + // 使用Pattern和Matcher处理字符串 + StringBuilder underscoreStr = new StringBuilder(); + for (int i = 0; i < camelCaseStr.length(); i++) { + char c = camelCaseStr.charAt(i); + if (Character.isUpperCase(c) && i > 0) { + // 如果当前字符是大写，并且不是第一个字符，则添加下划线 + underscoreStr.append("_"); + } + // 将字符添加到结果字符串中，如果是大写，则转换为小写 + underscoreStr.append(Character.toLowerCase(c)); + } + return underscoreStr.toString(); + } +} \ No newline at end of file diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/Apriori.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/Apriori.java new file mode 100644 index 0000000..b448841 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/Apriori.java @@ -0,0 +1,288 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +import com.sztzjy.linkCommerce.config.exception.handler.ServiceException; +import com.sztzjy.linkCommerce.entity.dto.AssociationRulesDTO; +import org.springframework.http.HttpStatus; + +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.*; + +public class Apriori { + +// private double min_support; //最小支持度 +// private double min_confident; //最小置信度 + //文件路径 +// static String filePath = "D:\\home\\linkCommerce\\1.txt"; + static ArrayList> D = new ArrayList>();//事务数据库D + public static HashMap, Integer> C = new HashMap, Integer>();//项目集C + public static HashMap, Integer> L = new HashMap, Integer>();//候选集L + // 用于存取候选集每次计算结果（即存放所有的频繁项集L），最后计算关联规则，就不用再次遍历事务数据库。 + static HashMap, Integer> L_ALL = new HashMap, Integer>(); + //从文件中读取内容,返回事务集 + + public static ArrayList> readTable(List inputList) { + ArrayList> result = new ArrayList<>(); + for (String line : inputList) { + String[] values = line.split(","); + ArrayList rowData = new ArrayList<>(); + for (String value : values) { + rowData.add(value); + } + result.add(rowData); + } +// System.out.println("\nD:" + result); + return result; + } +// public static ArrayList> readTable(MultipartFile file) { +// ArrayList> t = new ArrayList<>(); +// try (InputStream inputStream = file.getInputStream(); +// Workbook workbook = new XSSFWorkbook(inputStream)) { +// Sheet sheet = workbook.getSheetAt(0); +// for (Row row : sheet) { +// ArrayList rowData = new ArrayList<>(); +// for (Cell cell : row) { +// rowData.add(cell.getStringCellValue()); +// } +// t.add(rowData); +// } +// } catch (IOException e) { +// e.printStackTrace(); +// System.out.println("文件读取失败！"); +// } +// System.out.println("\nD:" + t); +// return t; +// } + + //剪枝步：从候选集C中删除小于最小支持度的，并放入频繁集L中 + public static void pruning(HashMap, Integer> C,HashMap, Integer> L,double min_support) { + if(min_support==0){ + throw new ServiceException(HttpStatus.ACCEPTED,"最小支持度阀值不能为0"); + } + L.clear(); + // 根据项目集生成候选集 + L.putAll(C); + // 删除少于最小支持度的元素 + ArrayList> delete_key = new ArrayList>(); + for (ArrayList key : L.keySet()) { + if (L.get(key) < min_support) { + delete_key.add(key); + } + } + for (int i = 0; i < delete_key.size(); i++) { + L.remove(delete_key.get(i)); + } + } + + /** + * 初始化事务数据库、项目集、候选集 + */ + public static void init( List strings,double min_support) throws IOException { +// //将文件中的数据放入集合D中 + D = readTable(strings); + // 扫描事务数据库。生成项目集，支持度=该元素在事务数据库出现的次数/事务数据库的事务数 + for (int i = 0; i < D.size(); i++) { + for (int j = 0; j < D.get(i).size(); j++) { + String[] e = { D.get(i).get(j) }; + //将数组e转化为List + ArrayList item = new ArrayList(Arrays.asList(e)); + //map中是否包含指定的键 + if (!C.containsKey(item)) { + C.put(item, 1); + //System.out.println(C.get(item)); + } else { + C.put(item, C.get(item) + 1); + //System.out.println(C.get(item)); + } + } + } + //System.out.println("D.size= "+D.size()); + pruning(C, L,min_support);// 剪枝 + //将频繁项集放入集合中 + L_ALL.clear(); + L_ALL.putAll(L); + } + + // 两个整数集求并集 + public static ArrayList arrayListUnion(ArrayList arraylist1, ArrayList arraylist2) { + ArrayList arraylist = new ArrayList(); + arraylist.addAll(arraylist1); + arraylist.addAll(arraylist2); + //将ArrayList转化为HashSet去掉重复元素，再将HashSet转换为ArrayList + arraylist = new ArrayList(new HashSet(arraylist)); + return arraylist; + } + + /** + * 迭代求出最终的候选频繁集 + * + * @param C + * 完成初始化的项目集 + * @param L + * 完成初始化的候选集 + * @return 最终的候选频繁集 + */ + public static HashMap, Integer> iteration(HashMap, Integer> C,HashMap, Integer> L,double min_support) { + HashMap, Integer> L_temp = new HashMap, Integer>();// 用于判断是否结束剪枝的临时变量 + String str = null; + int t = 1;// 迭代次数 + while (L.size() > 0) {// 一旦被剪枝后剪干净，剪枝之前则是最终要求的结果。 + t++; + L_temp.clear(); + L_temp.putAll(L); + // 一、连接步 + C.clear(); + // 1.将L中的项以一定的规则两两匹配 + ArrayList> L_key = new ArrayList>(L.keySet()); + for (int i = 0; i < L_key.size(); i++) { + for (int j = i + 1; j < L_key.size(); j++) { + ArrayList C_item = new ArrayList(); + C_item = new ArrayList(arrayListUnion(L_key.get(i), + L_key.get(j))); + if (C_item.size() == t) { + C.put(C_item, 0);// 频繁项集的所有非空子集都必须是频繁的 + } + } + } + // 2.通过扫描D，计算此项的支持度 + for (ArrayList key : C.keySet()) { + for (int i = 0; i < D.size(); i++) { + if (D.get(i).containsAll(key)) { + C.put(key, C.get(key) + 1 ); + } + } + } + str = C.toString(); + +// System.out.println("候选"+t+"项集：C: \n"+C); + // 二、剪枝步 + pruning(C, L,min_support); +// System.out.println("频繁"+t+"项集：L: \n"+L+"\n"); + str = L.toString(); + +// System.out.println("==="); + L_ALL.putAll(L); + } + return L_temp; + } + + // 求一个集合的所有子集 + public static ArrayList> getSubset(ArrayList L) { + if (L.size() > 0) { + ArrayList> result = new ArrayList>(); + for (int i = 0; i < Math.pow(2, L.size()); i++) {// 集合子集个数=2的该集合长度的乘方 + ArrayList subSet = new ArrayList(); + int index = i;// 索引从0一直到2的集合长度的乘方-1 + for (int j = 0; j < L.size(); j++) { + // 通过逐一位移，判断索引那一位是1，如果是，再添加此项 + if ((index & 1) == 1) {// 位与运算，判断最后一位是否为1 + subSet.add(L.get(j)); + } + index >>= 1;// 索引右移一位 + } + result.add(subSet); // 把子集存储起来 + } + return result; + } else { + return null; + } + } + + // 判断两个集合相交是否为空 + public static boolean intersectionIsNull(ArrayList l1, + ArrayList l2) { + Set s1 = new HashSet(l1); + Set s2 = new HashSet(l2); + + s1.retainAll(s2); + if (s1.size() > 0) { + return false; + } else { + return true; + } + } + + /** + * 根据最终的关联集，根据公式计算出各个关联事件 + */ + public static List connection(double min_confident) { + + List list=new ArrayList<>(); + + List lists=new ArrayList<>(); + + for (ArrayList key : L_ALL.keySet()) {// 对最终的关联集各个事件进行判断 + if (key != null && !key.isEmpty() && L_ALL != null) { + ArrayList> key_allSubset = getSubset(key); + //得到所有频繁集中每个集合的子集 + // System.out.println(key_allSubset); + for (int i = 0; i < key_allSubset.size(); i++) { + ArrayList item_pre = key_allSubset.get(i);//得到一个真子集 + if(item_pre.size()>1){ + continue; + } + if(item_pre.isEmpty()){ + continue; + } + if (0 < item_pre.size() && item_pre.size() < key.size()) {// 判断是否是非空真子集 + // 各个非空互补真子集之间形成关联事件 + double item_pre_support = L_ALL.get(item_pre);//得到真子集的支持度度 + //System.out.println("itempre="+item_pre_support); + for (int j = 0; j < key_allSubset.size(); j++) { + ArrayList item_post = key_allSubset.get(j); + if (0 < item_post.size() + && item_post.size() < key.size() + && arrayListUnion(item_pre, item_post).equals(key) + && intersectionIsNull(item_pre, item_post)) + //不相交的两个非空真子集，相并为频繁项集 + { + double d = L_ALL.get(arrayListUnion(item_pre, item_post)); + //double item_post_support = L_ALL.get(item_post);// 互补真子集的支持度比则是事件的置信度 + //System.out.println("item_post="+item_post_support); + double confident = d + / item_pre_support; // 事件的置信度 + if (confident > min_confident) {// 如果事件的置信度大于最小置信度 + //封装对象返回 + + DecimalFormat df = new DecimalFormat("#.00"); + AssociationRulesDTO associationRulesDTO=new AssociationRulesDTO(); + associationRulesDTO.setCorrelation(item_pre); + associationRulesDTO.setAssociated(item_post); + associationRulesDTO.setConfidenceLevel(Double.parseDouble(df.format(confident))); + + String string=item_pre + "==>" + item_post+"==>" + Double.parseDouble(df.format(confident)); + associationRulesDTO.setRule(string); + + list.add(associationRulesDTO); + + + +// System.out.print(item_pre + "==>" + item_post );// 则是一个关联事件 +// System.out.println("==>" + confident); + + + + lists.add(string); + } + } + + } + } + } + } + + } + return list; + } + + public static void main(String[] args) { +// Apriori apriori = new Apriori(); +// apriori.init(D); +// System.out.println("候选1项集：C：\n"+apriori.C); +// System.out.println("频繁1项集：L：\n"+apriori.L+"\n"); +// apriori.L = apriori.iteration(apriori.C, apriori.L); +// System.out.println("关联规则如下："); +// apriori.connection(min_confident); + } +} + diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/BaiDuZhiNengYun.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/BaiDuZhiNengYun.java new file mode 100644 index 0000000..8fe3bfd --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/BaiDuZhiNengYun.java @@ -0,0 +1,41 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +import cn.hutool.json.JSONObject; +import okhttp3.*; +//import org.json.JSONObject; + + +import java.io.IOException; + +/** + * @author tz + * @date 2024/7/19 15:25 + */ +public class BaiDuZhiNengYun { + public static final String API_KEY = "12hSC65yXQdFndZridHTaHtd"; + public static final String SECRET_KEY = "0SegCsSEahKBYB4z6ZLELebLMRt37gib"; + + static final OkHttpClient HTTP_CLIENT = new OkHttpClient().newBuilder().build(); + + /** + * 从用户的AK，SK生成鉴权签名（Access Token） + * + * @return 鉴权签名（Access Token） + * @throws IOException IO异常 + */ + public static String getAccessToken() throws IOException { + MediaType mediaType = MediaType.parse("application/x-www-form-urlencoded"); + RequestBody body = RequestBody.create(mediaType, "grant_type=client_credentials&client_id=" + API_KEY + + "&client_secret=" + SECRET_KEY); + Request request = new Request.Builder() + .url("https://aip.baidubce.com/oauth/2.0/token") + .method("POST", body) + .addHeader("Content-Type", "application/x-www-form-urlencoded") + .build(); + Response response = HTTP_CLIENT.newCall(request).execute(); + if (response.body() != null) { +// return new JSONObject(response.body().string()).getString("access_token"); + } + return null; + } +} diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/Data.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/Data.java new file mode 100644 index 0000000..00f67b9 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/Data.java @@ -0,0 +1,11 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +public class Data { + public double[] features; + public double label; + + public Data(double[] features, double label) { + this.features = features; + this.label = label; + } +} \ No newline at end of file diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/DataDeal.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/DataDeal.java new file mode 100644 index 0000000..bba05bf --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/DataDeal.java @@ -0,0 +1,237 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.util.*; +public class DataDeal { + static double min_support = 4; //最小支持度 + static double min_confident = 0.7; //最小置信度 + //文件路径 + static String filePath = "D:\\home\\linkCommerce\\1.txt"; + static ArrayList> D = new ArrayList>();//事务数据库D + static HashMap, Integer> C = new HashMap, Integer>();//项目集C + static HashMap, Integer> L = new HashMap, Integer>();//候选集L + // 用于存取候选集每次计算结果（即存放所有的频繁项集L），最后计算关联规则，就不用再次遍历事务数据库。 + static HashMap, Integer> L_ALL = new HashMap, Integer>(); + //从文件中读取内容,返回事务集 + public static ArrayList> readTable(String filePath){ + ArrayList> t = new ArrayList>(); + ArrayList t1 = null; + File file = new File(filePath); + try { + InputStreamReader isr = new InputStreamReader(new FileInputStream(file)); + BufferedReader bf = new BufferedReader(isr); + String str = null; + while((str = bf.readLine()) != null) { + t1 = new ArrayList(); + String[] str1 = str.split(","); + for(int i = 1; i < str1.length ; i++) { + t1.add(str1[i]); + } + t.add(t1); + } + bf.close(); + isr.close(); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("文件不存在！"); + } + System.out.println("\nD:"+t); + return t; + } + //剪枝步：从候选集C中删除小于最小支持度的，并放入频繁集L中 + public static void pruning(HashMap, Integer> C,HashMap, Integer> L) { + L.clear(); + // 根据项目集生成候选集 + L.putAll(C); + // 删除少于最小支持度的元素 + ArrayList> delete_key = new ArrayList>(); + for (ArrayList key : L.keySet()) { + if (L.get(key) < min_support) { + delete_key.add(key); + } + } + for (int i = 0; i < delete_key.size(); i++) { + L.remove(delete_key.get(i)); + } + } + + /** + * 初始化事务数据库、项目集、候选集 + */ + public static void init() { + //将文件中的数据放入集合D中 + D = readTable(filePath); + // 扫描事务数据库。生成项目集，支持度=该元素在事务数据库出现的次数/事务数据库的事务数 + for (int i = 0; i < D.size(); i++) { + for (int j = 0; j < D.get(i).size(); j++) { + String[] e = { D.get(i).get(j) }; + //将数组e转化为List + ArrayList item = new ArrayList(Arrays.asList(e)); + //map中是否包含指定的键 + if (!C.containsKey(item)) { + C.put(item, 1); + //System.out.println(C.get(item)); + } else { + C.put(item, C.get(item) + 1); + //System.out.println(C.get(item)); + } + } + } + //System.out.println("D.size= "+D.size()); + pruning(C, L);// 剪枝 + //将频繁项集放入集合中 + L_ALL.putAll(L); + } + + // 两个整数集求并集 + public static ArrayList arrayListUnion(ArrayList arraylist1, ArrayList arraylist2) { + ArrayList arraylist = new ArrayList(); + arraylist.addAll(arraylist1); + arraylist.addAll(arraylist2); + //将ArrayList转化为HashSet去掉重复元素，再将HashSet转换为ArrayList + arraylist = new ArrayList(new HashSet(arraylist)); + return arraylist; + } + + /** + * 迭代求出最终的候选频繁集 + * + * @param C + * 完成初始化的项目集 + * @param L + * 完成初始化的候选集 + * @return 最终的候选频繁集 + */ + public static HashMap, Integer> iteration(HashMap, Integer> C,HashMap, Integer> L) { + HashMap, Integer> L_temp = new HashMap, Integer>();// 用于判断是否结束剪枝的临时变量 + String str = null; + int t = 1;// 迭代次数 + while (L.size() > 0) {// 一旦被剪枝后剪干净，剪枝之前则是最终要求的结果。 + t++; + L_temp.clear(); + L_temp.putAll(L); + // 一、连接步 + C.clear(); + // 1.将L中的项以一定的规则两两匹配 + ArrayList> L_key = new ArrayList>(L.keySet()); + for (int i = 0; i < L_key.size(); i++) { + for (int j = i + 1; j < L_key.size(); j++) { + ArrayList C_item = new ArrayList(); + C_item = new ArrayList(arrayListUnion(L_key.get(i), + L_key.get(j))); + if (C_item.size() == t) { + C.put(C_item, 0);// 频繁项集的所有非空子集都必须是频繁的 + } + } + } + // 2.通过扫描D，计算此项的支持度 + for (ArrayList key : C.keySet()) { + for (int i = 0; i < D.size(); i++) { + if (D.get(i).containsAll(key)) { + C.put(key, C.get(key) + 1 ); + } + } + } + str = C.toString(); + + System.out.println("候选"+t+"项集：C: \n"+C); + // 二、剪枝步 + pruning(C, L); + System.out.println("频繁"+t+"项集：L: \n"+L+"\n"); + str = L.toString(); + + //System.out.println("==="); + L_ALL.putAll(L); + } + return L_temp; + } + + // 求一个集合的所有子集 + public static ArrayList> getSubset(ArrayList L) { + if (L.size() > 0) { + ArrayList> result = new ArrayList>(); + for (int i = 0; i < Math.pow(2, L.size()); i++) {// 集合子集个数=2的该集合长度的乘方 + ArrayList subSet = new ArrayList(); + int index = i;// 索引从0一直到2的集合长度的乘方-1 + for (int j = 0; j < L.size(); j++) { + // 通过逐一位移，判断索引那一位是1，如果是，再添加此项 + if ((index & 1) == 1) {// 位与运算，判断最后一位是否为1 + subSet.add(L.get(j)); + } + index >>= 1;// 索引右移一位 + } + result.add(subSet); // 把子集存储起来 + } + return result; + } else { + return null; + } + } + + // 判断两个集合相交是否为空 + public static boolean intersectionIsNull(ArrayList l1, + ArrayList l2) { + Set s1 = new HashSet(l1); + Set s2 = new HashSet(l2); + + s1.retainAll(s2); + if (s1.size() > 0) { + return false; + } else { + return true; + } + } + + /** + * 根据最终的关联集，根据公式计算出各个关联事件 + */ + public static void connection() { + for (ArrayList key : L_ALL.keySet()) {// 对最终的关联集各个事件进行判断 + ArrayList> key_allSubset = getSubset(key); + //得到所有频繁集中每个集合的子集 + // System.out.println(key_allSubset); + for (int i = 0; i < key_allSubset.size(); i++) { + ArrayList item_pre = key_allSubset.get(i);//得到一个真子集 + if (0 < item_pre.size() && item_pre.size() < key.size()) {// 判断是否是非空真子集 + // 各个非空互补真子集之间形成关联事件 + double item_pre_support = L_ALL.get(item_pre);//得到真子集的支持度度 + //System.out.println("itempre="+item_pre_support); + for (int j = 0; j < key_allSubset.size(); j++) { + ArrayList item_post = key_allSubset.get(j); + if (0 < item_post.size() + && item_post.size() < key.size() + && arrayListUnion(item_pre, item_post).equals(key) + && intersectionIsNull(item_pre, item_post)) + //不相交的两个非空真子集，相并为频繁项集 + { + double d = L_ALL.get(arrayListUnion(item_pre, item_post)); + //double item_post_support = L_ALL.get(item_post);// 互补真子集的支持度比则是事件的置信度 + //System.out.println("item_post="+item_post_support); + double confident = d + / item_pre_support; // 事件的置信度 + if (confident > min_confident) {// 如果事件的置信度大于最小置信度 + System.out.print(item_pre + "==>" + item_post );// 则是一个关联事件 + System.out.println("==>" + confident); + } + } + + } + } + } + } + } + + public static void main(String[] args) { + DataDeal apriori = new DataDeal(); + apriori.init(); + System.out.println("候选1项集：C：\n"+apriori.C); + System.out.println("频繁1项集：L：\n"+apriori.L+"\n"); + apriori.L = apriori.iteration(apriori.C, apriori.L); + System.out.println("关联规则如下："); + apriori.connection(); + } +} + diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/DescriptiveStatisticsUtil.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/DescriptiveStatisticsUtil.java new file mode 100644 index 0000000..ec58e6b --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/DescriptiveStatisticsUtil.java @@ -0,0 +1,135 @@ +package com.sztzjy.linkCommerce.util.algorithm; + + + +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; + +import java.util.*; + +public class DescriptiveStatisticsUtil { + + // 计算平均数 + public double getMean(List data) { + DescriptiveStatistics stats = new DescriptiveStatistics(); + for (double value : data) { + stats.addValue(value); + } + return stats.getMean(); + } + + + // 计算中位数 + public double getMedian(List data) { + Collections.sort(data); + int size = data.size(); + if (size % 2 == 0) { + return (data.get(size / 2 - 1) + data.get(size / 2)) / 2.0; + } else { + return data.get(size / 2); + } + } + + + // 计算众数 + + public Double getMode(List data) { + HashMap map = new HashMap(); + double imode = 0; + for (int i = 0; i < data.size(); i++) { + double x = data.get(i); + if (map.containsKey(String.valueOf(x))) { + // 如果出现多次，取出以前的计数，然后加1 + int len = Integer.parseInt(map.get(String.valueOf(x)).toString()); + map.put(String.valueOf(x), String.valueOf(len + 1)); + imode = Math.max(imode, len + 1); + } else { + // 如果是第一次出现，计数为1 + map.put(String.valueOf(x), String.valueOf(1)); + imode = Math.max(imode, 1); + } + } + Iterator iter = map.keySet().iterator(); + ArrayList lst = new ArrayList(); + while (iter.hasNext()) { + Object key = iter.next(); + Object v = map.get(key); + if (Integer.parseInt(v.toString()) == imode) { + lst.add(key); + } + } + + Object object = lst.get(0); + Double v = Double.valueOf(object.toString()); + return v; + } + + + // 计算标准差 + public double getStandardDeviation(List data) { + DescriptiveStatistics stats = new DescriptiveStatistics(); + for (double value : data) { + stats.addValue(value); + } + return stats.getStandardDeviation(); + } + + // 计算方差 + public double getVariance(List data) { + DescriptiveStatistics stats = new DescriptiveStatistics(); + for (double value : data) { + stats.addValue(value); + } + return stats.getVariance(); + } + + // 计算标准误差 + public double getStandardError(List data) { + double standardDeviation = getStandardDeviation(data); + int count = getCount(data); + return standardDeviation / Math.sqrt(count); + } + + // 计算峰度 + public double getKurtosis(List data) { + DescriptiveStatistics stats = new DescriptiveStatistics(); + for (double value : data) { + + stats.addValue(value); + } + return stats.getKurtosis(); + } + + // 计算偏度 + public double getSkewness(List data) { + DescriptiveStatistics stats = new DescriptiveStatistics(); + for (double value : data) { + stats.addValue(value); + } + double skewness = stats.getSkewness(); + return skewness; + } + + // 获取最大值 + public double getMax(List data) { + return Collections.max(data); + } + + // 获取最小值 + public double getMin(List data) { + return Collections.min(data); + } + + // 计算总和 + public double getSum(List data) { + double sum = 0; + for (double value : data) { + sum += value; + } + return sum; + } + + // 获取观测数 + public int getCount(List data) { + return data.size(); + } +} \ No newline at end of file diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/KMeans.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/KMeans.java new file mode 100644 index 0000000..c519759 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/KMeans.java @@ -0,0 +1,182 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; + +public class KMeans { + + private int k; // 簇数 + private int maxIterations; // 最大迭代次数 + private List centroids; // 质心 + private List points; // 数据集 + + + public KMeans(int k, int maxIterations, List points) { + this.k = k; + this.maxIterations = maxIterations; + this.points = points; + this.centroids = initializeCentroids(); + } + + public List getCentroids() { + return centroids; + } + + + // 初始化质心 + private List initializeCentroids() { + List centroids = new ArrayList<>(); + Random random = new Random(); + for (int i = 0; i < k; i++) { + int randomIndex = random.nextInt(points.size()); + centroids.add(points.get(randomIndex)); + } + return centroids; + } + + // 计算点到质心的距离 + private double distance(Point point1, Point point2) { + return Math.sqrt(Math.pow(point1.getX() - point2.getX(), 2) + Math.pow(point1.getY() - point2.getY(), 2)); + } + + // 找到每个点所属的最近质心 + private int closestCentroid(Point point) { + double minDistance = Double.MAX_VALUE; + int closestCentroidIndex = -1; + for (int i = 0; i < centroids.size(); i++) { + double dist = distance(point, centroids.get(i)); + if (dist < minDistance) { + minDistance = dist; + closestCentroidIndex = i; + } + } + return closestCentroidIndex; + } + + // 更新质心位置为每个簇的平均值 + private void updateCentroids(List> clusters) { + for (int i = 0; i < k; i++) { + double sumX = 0; + double sumY = 0; + List cluster = clusters.get(i); + int clusterSize = cluster.size(); + for (Point point : cluster) { + sumX += point.getX(); + sumY += point.getY(); + } + if (clusterSize > 0) { + centroids.get(i).setX(sumX / clusterSize); + centroids.get(i).setY(sumY / clusterSize); + } + } + } + + // K-means聚类算法 + public List> cluster() { + List> clusters=null; + for (int iter = 0; iter < maxIterations; iter++) { + clusters = new ArrayList<>(k); + for (int i = 0; i < k; i++) { + clusters.add(new ArrayList<>()); + } + + // 分配每个点到最近的质心 + for (Point point : points) { + int closestIndex = closestCentroid(point); + clusters.get(closestIndex).add(point); + } + + // 更新质心位置 + updateCentroids(clusters); + } + + // 返回每个簇的点集合 + return clusters; + } + + // 示例中的数据点类 + public static class Point { + private double x; + private double y; + + public Point(double x, double y) { + this.x = x; + this.y = y; + } + + public double getX() { + return x; + } + + public double getY() { + return y; + } + + public void setX(double x) { + this.x = x; + } + + public void setY(double y) { + this.y = y; + } + + @Override + public String toString() { + return "(" + x + ", " + y + ")"; + } + } + + public static void main(String[] args) { +// List irisData = readIrisData("D:\\home\\linkCommerce\\Iris.txt"); +// int k = 3; // 假设聚为3类 +// int maxIterations = 100; +// KMeans kMeans = new KMeans(k, maxIterations, irisData); +// List> clusters = kMeans.cluster(); +// +// // 打印每个簇的结果 +// for (int i = 0; i < clusters.size(); i++) { +// System.out.println("Cluster " + i + ": " + clusters.get(i)); +// } +// +// // 获取每个簇的中心点坐标 +// List centroids = kMeans.getCentroids(); +// for (int i = 0; i < centroids.size(); i++) { +// System.out.println("Centroid for cluster " + i + ": " + centroids.get(i)); +// } + + } + + public static List convertToDelimitedStringList(List> nestedList) { + List flatList = new ArrayList<>(); + for (Map innerMap : nestedList) { + StringBuilder sb = new StringBuilder(); + for (Object value : innerMap.values()) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(value); + } + flatList.add(sb.toString()); + } + return flatList; + } + + + public static List readIrisData(List data) { + List points = new ArrayList<>(); + for (String line : data) { + String[] values = line.split(","); + + double x = Double.parseDouble(values[0]); + double y = Double.parseDouble(values[1]); + + + points.add(new Point(x, y)); + } + return points; + } + + +} \ No newline at end of file diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/KMeansResult.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/KMeansResult.java new file mode 100644 index 0000000..ed8029f --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/KMeansResult.java @@ -0,0 +1,288 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +import com.sztzjy.linkCommerce.entity.dto.KmeansDTO; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * @author tz + * @date 2024/7/2 17:37 + */ +public class KMeansResult { + + // 记录迭代的次数 + static int count = 1; + // 文件所在路径 +// static String filePath = "D:\\home\\linkCommerce\\Segmentation_dataset(1).csv"; + // 储存从文件中读取的数据 + static ArrayList> table = new ArrayList>(); + // 储存分类一的结果 + static ArrayList> alist = new ArrayList>(); + // 储存分类二的结果 + static ArrayList> blist = new ArrayList>(); + // 储存分类三的结果 + static ArrayList> clist = new ArrayList>(); + // 记录初始随机产生的3个聚类中心 + static ArrayList> randomList = new ArrayList>(); + + // 读取文件中的数据，储存到集合中 + public static ArrayList> readTable(List data) { + table.clear(); + + + for (String line : data) { + ArrayList d = new ArrayList(); + String[] str1 = line.split(","); + for (String str2 : str1) { + d.add(Float.parseFloat(str2)); + } + table.add(d); + } + + return table; + } +// public static ArrayList> readTable(MultipartFile file){ +// table.clear(); +// ArrayList d = null; +// +// try (BufferedReader br = new BufferedReader(new InputStreamReader(file.getInputStream()))) { +// +// +// // 跳过标题行 +// br.readLine(); +// +// String str = null; +// while((str = br.readLine()) != null) { +// d = new ArrayList(); +// String[] str1 = str.split(","); +// for(int i = 0; i < str1.length ; i++) { +// d.add(Float.parseFloat(str1[i])); +// } +// table.add(d); +// } +//// System.out.println(table); +// +// } catch (Exception e) { +// e.printStackTrace(); +// System.out.println("文件不存在！"); +// } +// return table; +// } + + // 随机产生k个初始聚类中心 + public static ArrayList> randomList(Integer k,ArrayList> randomList) { + ArrayList> centerList=new ArrayList<>(); + + int[] list = new int[k]; + + // 生成k个不同的随机数 + int size = table.size(); //生成的随机数不能大于table中元素的个数 + do { + for (int i = 0; i < k; i++) { + list[i] = (int)(Math.random() * size); + } + } while (hasDuplicates(list)); + + + // 将数据集中的前k个元素作为初始集群中心 + for (int i = 0; i < k; i++) { + centerList.add(table.get(list[i])); + } + + return centerList; + } + + // helper方法来检查数组中是否包含重复项 + private static boolean hasDuplicates(int[] arr) { + Set set = new HashSet<>(); + for (int val : arr) { + if (!set.add(val)) { + return true; + } + } + return false; + } + + //比较两个数的大小，并返回其中较小的数 + public static double minNumber(double x, double y) { + if(x < y) { + return x; + } + return y; + } + + // 计算各个数据到中心点的距离，然后分成k类 + public static void eudistance(ArrayList> list, int k) { + ArrayList>> clusters = new ArrayList<>(k); + for (int i = 0; i < k; i++) { + clusters.add(new ArrayList<>()); + } + + for (int i = 0; i < table.size(); i++) { + double[] distances = new double[k]; + for (int j = 0; j < k; j++) { + distances[j] = calculateEuclideanDistance(table.get(i), list.get(j)); + } + int closestCluster = findClosestCluster(distances); + clusters.get(closestCluster).add(table.get(i)); + } + + System.out.println("第" + count + "次迭代:"); + for (int i = 0; i < k; i++) { + System.out.println("质心 " + i + ": " + clusters.get(i)); + } + System.out.println(); + count++; + } + + //计算两个数据点之间的欧几里得距离 + private static double calculateEuclideanDistance(ArrayList point1, ArrayList point2) { + double sum = 0; + for (int i = 1; i < point1.size(); i++) { + if(i<=point2.size()){ + sum += Math.pow(point1.get(i) - point2.get(i), 2); + } + } + return Math.sqrt(sum); + } + + //根据距离数组找到最近的簇的索引 + private static int findClosestCluster(double[] distances) { + double minDistance = Double.MAX_VALUE; + int closestCluster = 0; + for (int i = 0; i < distances.length; i++) { + if (distances[i] < minDistance) { + minDistance = distances[i]; + closestCluster = i; + } + } + return closestCluster; + } + + //最终聚类结果 + public static KmeansDTO kmeans(int k, Integer t, ArrayList> centroids) { + ArrayList>> clusters = new ArrayList<>(k); + for (int i = 0; i < k; i++) { + clusters.add(new ArrayList<>()); + } + + boolean converged = false; + int iterations = 0; + while (!converged && iterations < t) { + for (int i = 0; i < table.size(); i++) { + ArrayList point = table.get(i); + int closestCluster = findClosestCluster(point, centroids); + clusters.get(closestCluster).add(point); + } + + ArrayList> newCentroids = new ArrayList<>(k); + for (int i = 0; i < k; i++) { + newCentroids.add(calculateCentroid(clusters.get(i))); + } + + converged = checkConvergence(centroids, newCentroids); + + if(!converged){ + centroids = newCentroids; + for (int i = 0; i < k; i++) { + clusters.get(i).clear(); + } + iterations++; + continue; + }else { + break; + } + } + +// System.out.println("最终聚类结果:"); +// for (int i = 0; i < k; i++) { +// System.out.println("簇 " + i + ": " + clusters.get(i)); +// } + + KmeansDTO kmeansDTO=new KmeansDTO(); + kmeansDTO.setClusters(clusters); + kmeansDTO.setIterations(iterations); + return kmeansDTO; + } + + //计算一个簇的质心 + private static ArrayList calculateCentroid(ArrayList> cluster) { + float sum1 = 0, sum2 = 0, sum3 = 0, sum4 = 0; + for (ArrayList point : cluster) { + for (int i = 0; i < point.size(); i++) { + if(i==0){ + sum1 += point.get(0); + } + if(i==1){ + sum2 += point.get(1); + } + if(i==2){ + sum3 += point.get(2); + } + if(i==3){ + sum4 += point.get(3); + } + + } + + } + + int a=0; + for (ArrayList point : cluster) { + a=point.size(); + } + float size = cluster.size(); + ArrayList centroid = new ArrayList<>(); + for (int i = 0; i < a; i++) { + if(i==0){ + centroid.add(sum1 / size); + } + if(i==1){ + centroid.add(sum2 / size); + } + if(i==2){ + centroid.add(sum3 / size); + } + if(i==3){ + centroid.add(sum4 / size); + } + } + return centroid; + } + + //检查是否收敛，如果没有收敛,则用新质心替换旧质心,并清空所有簇,准备进行下一轮迭代 + private static boolean checkConvergence(ArrayList> oldCentroids, ArrayList> newCentroids) { + for (int i = 0; i < oldCentroids.size(); i++) { + if (!oldCentroids.get(i).equals(newCentroids.get(i))) { + return false; + } + } + return true; + } + + private static int findClosestCluster(ArrayList point, ArrayList> centroids) { + double minDistance = Double.MAX_VALUE; + int closestCluster = 0; + for (int i = 0; i < centroids.size(); i++) { + double distance = calculateEuclideanDistance(point, centroids.get(i)); + if (distance < minDistance) { + minDistance = distance; + closestCluster = i; + } + } + return closestCluster; + } + + + public static void main(String[] args) { +// ArrayList> rlist = new ArrayList>(); +// readTable(filePath); +// rlist = randomList(); +// eudistance(rlist); +// kmeans(); + } + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/LinearRegression.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/LinearRegression.java new file mode 100644 index 0000000..5614cfe --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/LinearRegression.java @@ -0,0 +1,96 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +import java.text.DecimalFormat; + +public class LinearRegression { + private double intercept; + private double[] slopes; + + // 线性回归的训练方法 + public void fit(double[][] x, double[] y) { + int n = y.length; + int m = x[0].length; + double[] sumX = new double[m]; + double sumY = 0.0; + double[] sumXY = new double[m]; + double[] sumX2 = new double[m]; + + for (int i = 0; i < n; i++) { + for (int j = 0; j < m; j++) { + sumX[j] += x[i][j]; + sumXY[j] += x[i][j] * y[i]; + sumX2[j] += x[i][j] * x[i][j]; + } + sumY += y[i]; + } + + double[] xMean = new double[m]; + for (int j = 0; j < m; j++) { + xMean[j] = sumX[j] / n; + } + double yMean = sumY / n; + + DecimalFormat df = new DecimalFormat("#.00"); + + + this.slopes = new double[m]; + for (int j = 0; j < m; j++) { + this.slopes[j] = Double.parseDouble(df.format((sumXY[j] - n * xMean[j] * yMean) / (sumX2[j] - n * xMean[j] * xMean[j]))); + } + + this.intercept = Double.parseDouble(df.format(yMean - dotProduct(this.slopes, xMean))); + } + + // 辅助方法：计算向量的点积 + private double dotProduct(double[] a, double[] b) { + double result = 0.0; + for (int i = 0; i < a.length; i++) { + result += a[i] * b[i]; + } + return result; + } + + // 预测方法 + public double predict(double[] x,double intercept,double[] slopes) { + if (x.length != slopes.length) { + throw new IllegalArgumentException("输入的x维度与模型不匹配"); + } + return intercept + dotProduct(slopes, x); + } + + // 获取截距 + public double getIntercept() { + return this.intercept; + } + + // 获取斜率 + public double[] getSlopes() { + return this.slopes; + } + +// public static void main(String[] args) { +// // 示例数据 +// double[][] x = {{1, 19}, {1, 21}, {2, 20}, {2, 23}, {2, 31}}; +// double[] y = {15, 15, 16, 16, 17}; +// +// // 创建线性回归模型 +// LinearRegression lr = new LinearRegression(); +// lr.fit(x, y); +// +// DecimalFormat df = new DecimalFormat("#.0"); +// +// List list=new ArrayList<>(); +// for (int i = 0; i < lr.getSlopes().length; i++) { +// String format = df.format(lr.getSlopes()[i]); +// list.add(format); +// } +// // 输出模型参数 +// System.out.println("常数项值: " + df.format(lr.getIntercept())); +// System.out.println("第一个特征系数值: " + list.get(0)); +// +// // 预测新数据 +// double[] newX = {2, 88}; +// double predictedY = lr.predict(newX); +// System.out.println("输入x: " + java.util.Arrays.toString(newX) + "预测y=" + df.format(predictedY)); +// } +} \ No newline at end of file diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/LogisticRegression.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/LogisticRegression.java new file mode 100644 index 0000000..e66553e --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/LogisticRegression.java @@ -0,0 +1,57 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +public class LogisticRegression { + + private double[] x; + + + private double[] y; + + public LogisticRegression(double[] x, double[] y) { + this.x = x; + this.y = y; + } + + // 计算线性回归参数 + public double[] fit() { + if (x.length != y.length || x.length == 0) { + throw new IllegalArgumentException("Invalid input data"); + } + + int n = x.length; + double sumX = 0; + double sumY = 0; + double sumXY = 0; + double sumXX = 0; + + // 计算各项和 + for (int i = 0; i < n; i++) { + sumX += x[i]; + sumY += y[i]; + sumXY += x[i] * y[i]; + sumXX += x[i] * x[i]; + } + + // 计算斜率和截距 + double slope = (n * sumXY - sumX * sumY) / (n * sumXX - sumX * sumX); + double intercept = (sumY - slope * sumX) / n; + + return new double[] { intercept, slope }; + } + + public double[][] getCoordinatesOnLine() { + double[] parameters = fit(); + double intercept = parameters[0]; + double slope = parameters[1]; + + double[][] coordinates = new double[x.length][2]; + + for (int i = 0; i < x.length; i++) { + coordinates[i][0] = x[i]; + coordinates[i][1] = slope * x[i] + intercept; + } + + return coordinates; + } + +} \ No newline at end of file diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/ResourceFileIoAdapter.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/ResourceFileIoAdapter.java new file mode 100644 index 0000000..f276ae4 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/ResourceFileIoAdapter.java @@ -0,0 +1,23 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +import cn.hutool.core.io.resource.ClassPathResource; +import com.hankcs.hanlp.corpus.io.IIOAdapter; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; + +public class ResourceFileIoAdapter implements IIOAdapter { + @Override + public InputStream open(String path) throws IOException { + ClassPathResource resource = new ClassPathResource(path); + return Files.newInputStream(resource.getFile().toPath()); + } + + @Override + public OutputStream create(String path) throws IOException { + ClassPathResource resource = new ClassPathResource(path); + return Files.newOutputStream(resource.getFile().toPath()); + } +} \ No newline at end of file diff --git a/src/main/java/com/sztzjy/linkCommerce/util/algorithm/TrainDataSet.java b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/TrainDataSet.java new file mode 100644 index 0000000..077b966 --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/algorithm/TrainDataSet.java @@ -0,0 +1,50 @@ +package com.sztzjy.linkCommerce.util.algorithm; + +import java.util.ArrayList; +import java.util.List; + +public class TrainDataSet { + + /** + * 特征集合 + **/ + public List features = new ArrayList<>(); + /** + * 标签集合 + **/ + public List labels = new ArrayList<>(); + /** + * 特征向量维度 + **/ + public int featureDim; + + public int size() { + return labels.size(); + } + + public double[] getFeature(int index) { + return features.get(index); + } + + public double getLabel(int index) { + return labels.get(index); + } + + public void addData(double[] feature, double label) { + if (features.isEmpty()) { + featureDim = feature.length; + } else { + if (featureDim != feature.length) { + throwDimensionMismatchException(feature.length); + } + } + features.add(feature); + labels.add(label); + } + + public void throwDimensionMismatchException(int errorLen) { + throw new RuntimeException("DimensionMismatchError: 你应该传入维度为 " + featureDim + " 的特征向量 , 但你传入了维度为 " + errorLen + " 的特征向量"); + } + + +} diff --git a/src/main/java/com/sztzjy/linkCommerce/util/excel/DataListener.java b/src/main/java/com/sztzjy/linkCommerce/util/excel/DataListener.java new file mode 100644 index 0000000..ee504ec --- /dev/null +++ b/src/main/java/com/sztzjy/linkCommerce/util/excel/DataListener.java @@ -0,0 +1,183 @@ +package com.sztzjy.linkCommerce.util.excel; + + +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.metadata.data.ReadCellData; +import com.alibaba.excel.read.listener.ReadListener; +import com.alibaba.excel.util.ConverterUtils; +import com.alibaba.excel.util.ListUtils; +import com.alibaba.fastjson.JSON; +import com.sztzjy.linkCommerce.entity.HyxqfxData; +import com.sztzjy.linkCommerce.entity.StuUploadExcelUser; +import com.sztzjy.linkCommerce.mapper.HyxqfxDataMapper; +import com.sztzjy.linkCommerce.mapper.StuUploadExcelUserMapper; +import lombok.extern.slf4j.Slf4j; + +import java.lang.reflect.Field; +import java.util.*; + + +// 有个很重要的点 DemoDataListener 不能被spring管理，要每次读取excel都要new,然后里面用到spring可以构造方法传进去 +@Slf4j + +public class DataListener implements ReadListener { + + + /** + * 每隔5条存储数据库，实际使用中可以100条，然后清理list ，方便内存回收 + * + */ + private static final int BATCH_COUNT = 150; + /** + * 缓存的数据 + */ + private List cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT); + /** + * 假设这个是一个DAO，当然有业务逻辑这个也可以是一个service。当然如果不用存储这个对象没用。 + */ + private HyxqfxDataMapper demoDAO; + private String userId; + private String name; + + List list = new ArrayList(); + + /** + * 如果使用了spring,请使用这个构造方法。每次创建Listener的时候需要把spring管理的类传进来 + * + * @param demoDAO + */ + public DataListener(HyxqfxDataMapper demoDAO, String userId, String name) { + this.name = name; + this.userId = userId; + this.demoDAO = demoDAO; + } + + + /** + * 这里会一行行的返回头 + * + * @param headMap + * @param context + */ + @Override + public void invokeHead(Map> headMap, AnalysisContext context) { + // log.info("解析到一条头数据:{}", JSON.toJSONString(headMap)); + + Map integerStringMap = ConverterUtils.convertToStringMap(headMap, context); + + //log.info("转换解析到的一条头数据:{}", integerStringMap); + + + Set integers = integerStringMap.keySet(); + for (Integer integer : integers) { + list.add(integerStringMap.get(integer)); + } + + + // 如果想转成成 Map + // 方案1： 不要implements ReadListener 而是 extends AnalysisEventListener + // 方案2： 调用 ConverterUtils.convertToStringMap(headMap, context) 自动会转换 + } + + /** + * 这个每一条数据解析都会来调用 + * + * @param data one row value. Is is same as {@link AnalysisContext#readRowHolder()} + * @param context + */ + @Override + public void invoke(HyxqfxData data, AnalysisContext context) { + //log.info("解析到一条数据:{}", JSON.toJSONString(data)); + cachedDataList.add(data); + // 达到BATCH_COUNT了，需要去存储一次数据库，防止数据几万条数据在内存，容易OOM + if (cachedDataList.size() >= BATCH_COUNT) { + saveData(); + // 存储完成清理 list + cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT); + } + } + + /** + * 所有数据解析完成了 都会来调用 + * + * @param context + */ + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + + saveData(); + log.info("所有数据解析完成！"+ "，当前线程：" + Thread.currentThread().getName()); + } + + /** + * 加上存储数据库 + */ + private void saveData() { + + + Map map = new HashMap<>(); + HyxqfxData stuUploadExcelUser = cachedDataList.get(0); + + // 获取StuUploadExcelUser类的所有字段（包括私有字段） + Field[] fields = stuUploadExcelUser.getClass().getDeclaredFields(); + + // 遍历每个字段 + for (Field field : fields) { + // 设置字段可访问（如果它是私有的） + field.setAccessible(true); + + + try { + // 获取字段的值 + Object value = field.get(stuUploadExcelUser); + if (value==null) + { + continue; + } + map.put(field.getName(),(String) value); + + } catch (IllegalAccessException e) { + // 如果字段不可访问（理论上不应该发生，因为我们已经设置了setAccessible(true)） + e.printStackTrace(); + } + } + + List rightList = new ArrayList<>(); + // 遍历Map的entrySet + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (value.contains(",")){ + value = value.replaceAll(",",""); + } + try { + // 尝试将字符串转换为 int + int intValue = Integer.parseInt(value); + rightList.add(key); + } catch (NumberFormatException e1) { + try { + // 如果不能转换为 int，尝试将字符串转换为 double + double doubleValue = Double.parseDouble(value); + rightList.add(key); + } catch (NumberFormatException e2) { + // 如果都不能转换，忽略这个字符串 + } + } + + } + + String jsonString = JSON.toJSONString(rightList); + + cachedDataList.forEach(item->{ + item.setUserId(userId); + item.setModule(name); +// item.setStepOneA(JSON.toJSONString(list)); +// item.setStepElevenC(jsonString); + }); + +// demoDAO.insertBatch(cachedDataList); + + + //log.info("存储数据库成功！"); + } +} \ No newline at end of file diff --git a/src/main/resources/mappers/HyxqfxStuAnswerMapper.xml b/src/main/resources/mappers/HyxqfxStuAnswerMapper.xml index fa98693..6ad0c87 100644 --- a/src/main/resources/mappers/HyxqfxStuAnswerMapper.xml +++ b/src/main/resources/mappers/HyxqfxStuAnswerMapper.xml @@ -6,6 +6,7 @@ + @@ -69,7 +70,7 @@ - id, module, time, user_id + id, module, time, user_id, error_number answer @@ -124,9 +125,11 @@ insert into hyxqfx_stu_answer (id, module, time, - user_id, answer) + user_id, error_number, answer + ) values (#{id,jdbcType=VARCHAR}, #{module,jdbcType=VARCHAR}, #{time,jdbcType=VARCHAR}, - #{userId,jdbcType=VARCHAR}, #{answer,jdbcType=LONGVARCHAR}) + #{userId,jdbcType=VARCHAR}, #{errorNumber,jdbcType=INTEGER}, #{answer,jdbcType=LONGVARCHAR} + ) insert into hyxqfx_stu_answer @@ -143,6 +146,9 @@ user_id, + + error_number, + answer, @@ -160,6 +166,9 @@ #{userId,jdbcType=VARCHAR}, + + #{errorNumber,jdbcType=INTEGER}, + #{answer,jdbcType=LONGVARCHAR}, @@ -186,6 +195,9 @@ user_id = #{record.userId,jdbcType=VARCHAR}, + + error_number = #{record.errorNumber,jdbcType=INTEGER}, + answer = #{record.answer,jdbcType=LONGVARCHAR}, @@ -200,6 +212,7 @@ module = #{record.module,jdbcType=VARCHAR}, time = #{record.time,jdbcType=VARCHAR}, user_id = #{record.userId,jdbcType=VARCHAR}, + error_number = #{record.errorNumber,jdbcType=INTEGER}, answer = #{record.answer,jdbcType=LONGVARCHAR} @@ -210,7 +223,8 @@ set id = #{record.id,jdbcType=VARCHAR}, module = #{record.module,jdbcType=VARCHAR}, time = #{record.time,jdbcType=VARCHAR}, - user_id = #{record.userId,jdbcType=VARCHAR} + user_id = #{record.userId,jdbcType=VARCHAR}, + error_number = #{record.errorNumber,jdbcType=INTEGER} @@ -227,6 +241,9 @@ user_id = #{userId,jdbcType=VARCHAR}, + + error_number = #{errorNumber,jdbcType=INTEGER}, + answer = #{answer,jdbcType=LONGVARCHAR}, @@ -238,6 +255,7 @@ set module = #{module,jdbcType=VARCHAR}, time = #{time,jdbcType=VARCHAR}, user_id = #{userId,jdbcType=VARCHAR}, + error_number = #{errorNumber,jdbcType=INTEGER}, answer = #{answer,jdbcType=LONGVARCHAR} where id = #{id,jdbcType=VARCHAR} @@ -245,7 +263,8 @@ update hyxqfx_stu_answer set module = #{module,jdbcType=VARCHAR}, time = #{time,jdbcType=VARCHAR}, - user_id = #{userId,jdbcType=VARCHAR} + user_id = #{userId,jdbcType=VARCHAR}, + error_number = #{errorNumber,jdbcType=INTEGER} where id = #{id,jdbcType=VARCHAR} \ No newline at end of file