chenyuan1
/
dianshang-houduan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

完成年度、月度数据查询、数据下载、蛋糕指数答案回显、分析回显、实训提交、重新实训接口开发

Browse Source
bugfix/ai-assessment-evidence-rescore
whb 2 years ago
parent 4d7d251c2f
commit 20de6611fd
45 changed files with 3386 additions and 10 deletions
Show Stats Download Patch File Download Diff File

15
pom.xml
Unescape Escape View File

@ -244,14 +244,19 @@
<!-- 情感分析--> <!-- 情感分析-->
<!-- <dependency>--> <dependency>
<!-- <groupId>com.hankcs</groupId>--> <groupId>com.hankcs</groupId>
<!-- <artifactId>hanlp</artifactId>--> <artifactId>hanlp</artifactId>
<!-- <version>portable-1.8.4</version>--> <version>portable-1.8.4</version>
<!-- </dependency>--> </dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.9.3</version> <!-- 使用最新版本 -->
</dependency>
<dependency> <dependency>

175
src/main/java/com/sztzjy/linkCommerce/controller/stu/IndustryDemandAnalysisController.java
Unescape Escape View File

@ -0,0 +1,175 @@
package com.sztzjy.linkCommerce.controller.stu;
import cn.hutool.core.util.IdUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.fastjson.JSONObject;
import com.nimbusds.jose.shaded.gson.Gson;
import com.sztzjy.linkCommerce.annotation.AnonymousAccess;
import com.sztzjy.linkCommerce.entity.*;
import com.sztzjy.linkCommerce.entity.dto.MonthlyCakeIndex;
import com.sztzjy.linkCommerce.entity.dto.StuIndustryAnalysisDTO;
import com.sztzjy.linkCommerce.entity.dto.YearCakeIndexDto;
import com.sztzjy.linkCommerce.mapper.HyxqfxAnswerMapper;
import com.sztzjy.linkCommerce.mapper.HyxqfxDataMapper;
import com.sztzjy.linkCommerce.mapper.HyxqfxStuAnswerMapper;
import com.sztzjy.linkCommerce.service.IndustryDemandAnalysisService;
import com.sztzjy.linkCommerce.util.ResultEntity;
import com.sztzjy.linkCommerce.util.excel.DataListener;
import com.sztzjy.linkCommerce.util.excel.DemoDataListener;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.checkerframework.checker.units.qual.A;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.*;
/**
* @author 17803
* @date 2025-04-24 13:30
*/
@Api(tags = "行业需求分析")
@RestController
@RequestMapping("api/industryAnalysis")
public class IndustryDemandAnalysisController {
@Autowired
private IndustryDemandAnalysisService service;
@Autowired
private HyxqfxDataMapper hyxqfxDataMapper;
@Autowired
private HyxqfxStuAnswerMapper hyxqfxStuAnswerMapper;
//数据查询
@ApiOperation("行业需求分析提交")
@PostMapping("/analysisSubmit")
@AnonymousAccess
@Transactional(rollbackFor = Exception.class)
public ResultEntity uploadExcel(@RequestBody StuIndustryAnalysisDTO analysisDTO) {
//查询数据
HyxqfxStuAnswerExample example = new HyxqfxStuAnswerExample();
example.createCriteria().andUserIdEqualTo(analysisDTO.getUserId());
List<HyxqfxStuAnswer> hyxqfxStuAnswerList = hyxqfxStuAnswerMapper.selectByExample(example);
if (!CollectionUtils.isEmpty(hyxqfxStuAnswerList)) {
return new ResultEntity<>(HttpStatus.OK);
}
//读取文件内容 批量导入excel中 以文件名区分\
Map<String, Double> yearMap = new HashMap<>();
yearMap.put("2018年",0.28);
yearMap.put("2019年",0.25);
yearMap.put("2020年",0.24);
yearMap.put("2021年",0.22);
yearMap.put("2022年",0.21);
yearMap.put("2023年",0.22);
yearMap.put("2024年",0.20);
Map<String, Double> monthMap = new HashMap<>();
monthMap.put("2024年4月",0.21);
monthMap.put("2024年5月",0.20);
monthMap.put("2024年6月",0.20);
monthMap.put("2024年7月",0.22);
monthMap.put("2024年8月",0.33);
monthMap.put("2024年9月",0.30);
monthMap.put("2024年10月",0.33);
monthMap.put("2024年11月",0.23);
monthMap.put("2024年12月",0.22);
monthMap.put("2025年1月",0.22);
monthMap.put("2025年2月",0.15);
monthMap.put("2025年3月",0.27);
int count = 0;
List<YearCakeIndexDto> yearCakeIndexDtoList = analysisDTO.getYearCakeIndexDtoList();
for (YearCakeIndexDto yearCakeIndexDto : yearCakeIndexDtoList) {
Double v = yearMap.get(yearCakeIndexDto.getYear());
if (yearCakeIndexDto.getYearAnswer() == null || v != yearCakeIndexDto.getYearAnswer()) {
count ++;
}
}
//分别将两个字段转为json格式
List<MonthlyCakeIndex> monthlyCakeIndexList = analysisDTO.getMonthlyCakeIndexList();
for (MonthlyCakeIndex yearCakeIndexDto : monthlyCakeIndexList) {
Double v = monthMap.get(yearCakeIndexDto.getYear());
if (yearCakeIndexDto.getMonthAnswer() == null || v != yearCakeIndexDto.getMonthAnswer()) {
count ++;
}
}
String jsonString = JSONObject.toJSONString(analysisDTO);
HyxqfxStuAnswer hyxqfxStuAnswer = new HyxqfxStuAnswer();
hyxqfxStuAnswer.setId(IdUtil.randomUUID());
hyxqfxStuAnswer.setUserId(analysisDTO.getUserId());
hyxqfxStuAnswer.setAnswer(jsonString);
hyxqfxStuAnswer.setErrorNumber(count);
hyxqfxStuAnswerMapper.insertSelective(hyxqfxStuAnswer);
//插入主观题
SubjectiveQuestions subjectiveQuestions = new SubjectiveQuestions();
subjectiveQuestions.setId(IdUtil.randomUUID());
subjectiveQuestions.setUserId(analysisDTO.getUserId());
subjectiveQuestions.setModule("行业需求分析");
subjectiveQuestions.setProject("市场需求挖掘");
subjectiveQuestions.setCreateTime(new Date());
subjectiveQuestions.setSubjectiveContent(analysisDTO.getContext());
subjectiveQuestions.setSubjectiveContent(analysisDTO.getContext());
return new ResultEntity<>(HttpStatus.OK, "上传成功");
}
@ApiOperation("行业需求分析根据用户ID查询")
@GetMapping("/selectByUserId")
@AnonymousAccess
public ResultEntity selectByUserId(@ApiParam("用户ID") String userId) {
//读取文件内容 批量导入excel中 以文件名区分\
HyxqfxStuAnswerExample hyxqfxStuAnswer = new HyxqfxStuAnswerExample();
hyxqfxStuAnswer.createCriteria().andUserIdEqualTo(userId);
List<HyxqfxStuAnswer> hyxqfxStuAnswersList = hyxqfxStuAnswerMapper.selectByExampleWithBLOBs(hyxqfxStuAnswer);
if (CollectionUtils.isEmpty(hyxqfxStuAnswersList)) {
return new ResultEntity<>(HttpStatus.OK);
}else {
JSONObject jsonObject = JSONObject.parseObject(hyxqfxStuAnswersList.get(0).getAnswer());
return new ResultEntity<>(HttpStatus.OK,jsonObject);
}
}
public void jiaoYanCodeSize(String code){
String[] strings = code.split(",");
if (strings.length>=20)
{
//计分规则
}
}
}

317
src/main/java/com/sztzjy/linkCommerce/controller/stu/StuDigitalMarketingModelController.java
Unescape Escape View File

@ -0,0 +1,317 @@
package com.sztzjy.linkCommerce.controller.stu;
import com.sztzjy.linkCommerce.annotation.AnonymousAccess;
import com.sztzjy.linkCommerce.entity.*;
import com.sztzjy.linkCommerce.entity.dto.*;
import com.sztzjy.linkCommerce.mapper.StuUserCommentMapper;
import com.sztzjy.linkCommerce.service.StuDigitalMarketingModelService;
import com.sztzjy.linkCommerce.util.BigDecimalUtils;
import com.sztzjy.linkCommerce.util.ResultEntity;
import com.sztzjy.linkCommerce.util.algorithm.Apriori;
import com.sztzjy.linkCommerce.util.algorithm.KMeans;
import com.sztzjy.linkCommerce.util.algorithm.LogisticRegression;
import com.sztzjy.linkCommerce.util.excel.ImportExcelUtil;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.*;
import static com.sztzjy.linkCommerce.util.algorithm.KMeans.*;
import static com.sztzjy.linkCommerce.util.algorithm.KMeansResult.*;
/**
* @author tz
* @date 2024/6/13 10:43
*/
@Api(tags = "数字营销模型")
@RequestMapping("api/model")
@RestController
public class StuDigitalMarketingModelController {
@Value("${file.path}")
private String filePath;
@Resource
StuDigitalMarketingModelService modelService;
@Resource
StuUserCommentMapper stuUserCommentMapper;
@ApiOperation("选择指标--下拉框")
@PostMapping("/dropdownBox")
@AnonymousAccess
public ResultEntity dropdownBox(String userId) {
List<String> list=modelService.dropdownBox(userId);
return new ResultEntity(HttpStatus.OK,"成功",list);
}
@ApiOperation("选择指标--查看指标")
@PostMapping("/viewMetrics")
@AnonymousAccess
public ResultEntity viewMetrics(@ApiParam("用户ID") String userId,
@ApiParam("表格名") String tableName,
@ApiParam("算法名称") String algorithmName) {
List<String> list=modelService.viewMetrics(userId,tableName,algorithmName);
return new ResultEntity(HttpStatus.OK,"成功",list);
}
@ApiOperation("选择指标--分析数据展示")
@PostMapping("/viewAnalyzeData")
@AnonymousAccess
public ResultEntity viewAnalyzeData(@RequestBody AnalyzeDataDTO analyzeDataDTO) {
return modelService.viewAnalyzeData(analyzeDataDTO);
}
@ApiOperation("数据预处理")
@PostMapping("/dataPreprocessing")
@AnonymousAccess
public ResultEntity dataPreprocessing(@RequestBody DataPreprocessingDTO dto) {
String userId = dto.getUserId();
String method = dto.getMethod();
List<Map<String, Object>> mapList = dto.getMapList();
return modelService.dataPreprocessing(userId,method,mapList);
}
@ApiOperation("描述性统计")
@PostMapping("/descriptiveStatistics")
@AnonymousAccess
public ResultEntity descriptiveStatistics(@ApiParam("数据集、需要计算的方式、用户ID")@RequestBody StatisticsDTO statisticsDTO) {
Map<String,List<Double>> map = statisticsDTO.getMap();
List<String> statistic = statisticsDTO.getStatistic();
String userId = statisticsDTO.getUserId();
return modelService.descriptiveStatistics(map,statistic,userId);
}
@ApiOperation("聚类分析--散点图")
@PostMapping("/clusterScatterPlot")
@AnonymousAccess
public ResultEntity clusterScatterPlot(@RequestBody ClusterScatterPlotDTO clusterScatterPlotDTO) {
Integer t = clusterScatterPlotDTO.getT();
Integer k = clusterScatterPlotDTO.getK();
List<Map<String, Object>> deduplicatedDataList = clusterScatterPlotDTO.getDeduplicatedDataList();
List<String> strings = convertToDelimitedStringList(deduplicatedDataList);
List<KMeans.Point> irisData = readIrisData(strings);
//获取数据集
KMeans kMeans = new KMeans(k, t, irisData);
List<List<KMeans.Point>> clusters = kMeans.cluster();
//封装成map
//数据集
Map<Integer,List<KMeans.Point>> dataSetMap=new HashMap<>();
for (int i = 0; i < clusters.size(); i++) {
dataSetMap.put(i,clusters.get(i));
}
//质心
Map<Integer,List<KMeans.Point>> centroidMap=new HashMap<>();
List<KMeans.Point> centroids = kMeans.getCentroids();
for (int i = 0; i < centroids.size(); i++) {
centroidMap.put(i, Collections.singletonList(centroids.get(i)));
}
ClusterAnalysisDTO clusterAnalysisDTO=new ClusterAnalysisDTO();
clusterAnalysisDTO.setDataSet(dataSetMap);
clusterAnalysisDTO.setCentroid(centroidMap);
return new ResultEntity(HttpStatus.OK,clusterAnalysisDTO);
}
@ApiOperation("聚类分析--分析结果")
@PostMapping("/clusterAnalysisResult")
@AnonymousAccess
public ResultEntity clusterAnalysisResult(@RequestBody ClusterScatterPlotDTO clusterScatterPlotDTO) {
Integer t = clusterScatterPlotDTO.getT();
Integer k = clusterScatterPlotDTO.getK();
List<Map<String, Object>> deduplicatedDataList = clusterScatterPlotDTO.getDeduplicatedDataList();
if(k>deduplicatedDataList.size()){
return new ResultEntity(HttpStatus.ACCEPTED,"簇数K值不能大于数据条数");
}
List<String> strings = convertToDelimitedStringList(deduplicatedDataList);
//初始化数据
ArrayList<ArrayList<Float>> arrayLists = readTable(strings);
//随机选择 k 个数据点作为初始聚类中心
ArrayList<ArrayList<Float>> centerList = randomList(k, arrayLists);
//开始迭代
KmeansDTO kmeansDTO = kmeans(k, t, centerList);
return new ResultEntity(HttpStatus.OK,kmeansDTO);
}
@ApiOperation("关联规则挖掘")
@PostMapping("/apriori")
@AnonymousAccess
public ResultEntity apriori(@RequestBody AprioriDTO aprioriDTO) throws IOException {
double support = aprioriDTO.getSupport();
double confidence = aprioriDTO.getConfidence();
String userId = aprioriDTO.getUserId();
List<Map<String, Object>> deduplicatedDataList = aprioriDTO.getDeduplicatedDataList();
List<String> strings = convertToDelimitedStringList(deduplicatedDataList);
//初始化事务数据库、项目集、候选集再进行剪枝
Apriori.init(strings,confidence);
//迭代求出最终的候选频繁集
Apriori.iteration(Apriori.C,Apriori.L,confidence);
//根据最终的关联集，根据公式计算出各个关联事件
List<AssociationRulesDTO> connection = Apriori.connection(support);
return new ResultEntity(HttpStatus.OK,"成功",connection);
}
// @ApiOperation("回归分析--线性回归")
// @PostMapping("/regressionAnalysis")
// @AnonymousAccess
// public ResultEntity regressionAnalysis(@RequestBody RegressionAnalysisDTO analysisDTO) {
// double[][] x = analysisDTO.getX();
// double[] y = analysisDTO.getY();
//
// // 创建线性回归模型
// LinearRegression regression=new LinearRegression();
// regression.fit(x,y);
//
//
// RAnalysisDTO rAnalysisDTO=new RAnalysisDTO();
// rAnalysisDTO.setIntercept(regression.getIntercept());
// rAnalysisDTO.setSlopes(regression.getSlopes());
// rAnalysisDTO.setType("线性回归");
//
// return new ResultEntity(HttpStatus.OK,"成功",rAnalysisDTO);
// }
@ApiOperation("回归分析--逻辑回归")
@PostMapping("/logistic")
@AnonymousAccess
public ResultEntity logistic(@RequestBody LogisticDTO logisticDTO) {
double[] x = logisticDTO.getX();
double[] y = logisticDTO.getY();
LogisticRegression lr = new LogisticRegression(x, y);
double[] coefficients = lr.fit();
DecimalFormat df = new DecimalFormat("#.00");
RAnalysisDTO rAnalysisDTO=new RAnalysisDTO();
rAnalysisDTO.setIntercept(Double.parseDouble(df.format(coefficients[0])));
rAnalysisDTO.setSlope(Double.parseDouble(df.format(coefficients[1])));
return new ResultEntity(HttpStatus.OK,"成功",rAnalysisDTO);
}
@ApiOperation("回归分析--走势图")
@PostMapping("/trendChart")
@AnonymousAccess
public ResultEntity trendChart(@RequestBody LogisticDTO logisticDTO) {
double[] x = logisticDTO.getX();
double[] y = logisticDTO.getY();
LogisticRegression lr = new LogisticRegression(x, y);
double[][] coefficients = lr.getCoordinatesOnLine();
return new ResultEntity(HttpStatus.OK,"成功",coefficients);
}
@ApiOperation("线性回归/逻辑回归--预测")
@PostMapping("/prediction")
@AnonymousAccess
public ResultEntity prediction(@RequestBody LogisticPredictDTO logisticPredictDTO) {
DecimalFormat df = new DecimalFormat("#.0");
BigDecimalUtils bigDecimalUtils=new BigDecimalUtils();
double predict = bigDecimalUtils.add(bigDecimalUtils.mul(logisticPredictDTO.getSlope(),logisticPredictDTO.getX()),logisticPredictDTO.getIntercept());
return new ResultEntity(HttpStatus.OK,"成功",df.format(predict));
}
@ApiOperation("情感分析/文本挖掘")
@PostMapping("/emotionalAnalysis")
@AnonymousAccess
public ResultEntity emotionalAnalysis(@RequestBody EmotionalAnalysisDto emotionalAnalysisDto) throws IOException {
String id = emotionalAnalysisDto.getId();
String modelType = emotionalAnalysisDto.getModelType();
String content = emotionalAnalysisDto.getContent();
String userId = emotionalAnalysisDto.getUserId();
return modelService.emotionalAnalysis(userId,modelType,content,id);
}
@ApiOperation("批量导入")
@PostMapping("/batchImport")
@AnonymousAccess
public ResultEntity batchImport(@RequestParam(required = false) @RequestPart MultipartFile file) {
//验证文件类型
if (!file.getOriginalFilename().substring(file.getOriginalFilename().lastIndexOf(".")).equals(".xls") && !file.getOriginalFilename().substring(file.getOriginalFilename().lastIndexOf(".")).equals(".xlsx")) {
return new ResultEntity<>(HttpStatus.ACCEPTED, "文件类型有误！请上传Excel文件");
}
try {
List<StuUserComment> stuUserComments=new ArrayList<>();
//获取数据
List<List<Object>> listByExcel = ImportExcelUtil.getListByExcel(file.getInputStream(), file.getOriginalFilename());
//封装数据
for (int i = 0; i < listByExcel.size(); i++) {
List<Object> list = listByExcel.get(i);
if (list.get(0) == "" || ("序号").equals(list.get(0))) {
continue;
}
StuUserComment stuUserComment=new StuUserComment();
Integer uuid = UUID.randomUUID().toString().replaceAll("-", "").hashCode();
uuid = uuid < 0 ? -uuid : uuid;//String.hashCode() 值会为空
stuUserComment.setId(uuid);
stuUserComment.setContent(String.valueOf(list.get(0)));
stuUserComment.setCreationTime(new Date(String.valueOf(list.get(1))));
stuUserComment.setNickName(String.valueOf((list.get(2))));
stuUserComment.setReferenceName(String.valueOf(list.get(3)));
stuUserComment.setContentType(String.valueOf(list.get(4)));
stuUserComments.add(stuUserComment);
}
stuUserCommentMapper.addList(stuUserComments);
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
file.getInputStream().close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return new ResultEntity(HttpStatus.OK,"成功");
}
}

11
src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswer.java
Unescape Escape View File

@ -20,6 +20,9 @@ public class HyxqfxStuAnswer {
@ApiModelProperty(notes = "用户ID（关联userinfo表）") @ApiModelProperty(notes = "用户ID（关联userinfo表）")
private String userId; private String userId;
@ApiModelProperty(notes = "错误次数")
private Integer errorNumber;
@ApiModelProperty(notes = "学生答案内容（富文本格式）") @ApiModelProperty(notes = "学生答案内容（富文本格式）")
private String answer; private String answer;
@ -55,6 +58,14 @@ public class HyxqfxStuAnswer {
this.userId = userId == null ? null : userId.trim(); this.userId = userId == null ? null : userId.trim();
} }
public Integer getErrorNumber() {
return errorNumber;
}
public void setErrorNumber(Integer errorNumber) {
this.errorNumber = errorNumber;
}
public String getAnswer() { public String getAnswer() {
return answer; return answer;
} }

60
src/main/java/com/sztzjy/linkCommerce/entity/HyxqfxStuAnswerExample.java
Unescape Escape View File

@ -383,6 +383,66 @@ public class HyxqfxStuAnswerExample {
addCriterion("user_id not between", value1, value2, "userId"); addCriterion("user_id not between", value1, value2, "userId");
return (Criteria) this; return (Criteria) this;
} }
public Criteria andErrorNumberIsNull() {
addCriterion("error_number is null");
return (Criteria) this;
}
public Criteria andErrorNumberIsNotNull() {
addCriterion("error_number is not null");
return (Criteria) this;
}
public Criteria andErrorNumberEqualTo(Integer value) {
addCriterion("error_number =", value, "errorNumber");
return (Criteria) this;
}
public Criteria andErrorNumberNotEqualTo(Integer value) {
addCriterion("error_number <>", value, "errorNumber");
return (Criteria) this;
}
public Criteria andErrorNumberGreaterThan(Integer value) {
addCriterion("error_number >", value, "errorNumber");
return (Criteria) this;
}
public Criteria andErrorNumberGreaterThanOrEqualTo(Integer value) {
addCriterion("error_number >=", value, "errorNumber");
return (Criteria) this;
}
public Criteria andErrorNumberLessThan(Integer value) {
addCriterion("error_number <", value, "errorNumber");
return (Criteria) this;
}
public Criteria andErrorNumberLessThanOrEqualTo(Integer value) {
addCriterion("error_number <=", value, "errorNumber");
return (Criteria) this;
}
public Criteria andErrorNumberIn(List<Integer> values) {
addCriterion("error_number in", values, "errorNumber");
return (Criteria) this;
}
public Criteria andErrorNumberNotIn(List<Integer> values) {
addCriterion("error_number not in", values, "errorNumber");
return (Criteria) this;
}
public Criteria andErrorNumberBetween(Integer value1, Integer value2) {
addCriterion("error_number between", value1, value2, "errorNumber");
return (Criteria) this;
}
public Criteria andErrorNumberNotBetween(Integer value1, Integer value2) {
addCriterion("error_number not between", value1, value2, "errorNumber");
return (Criteria) this;
}
} }
public static class Criteria extends GeneratedCriteria { public static class Criteria extends GeneratedCriteria {

16
src/main/java/com/sztzjy/linkCommerce/entity/dto/AnalyzeDataDTO.java
Unescape Escape View File

@ -0,0 +1,16 @@
package com.sztzjy.linkCommerce.entity.dto;
import lombok.Data;
import java.util.List;
/**
* @author tz
* @date 2024/8/16 15:26
*/
@Data
public class AnalyzeDataDTO {
private String userId;
private String tableName;
private List<String> fieldList;
}

23
src/main/java/com/sztzjy/linkCommerce/entity/dto/AprioriDTO.java
Unescape Escape View File

@ -0,0 +1,23 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* @author tz
* @date 2024/8/17 14:16
*/
@Data
public class AprioriDTO {
@ApiModelProperty("最小支持度阀值")
private double support;
@ApiModelProperty("最小置信度")
private double confidence;
@ApiModelProperty("用户ID")
private String userId;
@ApiModelProperty("数据集")
private List<Map<String, Object>> deduplicatedDataList;
}

25
src/main/java/com/sztzjy/linkCommerce/entity/dto/AssociationRulesDTO.java
Unescape Escape View File

@ -0,0 +1,25 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
/**
* @author tz
* @date 2024/6/20 8:51
*/
@Data
public class AssociationRulesDTO {
@ApiModelProperty("关联")
private List<String> correlation;
@ApiModelProperty("被关联")
private List<String> associated;
@ApiModelProperty("置信度")
private double confidenceLevel;
@ApiModelProperty("规则")
private String rule;
}

18
src/main/java/com/sztzjy/linkCommerce/entity/dto/ClusterAnalysisDTO.java
Unescape Escape View File

@ -0,0 +1,18 @@
package com.sztzjy.linkCommerce.entity.dto;
import com.sztzjy.linkCommerce.util.algorithm.KMeans;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* @author tz
* @date 2024/6/14 10:12
*/
@Data
public class ClusterAnalysisDTO {
private Map<Integer, List<KMeans.Point>> dataSet;
private Map<Integer,List<KMeans.Point>> centroid;
}

23
src/main/java/com/sztzjy/linkCommerce/entity/dto/ClusterScatterPlotDTO.java
Unescape Escape View File

@ -0,0 +1,23 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* @author tz
* @date 2024/8/16 16:26
*/
@Data
public class ClusterScatterPlotDTO {
@ApiModelProperty("簇数")
private Integer k;
@ApiModelProperty("最大迭代次数")
private Integer t;
@ApiModelProperty("用户ID")
private String userId;
@ApiModelProperty("数据集")
private List<Map<String, Object>> deduplicatedDataList;
}

18
src/main/java/com/sztzjy/linkCommerce/entity/dto/DSExportDTO.java
Unescape Escape View File

@ -0,0 +1,18 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author tz
* @date 2024/8/19 10:33
*/
@Data
@Api(tags = "描述性统计下载表格数据")
public class DSExportDTO extends DescriptiveStatistics{
@ApiModelProperty("字段名")
private String field;
}

19
src/main/java/com/sztzjy/linkCommerce/entity/dto/DataPreprocessingDTO.java
Unescape Escape View File

@ -0,0 +1,19 @@
package com.sztzjy.linkCommerce.entity.dto;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* @author tz
* @date 2024/6/25 14:21
*/
@Data
public class DataPreprocessingDTO {
private String userId;
private String method;
private List<Map<String,Object>> mapList;
}

50
src/main/java/com/sztzjy/linkCommerce/entity/dto/DescriptiveStatistics.java
Unescape Escape View File

@ -0,0 +1,50 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author tz
* @date 2024/6/21 9:51
*/
@Data
public class DescriptiveStatistics {
@ApiModelProperty("平均数")
private double average;
@ApiModelProperty("中位数")
private double median;
@ApiModelProperty("众数")
private double mode;
@ApiModelProperty("标准差")
private double standardDeviation;
@ApiModelProperty("方差")
private double variance;
@ApiModelProperty("标准误差")
private double standardError;
@ApiModelProperty("峰度")
private double kurtosis;
@ApiModelProperty("偏度")
private double skewness;
@ApiModelProperty("最大值")
private double max;
@ApiModelProperty("最小值")
private double min;
@ApiModelProperty("求和")
private double summation;
@ApiModelProperty("观测数")
private Integer observations;
@ApiModelProperty("字段名")
private String field;
}

15
src/main/java/com/sztzjy/linkCommerce/entity/dto/DescriptiveStatisticsDTO.java
Unescape Escape View File

@ -0,0 +1,15 @@
package com.sztzjy.linkCommerce.entity.dto;
import lombok.Data;
/**
* @author tz
* @date 2024/6/21 10:00
*/
@Data
public class DescriptiveStatisticsDTO {
private String field;
private DescriptiveStatistics statistics;
}

19
src/main/java/com/sztzjy/linkCommerce/entity/dto/EmotionalAnalysisDto.java
Unescape Escape View File

@ -0,0 +1,19 @@
package com.sztzjy.linkCommerce.entity.dto;
import lombok.Data;
import javax.xml.bind.annotation.XmlElement;
/**
* @author tz
* @date 2024/8/19 16:09
*/
@Data
public class EmotionalAnalysisDto {
private String userId;
private String modelType;
private String content;
@XmlElement(name = "optionalField", required = false)
private String id;
}

17
src/main/java/com/sztzjy/linkCommerce/entity/dto/KmeansDTO.java
Unescape Escape View File

@ -0,0 +1,17 @@
package com.sztzjy.linkCommerce.entity.dto;
import lombok.Data;
import java.util.ArrayList;
/**
* @author tz
* @date 2024/8/28 9:09
*/
@Data
public class KmeansDTO {
private ArrayList<ArrayList<ArrayList<Float>>> clusters;
private Integer iterations;
}

21
src/main/java/com/sztzjy/linkCommerce/entity/dto/LogisticDTO.java
Unescape Escape View File

@ -0,0 +1,21 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author tz
* @date 2024/7/23 16:52
*/
@Data
public class LogisticDTO {
@ApiModelProperty("自变量x")
private double[] x;
@ApiModelProperty("因变量y")
private double[] y;
@ApiModelProperty("用户ID")
private String userId;
}

16
src/main/java/com/sztzjy/linkCommerce/entity/dto/LogisticPredictDTO.java
Unescape Escape View File

@ -0,0 +1,16 @@
package com.sztzjy.linkCommerce.entity.dto;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* @author tz
* @date 2024/8/19 9:38
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class LogisticPredictDTO extends RAnalysisDTO {
private double x;
}

19
src/main/java/com/sztzjy/linkCommerce/entity/dto/MonthlyCakeIndex.java
Unescape Escape View File

@ -0,0 +1,19 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author 17803
* @date 2025-04-24 14:22
*/
@Data
public class MonthlyCakeIndex {
@ApiModelProperty(notes = "月份")
private String year;
@ApiModelProperty(notes = "月度蛋糕指数")
private Double monthAnswer;
}

48
src/main/java/com/sztzjy/linkCommerce/entity/dto/NewDescriptiveStatisticsDTO.java
Unescape Escape View File

@ -0,0 +1,48 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author tz
* @date 2024/11/26 13:42
*/
//新版描述性统计分析返回数据格式
@Data
public class NewDescriptiveStatisticsDTO {
@ApiModelProperty("平均数")
private double average;
@ApiModelProperty("中位数")
private double median;
@ApiModelProperty("众数")
private double mode;
@ApiModelProperty("标准差")
private double standardDeviation;
@ApiModelProperty("方差")
private double variance;
@ApiModelProperty("标准误差")
private double standardError;
@ApiModelProperty("峰度")
private double kurtosis;
@ApiModelProperty("偏度")
private double skewness;
@ApiModelProperty("最大值")
private double max;
@ApiModelProperty("最小值")
private double min;
@ApiModelProperty("求和")
private double summation;
@ApiModelProperty("观测数")
private Integer observations;
}

16
src/main/java/com/sztzjy/linkCommerce/entity/dto/RAnalysisDTO.java
Unescape Escape View File

@ -0,0 +1,16 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author tz
* @date 2024/8/17 16:30
*/
@Data
public class RAnalysisDTO {
@ApiModelProperty("回归系数值")
private double intercept;
@ApiModelProperty("逻辑回归常数项值")
private double slope;
}

16
src/main/java/com/sztzjy/linkCommerce/entity/dto/RAnalysisExportDTO.java
Unescape Escape View File

@ -0,0 +1,16 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* @author tz
* @date 2024/8/19 14:55
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class RAnalysisExportDTO extends RAnalysisDTO{
@ApiModelProperty("预测值")
private double prediction;
}

20
src/main/java/com/sztzjy/linkCommerce/entity/dto/RegressionAnalysisDTO.java
Unescape Escape View File

@ -0,0 +1,20 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author tz
* @date 2024/7/19 13:56
*/
@Data
public class RegressionAnalysisDTO {
@ApiModelProperty("自变量x")
private double[][] x;
@ApiModelProperty("因变量y")
private double[] y;
@ApiModelProperty("用户ID")
private String userId;
}

17
src/main/java/com/sztzjy/linkCommerce/entity/dto/StatisticsDTO.java
Unescape Escape View File

@ -0,0 +1,17 @@
package com.sztzjy.linkCommerce.entity.dto;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* @author tz
* @date 2024/6/21 10:57
*/
@Data
public class StatisticsDTO {
private Map<String, List<Double>> map;
private List<String> statistic;
private String userId;
}

30
src/main/java/com/sztzjy/linkCommerce/entity/dto/StuIndustryAnalysisDTO.java
Unescape Escape View File

@ -0,0 +1,30 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
/**
* @author 17803
* @date 2025-04-24 14:12
*/
//
@Data
public class StuIndustryAnalysisDTO {
@ApiModelProperty(notes = "年度蛋糕指数")
private List<YearCakeIndexDto> yearCakeIndexDtoList;
@ApiModelProperty(notes = "月度蛋糕指数")
private List<MonthlyCakeIndex> monthlyCakeIndexList;
@ApiModelProperty(notes = "用户ID")
private String userId;
@ApiModelProperty(notes = "主观题")
private String context;
}

14
src/main/java/com/sztzjy/linkCommerce/entity/dto/WordFrequencyDTO.java
Unescape Escape View File

@ -0,0 +1,14 @@
package com.sztzjy.linkCommerce.entity.dto;
import lombok.Data;
/**
* @author tz
* @date 2024/6/18 16:48
*/
@Data
public class WordFrequencyDTO {
private String keyword;
private Integer frequency;
}

21
src/main/java/com/sztzjy/linkCommerce/entity/dto/YearCakeIndexDto.java
Unescape Escape View File

@ -0,0 +1,21 @@
package com.sztzjy.linkCommerce.entity.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
/**
* @author 17803
* @date 2025-04-24 14:22
*/
@Data
public class YearCakeIndexDto {
@ApiModelProperty(notes = "月份")
private String year;
@ApiModelProperty(notes = "童装童鞋类目年度蛋糕指数")
private Double yearAnswer;
}

8
src/main/java/com/sztzjy/linkCommerce/service/IndustryDemandAnalysisService.java
Unescape Escape View File

@ -0,0 +1,8 @@
package com.sztzjy.linkCommerce.service;
/**
* @author 17803
* @date 2025-04-24 13:35
*/
public interface IndustryDemandAnalysisService {
}

28
src/main/java/com/sztzjy/linkCommerce/service/StuDigitalMarketingModelService.java
Unescape Escape View File

@ -0,0 +1,28 @@
package com.sztzjy.linkCommerce.service;
import com.sztzjy.linkCommerce.entity.dto.AnalyzeDataDTO;
import com.sztzjy.linkCommerce.util.ResultEntity;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* @author tz
* @date 2024/6/14 11:05
*/
public interface StuDigitalMarketingModelService {
List<String> dropdownBox(String userId);
ResultEntity emotionalAnalysis(String userId, String modelType, String content,String id) throws IOException;
ResultEntity descriptiveStatistics(Map<String, List<Double>> map, List<String> statistic, String userId);
List<String> viewMetrics(String userId, String tableName,String algorithmName);
ResultEntity viewAnalyzeData(AnalyzeDataDTO analyzeDataDTO);
ResultEntity dataPreprocessing(String userId, String method, List<Map<String, Object>> mapList);
}

12
src/main/java/com/sztzjy/linkCommerce/service/impl/IndustryDemandAnalysisServiceImpl.java
Unescape Escape View File

@ -0,0 +1,12 @@
package com.sztzjy.linkCommerce.service.impl;
import com.sztzjy.linkCommerce.service.IndustryDemandAnalysisService;
import org.springframework.stereotype.Service;
/**
* @author 17803
* @date 2025-04-24 13:35
*/
@Service
public class IndustryDemandAnalysisServiceImpl implements IndustryDemandAnalysisService {
}

645
src/main/java/com/sztzjy/linkCommerce/service/impl/StuDigitalMarketingModelServiceImpl.java
Unescape Escape View File

@ -0,0 +1,645 @@
package com.sztzjy.linkCommerce.service.impl;
import com.alibaba.fastjson.JSON;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.seg.common.Term;
import com.sztzjy.linkCommerce.config.Constant;
import com.sztzjy.linkCommerce.entity.StuTableName;
import com.sztzjy.linkCommerce.entity.dto.AnalyzeDataDTO;
import com.sztzjy.linkCommerce.entity.dto.DescriptiveStatistics;
import com.sztzjy.linkCommerce.entity.dto.DescriptiveStatisticsDTO;
import com.sztzjy.linkCommerce.entity.dto.WordFrequencyDTO;
import com.sztzjy.linkCommerce.mapper.StuTableNameMapper;
import com.sztzjy.linkCommerce.mapper.StuUploadExcelUserMapper;
import com.sztzjy.linkCommerce.service.StuDigitalMarketingModelService;
import com.sztzjy.linkCommerce.util.ResultEntity;
import com.sztzjy.linkCommerce.util.algorithm.DescriptiveStatisticsUtil;
import okhttp3.*;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import javax.annotation.Resource;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.*;
import static com.sztzjy.linkCommerce.util.StringUtils.convertCamelCaseToUnderscore;
import static com.sztzjy.linkCommerce.util.algorithm.BaiDuZhiNengYun.getAccessToken;
/**
* @author tz
* @date 2024/6/14 11:05
*/
@Service
public class StuDigitalMarketingModelServiceImpl implements StuDigitalMarketingModelService {
private final OkHttpClient HTTP_CLIENT = new OkHttpClient().newBuilder().build();
@Resource
StuTableNameMapper stuTableNameMapper;
@Resource
StuUploadExcelUserMapper uploadExcelUserMapper;
@Override
public List<String> dropdownBox(String userId) {
List<String> list=stuTableNameMapper.getTableName(userId);
if(list.isEmpty()){ //设置默认值
String[] arr={"客户细分数据表","销售预测数据表","购物车数据表"};
StuTableName stuTableName=new StuTableName();
for (int i = 0; i < arr.length; i++) {
Integer uuid = UUID.randomUUID().toString().replaceAll("-", "").hashCode();
uuid = uuid < 0 ? -uuid : uuid;//String.hashCode() 值会为空
stuTableName.setId(uuid);
stuTableName.setTableName(arr[i]);
stuTableName.setUserId(userId);
stuTableName.setCreateTime(new Date());
stuTableNameMapper.insert(stuTableName);
list.add(arr[i]);
}
}
return list;
}
@Override
public ResultEntity emotionalAnalysis(String userId, String modelType, String content,String id) throws IOException {
MediaType mediaType = MediaType.parse("application/json");
if(modelType.equals(Constant.COMMENT_EXTRACTION)){ //评论观点抽取
String commentUrl = "https://aip.baidubce.com/rpc/2.0/nlp/v2/comment_tag?charset=UTF-8&access_token="+ getAccessToken();
RequestBody body = RequestBody.create(mediaType, "{\"text\":\""+content+"\",\"type\":\""+id+"\"}");
Request request = new Request.Builder()
.url(commentUrl)
.addHeader("Content-Type", "application/json")
.addHeader("Accept", "application/json")
.method("POST", body)
.build();
Response response = HTTP_CLIENT.newCall(request).execute();
String string = response.body().string();
// Gson对象
Gson gson = new Gson();
Map<String, Object> data = gson.fromJson(string, new TypeToken<Map<String, Object>>(){}.getType());
// Object object = data.get("items");
//
//
// if(object instanceof List){
// ArrayList<String> tagList = (ArrayList<String>) object;
// if (tagList.isEmpty()) {
// return new ResultEntity(HttpStatus.ACCEPTED,"请重新选择评论类型");
// }
// }
if(data.containsKey("error_msg")){
return new ResultEntity(HttpStatus.ACCEPTED,"评论异常");
}
if (response.body() != null) {
return new ResultEntity(HttpStatus.OK, "成功", string);
}else {
return new ResultEntity(HttpStatus.ACCEPTED,"评论异常");
}
}
if (modelType.equals(Constant.EMOTIONAL_TENDENCIES)) { //情感倾向分析
String emoUrl = "https://aip.baidubce.com/rpc/2.0/nlp/v1/sentiment_classify?charset=UTF-8&access_token=" + getAccessToken();
RequestBody body = RequestBody.create(mediaType, "{\"text\":\"" + content + "\"}");
Request request = new Request.Builder()
.url(emoUrl)
.method("POST", body)
.build();
Response response = HTTP_CLIENT.newCall(request).execute();
String string = response.body().string();
// Gson对象
Gson gson = new Gson();
Map<String, Object> data = gson.fromJson(string, new TypeToken<Map<String, Object>>(){}.getType());
if(data.containsKey("error_msg")){
return new ResultEntity(HttpStatus.ACCEPTED,"评论异常");
}
if (response.body() != null) {
return new ResultEntity(HttpStatus.OK, "成功", string);
}
}
if(modelType.equals(Constant.WORD_FREQUENCY)){ //词频分析
String string = this.filterPunctuation(content);
// 使用 HanLP 分词
List<Term> terms = HanLP.segment(string);
// 创建词汇表
Map<String, Integer> wordCount = new HashMap<>();
// 读取停用词
Set<String> stopwords = this.loadStopwords("/usr/local/tianzeProject/digitalMarketing/jarAndDockerFile/停用词.txt");
// Set<String> stopwords = this.loadStopwords("D:\\project\\digital_marketing\\src\\main\\resources\\words\\停用词.txt");
// 统计词频
int count = 0;
for (Term term : terms) {
String word = term.word.trim(); // 去除空格
if (!word.isEmpty() && !stopwords.contains(word)) { // 过滤停用词
count++;
if (wordCount.containsKey(word)) {
wordCount.put(word, wordCount.get(word) + 1);
} else {
wordCount.put(word, 1);
}
}
if (count >= 200) { // 达到200个关键词时结束统计
break;
}
}
// 将词汇表转换为列表，便于排序
List<Map.Entry<String, Integer>> wordList = new ArrayList<>(wordCount.entrySet());
// 按词频排序
wordList.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue()));
List<WordFrequencyDTO> list = new ArrayList<>();
for (Map.Entry<String, Integer> entry : wordList) {
WordFrequencyDTO frequencyDTO = new WordFrequencyDTO();
frequencyDTO.setKeyword(entry.getKey());
frequencyDTO.setFrequency(entry.getValue());
list.add(frequencyDTO);
}
return new ResultEntity<>(HttpStatus.OK, "成功", list);
}
// if(modelType.equals(Constant.WORD_CLOUD)){ //词云分析
//
// }
return null;
}
// 过滤标点符号的函数
private String filterPunctuation(String content) {
//标点符号
return content.replaceAll("[，。、？！：;,.?!:'“”’‘]", "");
}
@Override
public ResultEntity descriptiveStatistics(Map<String, List<Double>> map, List<String> statistic, String userId) {
DescriptiveStatisticsUtil statisticsUtil=new DescriptiveStatisticsUtil();
List<DescriptiveStatisticsDTO> dtoList=new ArrayList<>();
for(String key: map.keySet()){
DescriptiveStatisticsDTO statisticsDTO=new DescriptiveStatisticsDTO();
statisticsDTO.setField(key);
DescriptiveStatistics statistics=new DescriptiveStatistics();
DecimalFormat df = new DecimalFormat("#.0");
for (int i = 0; i < statistic.size(); i++) {
if(statistic.get(i).equals("平均数")){
double mean = statisticsUtil.getMean(map.get(key));
statistics.setAverage(Double.parseDouble(df.format(mean)));
}
if(statistic.get(i).equals("中位数")){
double median = statisticsUtil.getMedian(map.get(key));
statistics.setMedian(Double.parseDouble(df.format(median)));
}
if(statistic.get(i).equals("众数")){
Double mode = statisticsUtil.getMode(map.get(key));
statistics.setMode(Double.parseDouble(df.format(mode)));
}
if(statistic.get(i).equals("标准差")){
double standardDeviation = statisticsUtil.getStandardDeviation(map.get(key));
statistics.setStandardDeviation(Double.parseDouble(df.format(standardDeviation)));
}
if(statistic.get(i).equals("方差")){
double variance = statisticsUtil.getVariance(map.get(key));
statistics.setVariance(Double.parseDouble(df.format(variance)));
}
if(statistic.get(i).equals("标准误差")){
double standardError = statisticsUtil.getStandardError(map.get(key));
statistics.setStandardError(Double.parseDouble(df.format(standardError)));
}
if(statistic.get(i).equals("峰度")){
double kurtosis = statisticsUtil.getKurtosis(map.get(key));
if(Double.isNaN(kurtosis)){
statistics.setKurtosis(0);
}else {
statistics.setKurtosis(Double.parseDouble(df.format(kurtosis)));
}
}
if(statistic.get(i).equals("偏度")){
double skewness = statisticsUtil.getSkewness(map.get(key));
if(Double.isNaN(skewness)){
statistics.setSkewness(0);
}else {
statistics.setSkewness(Double.parseDouble(df.format(skewness)));
}
}
if(statistic.get(i).equals("最大值")){
double max = statisticsUtil.getMax(map.get(key));
statistics.setMax(Double.parseDouble(df.format(max)));
}
if(statistic.get(i).equals("最小值")){
double min = statisticsUtil.getMin(map.get(key));
statistics.setMin(Double.parseDouble(df.format(min)));
}
if(statistic.get(i).equals("求和")){
double sum = statisticsUtil.getSum(map.get(key));
statistics.setSummation(Double.parseDouble(df.format(sum)));
}
if(statistic.get(i).equals("观测数")){
int count = statisticsUtil.getCount(map.get(key));
statistics.setObservations(count);
}
}
statisticsDTO.setStatistics(statistics);
dtoList.add(statisticsDTO);
}
return new ResultEntity(HttpStatus.OK,"成功",dtoList);
}
@Override
public List<String> viewMetrics(String userId, String tableName,String algorithmName) {
List<String> list=new ArrayList<>();
if (!StringUtils.hasText(tableName))
{
tableName = "空表";
}
if(tableName.equals(Constant.KEHUXISHUJU)){
// list=indicatorsMapper.getYHSXB();
// list.add("id");
list.add("gender");
list.add("age");
list.add("annual_income");
list.add("spending_score");
}
else if(tableName.equals(Constant.YONGHUXIAOSHOUNLB)){
// list=indicatorsMapper.getYHDLHYB();
list.add("sales_volume");
list.add("sales_forehead");
}
else if(tableName.equals(Constant.GOUWUCHESHUJU)){
// list=indicatorsMapper.getYHXFNLB();
if(algorithmName.equals("关联规则挖掘")){
list.add("consumer_goods");
}
}
else if(tableName.equals(Constant.YHPLB)){
// list=indicatorsMapper.getYHPLB();
if(algorithmName.equals("情感分析")){
list.add("content");
}else {
list.add("id");
}
}else {
//查询导入的表数据
List<String> stringList=new ArrayList<>();
stringList=uploadExcelUserMapper.selectByString(userId,tableName);
//查询默认数据
if(stringList.isEmpty()){
stringList=uploadExcelUserMapper.selectByString("999999999",tableName);
}
for (String s : stringList) {
List<String> rightList = JSON.parseArray(s, String.class);
List<String> strings=new ArrayList<>();
for (int i = 0; i < rightList.size(); i++) { //转为驼峰命名
String string = convertCamelCaseToUnderscore(rightList.get(i));
strings.add(string);
}
if(!list.isEmpty()){
if(list.size()<strings.size()){
list=strings;
}
}else {
list=strings;
}
}
}
return list;
}
@Override
public ResultEntity viewAnalyzeData(AnalyzeDataDTO analyzeDataDTO) {
String table=null;
if(analyzeDataDTO.getTableName().equals(Constant.KEHUXISHUJU)){ //查询客户细分数据表
table="stu_spending_level";
}
else if(analyzeDataDTO.getTableName().equals(Constant.YONGHUXIAOSHOUNLB)){ //查询销售预测数据表
table="stu_user_sales_ability";
}
else if(analyzeDataDTO.getTableName().equals(Constant.GOUWUCHESHUJU)){ //查询购物车数据表
table="stu_user_shopping_cart";
}
else if(analyzeDataDTO.getTableName().equals(Constant.YHPLB)){ //查询用户商品评论数据
table="stu_user_comment";
}else {
table="stu_upload_excel_user"; //查询用户自建表
List<String> fieldList = analyzeDataDTO.getFieldList();
String userId = analyzeDataDTO.getUserId();
String tableName = analyzeDataDTO.getTableName();
List<Map<String,Object>> attributes=new ArrayList<>();
attributes = stuTableNameMapper.selectBuildTable(fieldList,table,userId,tableName);
if(attributes.isEmpty()){
attributes = stuTableNameMapper.selectBuildTable(fieldList,table,"999999999",tableName);
}
for (int i = 0; i < attributes.size(); i++) {
if(attributes.get(i).size()<fieldList.size()){
Collections.rotate(attributes,-1);
}
}
// 过滤掉包含逗号的字符串值,并将负数转换为正数（对于数字类型）
List<Map<String, Object>> filteredAttributes = filterAndConvertAttributes(attributes);
return new ResultEntity(HttpStatus.OK,filteredAttributes);
}
List<String> fieldList = analyzeDataDTO.getFieldList();
List<Map<String,Object>> attributes = stuTableNameMapper.selectByFields(fieldList,table);
for (int i = 0; i < attributes.size(); i++) {
if(attributes.get(i)!=null){
if(attributes.get(i).size()<fieldList.size()){
Collections.rotate(attributes,-1);
}
}
}
// //将日期类型转为时间戳返回
// for (Map<String, Object> row : attributes) {
// for (Map.Entry<String, Object> entry : row.entrySet()) {
// if (entry.getValue() instanceof LocalDateTime) {
// LocalDateTime localDateTime = (LocalDateTime) entry.getValue();
// ZonedDateTime zonedDateTime = localDateTime.atZone(ZoneId.systemDefault());
// Instant instant = zonedDateTime.toInstant();
// entry.setValue(instant.toEpochMilli());
// }
// }
// }
return new ResultEntity(HttpStatus.OK,attributes);
}
@Override
public ResultEntity dataPreprocessing(String userId, String method, List<Map<String, Object>> mapList) {
if(mapList.isEmpty()){
return null;
}
//数据去重
Set<String> set = new HashSet<>();
List<Map<String, Object>> deduplicatedDataList = new ArrayList<>();
for (Map<String, Object> map : mapList) {
if(map.isEmpty()){
continue;
}
String mapString = map.toString();
if (!set.contains(mapString)) {
set.add(mapString);
deduplicatedDataList.add(map);
}
}
//判断缺失值处理方式
if(method.equals("缺失值处理--剔除数据")){
int a=0;
List<Integer> list=new ArrayList<>();
for (int i = 0; i < deduplicatedDataList.size(); i++) {
a = deduplicatedDataList.get(0).size();
if(deduplicatedDataList.get(i).size()<a){
list.add(i);
}
}
// 反转列表
Collections.reverse(list);
for (int i = 0; i < list.size(); i++) {
deduplicatedDataList.remove(deduplicatedDataList.get(list.get(i)));
}
deduplicatedDataList.removeIf(record -> record.containsValue(null));
}
if(method.equals("缺失值处理--均值代替")){
// // 均值代替处理
// for (Map<String, Object> record : deduplicatedDataList) {
// for (Map.Entry<String, Object> entry : record.entrySet()) {
// if (entry.getValue() == null) {
// double mean = calculateMean(deduplicatedDataList, entry.getKey());
// entry.setValue(mean);
// }
// }
// }
int a=0;
List<Integer> list=new ArrayList<>();
for (int i = 0; i < deduplicatedDataList.size(); i++) {
a = deduplicatedDataList.get(0).size();
if(deduplicatedDataList.get(i).size()<a){
list.add(i);
}
}
// 反转列表
Collections.reverse(list);
for (int i = 0; i < list.size(); i++) {
deduplicatedDataList.remove(deduplicatedDataList.get(list.get(i)));
}
//众数代替
for (Map<String, Object> record : deduplicatedDataList) {
for (Map.Entry<String, Object> entry : record.entrySet()) {
if (entry.getValue() == null) {
String mode = calculateMode(deduplicatedDataList, entry.getKey());
entry.setValue(mode);
}
}
}
}
return new ResultEntity(HttpStatus.OK,"成功",deduplicatedDataList);
}
//均值代替计算方式
private static double calculateMean(List<Map<String, Object>> dataList, String key) {
double sum = 0;
int count = 0;
for (Map<String, Object> record : dataList) {
Object value = record.get(key);
if (value != null && value instanceof Number) {
sum += ((Number) value).doubleValue();
count++;
}
}
return count > 0 ? (sum / count) : 0;
}
//众数代替计算方式
private static String calculateMode(List<Map<String, Object>> dataList, String key) {
Map<String, Integer> counts = new HashMap<>();
for (Map<String, Object> record : dataList) {
Object value = record.get(key);
if (value != null && value instanceof String) {
String stringValue = (String) value;
counts.put(stringValue, counts.getOrDefault(stringValue, 0) + 1);
}
}
int maxCount = 0;
String mode = null;
for (Map.Entry<String, Integer> entry : counts.entrySet()) {
if (entry.getValue() > maxCount) {
maxCount = entry.getValue();
mode = entry.getKey();
}
}
return mode != null ? mode : ""; // 如果存在众数则返回，否则返回空字符串
}
public static List<Map<String, Object>> filterEmptyValues(List<Map<String, Object>> dataList) {
List<Map<String, Object>> filteredList = new ArrayList<>();
for (Map<String, Object> map : dataList) {
boolean allValuesEmpty = true;
for (Object value : map.values()) {
if (value != null) {
allValuesEmpty = false;
break;
}
}
if (!allValuesEmpty) {
filteredList.add(map);
}
}
return filteredList;
}
// 过滤掉包含逗号的字符串值,并将负数转换为正数（对于数字类型）
public static List<Map<String, Object>> filterAndConvertAttributes(List<Map<String, Object>> attributes) {
List<Map<String, Object>> filteredAndConvertedList = new ArrayList<>();
for (Map<String, Object> map : attributes) {
Map<String, Object> convertedMap = new HashMap<>();
for (Map.Entry<String, Object> entry : map.entrySet()) {
Object value = entry.getValue();
// 检查值是否为字符串且包含逗号
if (value instanceof String) {
// 去除字符串中的逗号
String strValue = (String) value;
value = strValue.replace(",", "").replace("-","");
}
// 检查值是否为数字类型且为负数
if (value instanceof Number && ((Number) value).doubleValue() < 0) {
// 转换为正数
value = Math.abs(((Number) value).doubleValue());
// 注意：这里为了保持类型一致，我们将其转换回原始类型（如果需要）
// 这里简化为double类型，实际中可能需要更复杂的处理来保持原始类型
}
convertedMap.put(entry.getKey(), value);
}
filteredAndConvertedList.add(convertedMap);
}
return filteredAndConvertedList;
}
// 读取停用词
Set<String> loadStopwords(String filePath) throws IOException {
Set<String> stopwords = new HashSet<>();
try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
String line;
while ((line = br.readLine()) != null) {
stopwords.add(line.trim());
}
}
return stopwords;
}
}

24
src/main/java/com/sztzjy/linkCommerce/util/StringUtils.java
Unescape Escape View File

@ -0,0 +1,24 @@
package com.sztzjy.linkCommerce.util;
public class StringUtils {
/**
* 将驼峰命名法的字符串转换为下划线命名法的字符串。
* @param camelCaseStr 驼峰命名的字符串。
* @return 下划线命名法的字符串。
*/
public static String convertCamelCaseToUnderscore(String camelCaseStr) {
// 使用Pattern和Matcher处理字符串
StringBuilder underscoreStr = new StringBuilder();
for (int i = 0; i < camelCaseStr.length(); i++) {
char c = camelCaseStr.charAt(i);
if (Character.isUpperCase(c) && i > 0) {
// 如果当前字符是大写，并且不是第一个字符，则添加下划线
underscoreStr.append("_");
}
// 将字符添加到结果字符串中，如果是大写，则转换为小写
underscoreStr.append(Character.toLowerCase(c));
}
return underscoreStr.toString();
}
}

288
src/main/java/com/sztzjy/linkCommerce/util/algorithm/Apriori.java
Unescape Escape View File

@ -0,0 +1,288 @@
package com.sztzjy.linkCommerce.util.algorithm;
import com.sztzjy.linkCommerce.config.exception.handler.ServiceException;
import com.sztzjy.linkCommerce.entity.dto.AssociationRulesDTO;
import org.springframework.http.HttpStatus;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.*;
public class Apriori {
// private double min_support; //最小支持度
// private double min_confident; //最小置信度
//文件路径
// static String filePath = "D:\\home\\linkCommerce\\1.txt";
static ArrayList<ArrayList<String>> D = new ArrayList<ArrayList<String>>();//事务数据库D
public static HashMap<ArrayList<String>, Integer> C = new HashMap<ArrayList<String>, Integer>();//项目集C
public static HashMap<ArrayList<String>, Integer> L = new HashMap<ArrayList<String>, Integer>();//候选集L
// 用于存取候选集每次计算结果（即存放所有的频繁项集L），最后计算关联规则，就不用再次遍历事务数据库。
static HashMap<ArrayList<String>, Integer> L_ALL = new HashMap<ArrayList<String>, Integer>();
//从文件中读取内容,返回事务集
public static ArrayList<ArrayList<String>> readTable(List<String> inputList) {
ArrayList<ArrayList<String>> result = new ArrayList<>();
for (String line : inputList) {
String[] values = line.split(",");
ArrayList<String> rowData = new ArrayList<>();
for (String value : values) {
rowData.add(value);
}
result.add(rowData);
}
// System.out.println("\nD:" + result);
return result;
}
// public static ArrayList<ArrayList<String>> readTable(MultipartFile file) {
// ArrayList<ArrayList<String>> t = new ArrayList<>();
// try (InputStream inputStream = file.getInputStream();
// Workbook workbook = new XSSFWorkbook(inputStream)) {
// Sheet sheet = workbook.getSheetAt(0);
// for (Row row : sheet) {
// ArrayList<String> rowData = new ArrayList<>();
// for (Cell cell : row) {
// rowData.add(cell.getStringCellValue());
// }
// t.add(rowData);
// }
// } catch (IOException e) {
// e.printStackTrace();
// System.out.println("文件读取失败！");
// }
// System.out.println("\nD:" + t);
// return t;
// }
//剪枝步：从候选集C中删除小于最小支持度的，并放入频繁集L中
public static void pruning(HashMap<ArrayList<String>, Integer> C,HashMap<ArrayList<String>, Integer> L,double min_support) {
if(min_support==0){
throw new ServiceException(HttpStatus.ACCEPTED,"最小支持度阀值不能为0");
}
L.clear();
// 根据项目集生成候选集
L.putAll(C);
// 删除少于最小支持度的元素
ArrayList<ArrayList<String>> delete_key = new ArrayList<ArrayList<String>>();
for (ArrayList<String> key : L.keySet()) {
if (L.get(key) < min_support) {
delete_key.add(key);
}
}
for (int i = 0; i < delete_key.size(); i++) {
L.remove(delete_key.get(i));
}
}
/**
* 初始化事务数据库、项目集、候选集
*/
public static void init( List<String> strings,double min_support) throws IOException {
// //将文件中的数据放入集合D中
D = readTable(strings);
// 扫描事务数据库。生成项目集，支持度=该元素在事务数据库出现的次数/事务数据库的事务数
for (int i = 0; i < D.size(); i++) {
for (int j = 0; j < D.get(i).size(); j++) {
String[] e = { D.get(i).get(j) };
//将数组e转化为List
ArrayList<String> item = new ArrayList<String>(Arrays.asList(e));
//map中是否包含指定的键
if (!C.containsKey(item)) {
C.put(item, 1);
//System.out.println(C.get(item));
} else {
C.put(item, C.get(item) + 1);
//System.out.println(C.get(item));
}
}
}
//System.out.println("D.size= "+D.size());
pruning(C, L,min_support);// 剪枝
//将频繁项集放入集合中
L_ALL.clear();
L_ALL.putAll(L);
}
// 两个整数集求并集
public static ArrayList<String> arrayListUnion(ArrayList<String> arraylist1, ArrayList<String> arraylist2) {
ArrayList<String> arraylist = new ArrayList<String>();
arraylist.addAll(arraylist1);
arraylist.addAll(arraylist2);
//将ArrayList转化为HashSet去掉重复元素，再将HashSet转换为ArrayList
arraylist = new ArrayList<String>(new HashSet<String>(arraylist));
return arraylist;
}
/**
* 迭代求出最终的候选频繁集
*
* @param C
* 完成初始化的项目集
* @param L
* 完成初始化的候选集
* @return 最终的候选频繁集
*/
public static HashMap<ArrayList<String>, Integer> iteration(HashMap<ArrayList<String>, Integer> C,HashMap<ArrayList<String>, Integer> L,double min_support) {
HashMap<ArrayList<String>, Integer> L_temp = new HashMap<ArrayList<String>, Integer>();// 用于判断是否结束剪枝的临时变量
String str = null;
int t = 1;// 迭代次数
while (L.size() > 0) {// 一旦被剪枝后剪干净，剪枝之前则是最终要求的结果。
t++;
L_temp.clear();
L_temp.putAll(L);
// 一、连接步
C.clear();
// 1.将L中的项以一定的规则两两匹配
ArrayList<ArrayList<String>> L_key = new ArrayList<ArrayList<String>>(L.keySet());
for (int i = 0; i < L_key.size(); i++) {
for (int j = i + 1; j < L_key.size(); j++) {
ArrayList<String> C_item = new ArrayList<String>();
C_item = new ArrayList<String>(arrayListUnion(L_key.get(i),
L_key.get(j)));
if (C_item.size() == t) {
C.put(C_item, 0);// 频繁项集的所有非空子集都必须是频繁的
}
}
}
// 2.通过扫描D，计算此项的支持度
for (ArrayList<String> key : C.keySet()) {
for (int i = 0; i < D.size(); i++) {
if (D.get(i).containsAll(key)) {
C.put(key, C.get(key) + 1 );
}
}
}
str = C.toString();
// System.out.println("候选"+t+"项集：C: \n"+C);
// 二、剪枝步
pruning(C, L,min_support);
// System.out.println("频繁"+t+"项集：L: \n"+L+"\n");
str = L.toString();
// System.out.println("===");
L_ALL.putAll(L);
}
return L_temp;
}
// 求一个集合的所有子集
public static ArrayList<ArrayList<String>> getSubset(ArrayList<String> L) {
if (L.size() > 0) {
ArrayList<ArrayList<String>> result = new ArrayList<ArrayList<String>>();
for (int i = 0; i < Math.pow(2, L.size()); i++) {// 集合子集个数=2的该集合长度的乘方
ArrayList<String> subSet = new ArrayList<String>();
int index = i;// 索引从0一直到2的集合长度的乘方-1
for (int j = 0; j < L.size(); j++) {
// 通过逐一位移，判断索引那一位是1，如果是，再添加此项
if ((index & 1) == 1) {// 位与运算，判断最后一位是否为1
subSet.add(L.get(j));
}
index >>= 1;// 索引右移一位
}
result.add(subSet); // 把子集存储起来
}
return result;
} else {
return null;
}
}
// 判断两个集合相交是否为空
public static boolean intersectionIsNull(ArrayList<String> l1,
ArrayList<String> l2) {
Set<String> s1 = new HashSet<String>(l1);
Set<String> s2 = new HashSet<String>(l2);
s1.retainAll(s2);
if (s1.size() > 0) {
return false;
} else {
return true;
}
}
/**
* 根据最终的关联集，根据公式计算出各个关联事件
*/
public static List<AssociationRulesDTO> connection(double min_confident) {
List<AssociationRulesDTO> list=new ArrayList<>();
List<String> lists=new ArrayList<>();
for (ArrayList<String> key : L_ALL.keySet()) {// 对最终的关联集各个事件进行判断
if (key != null && !key.isEmpty() && L_ALL != null) {
ArrayList<ArrayList<String>> key_allSubset = getSubset(key);
//得到所有频繁集中每个集合的子集
// System.out.println(key_allSubset);
for (int i = 0; i < key_allSubset.size(); i++) {
ArrayList<String> item_pre = key_allSubset.get(i);//得到一个真子集
if(item_pre.size()>1){
continue;
}
if(item_pre.isEmpty()){
continue;
}
if (0 < item_pre.size() && item_pre.size() < key.size()) {// 判断是否是非空真子集
// 各个非空互补真子集之间形成关联事件
double item_pre_support = L_ALL.get(item_pre);//得到真子集的支持度度
//System.out.println("itempre="+item_pre_support);
for (int j = 0; j < key_allSubset.size(); j++) {
ArrayList<String> item_post = key_allSubset.get(j);
if (0 < item_post.size()
&& item_post.size() < key.size()
&& arrayListUnion(item_pre, item_post).equals(key)
&& intersectionIsNull(item_pre, item_post))
//不相交的两个非空真子集，相并为频繁项集
{
double d = L_ALL.get(arrayListUnion(item_pre, item_post));
//double item_post_support = L_ALL.get(item_post);// 互补真子集的支持度比则是事件的置信度
//System.out.println("item_post="+item_post_support);
double confident = d
/ item_pre_support; // 事件的置信度
if (confident > min_confident) {// 如果事件的置信度大于最小置信度
//封装对象返回
DecimalFormat df = new DecimalFormat("#.00");
AssociationRulesDTO associationRulesDTO=new AssociationRulesDTO();
associationRulesDTO.setCorrelation(item_pre);
associationRulesDTO.setAssociated(item_post);
associationRulesDTO.setConfidenceLevel(Double.parseDouble(df.format(confident)));
String string=item_pre + "==>" + item_post+"==>" + Double.parseDouble(df.format(confident));
associationRulesDTO.setRule(string);
list.add(associationRulesDTO);
// System.out.print(item_pre + "==>" + item_post );// 则是一个关联事件
// System.out.println("==>" + confident);
lists.add(string);
}
}
}
}
}
}
}
return list;
}
public static void main(String[] args) {
// Apriori apriori = new Apriori();
// apriori.init(D);
// System.out.println("候选1项集：C：\n"+apriori.C);
// System.out.println("频繁1项集：L：\n"+apriori.L+"\n");
// apriori.L = apriori.iteration(apriori.C, apriori.L);
// System.out.println("关联规则如下：");
// apriori.connection(min_confident);
}
}

41
src/main/java/com/sztzjy/linkCommerce/util/algorithm/BaiDuZhiNengYun.java
Unescape Escape View File

@ -0,0 +1,41 @@
package com.sztzjy.linkCommerce.util.algorithm;
import cn.hutool.json.JSONObject;
import okhttp3.*;
//import org.json.JSONObject;
import java.io.IOException;
/**
* @author tz
* @date 2024/7/19 15:25
*/
public class BaiDuZhiNengYun {
public static final String API_KEY = "12hSC65yXQdFndZridHTaHtd";
public static final String SECRET_KEY = "0SegCsSEahKBYB4z6ZLELebLMRt37gib";
static final OkHttpClient HTTP_CLIENT = new OkHttpClient().newBuilder().build();
/**
* 从用户的AK，SK生成鉴权签名（Access Token）
*
* @return 鉴权签名（Access Token）
* @throws IOException IO异常
*/
public static String getAccessToken() throws IOException {
MediaType mediaType = MediaType.parse("application/x-www-form-urlencoded");
RequestBody body = RequestBody.create(mediaType, "grant_type=client_credentials&client_id=" + API_KEY
+ "&client_secret=" + SECRET_KEY);
Request request = new Request.Builder()
.url("https://aip.baidubce.com/oauth/2.0/token")
.method("POST", body)
.addHeader("Content-Type", "application/x-www-form-urlencoded")
.build();
Response response = HTTP_CLIENT.newCall(request).execute();
if (response.body() != null) {
// return new JSONObject(response.body().string()).getString("access_token");
}
return null;
}
}

11
src/main/java/com/sztzjy/linkCommerce/util/algorithm/Data.java
Unescape Escape View File

@ -0,0 +1,11 @@
package com.sztzjy.linkCommerce.util.algorithm;
public class Data {
public double[] features;
public double label;
public Data(double[] features, double label) {
this.features = features;
this.label = label;
}
}

237
src/main/java/com/sztzjy/linkCommerce/util/algorithm/DataDeal.java
Unescape Escape View File

@ -0,0 +1,237 @@
package com.sztzjy.linkCommerce.util.algorithm;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.*;
public class DataDeal {
static double min_support = 4; //最小支持度
static double min_confident = 0.7; //最小置信度
//文件路径
static String filePath = "D:\\home\\linkCommerce\\1.txt";
static ArrayList<ArrayList<String>> D = new ArrayList<ArrayList<String>>();//事务数据库D
static HashMap<ArrayList<String>, Integer> C = new HashMap<ArrayList<String>, Integer>();//项目集C
static HashMap<ArrayList<String>, Integer> L = new HashMap<ArrayList<String>, Integer>();//候选集L
// 用于存取候选集每次计算结果（即存放所有的频繁项集L），最后计算关联规则，就不用再次遍历事务数据库。
static HashMap<ArrayList<String>, Integer> L_ALL = new HashMap<ArrayList<String>, Integer>();
//从文件中读取内容,返回事务集
public static ArrayList<ArrayList<String>> readTable(String filePath){
ArrayList<ArrayList<String>> t = new ArrayList<ArrayList<String>>();
ArrayList<String> t1 = null;
File file = new File(filePath);
try {
InputStreamReader isr = new InputStreamReader(new FileInputStream(file));
BufferedReader bf = new BufferedReader(isr);
String str = null;
while((str = bf.readLine()) != null) {
t1 = new ArrayList<String>();
String[] str1 = str.split(",");
for(int i = 1; i < str1.length ; i++) {
t1.add(str1[i]);
}
t.add(t1);
}
bf.close();
isr.close();
} catch (Exception e) {
e.printStackTrace();
System.out.println("文件不存在！");
}
System.out.println("\nD:"+t);
return t;
}
//剪枝步：从候选集C中删除小于最小支持度的，并放入频繁集L中
public static void pruning(HashMap<ArrayList<String>, Integer> C,HashMap<ArrayList<String>, Integer> L) {
L.clear();
// 根据项目集生成候选集
L.putAll(C);
// 删除少于最小支持度的元素
ArrayList<ArrayList<String>> delete_key = new ArrayList<ArrayList<String>>();
for (ArrayList<String> key : L.keySet()) {
if (L.get(key) < min_support) {
delete_key.add(key);
}
}
for (int i = 0; i < delete_key.size(); i++) {
L.remove(delete_key.get(i));
}
}
/**
* 初始化事务数据库、项目集、候选集
*/
public static void init() {
//将文件中的数据放入集合D中
D = readTable(filePath);
// 扫描事务数据库。生成项目集，支持度=该元素在事务数据库出现的次数/事务数据库的事务数
for (int i = 0; i < D.size(); i++) {
for (int j = 0; j < D.get(i).size(); j++) {
String[] e = { D.get(i).get(j) };
//将数组e转化为List
ArrayList<String> item = new ArrayList<String>(Arrays.asList(e));
//map中是否包含指定的键
if (!C.containsKey(item)) {
C.put(item, 1);
//System.out.println(C.get(item));
} else {
C.put(item, C.get(item) + 1);
//System.out.println(C.get(item));
}
}
}
//System.out.println("D.size= "+D.size());
pruning(C, L);// 剪枝
//将频繁项集放入集合中
L_ALL.putAll(L);
}
// 两个整数集求并集
public static ArrayList<String> arrayListUnion(ArrayList<String> arraylist1, ArrayList<String> arraylist2) {
ArrayList<String> arraylist = new ArrayList<String>();
arraylist.addAll(arraylist1);
arraylist.addAll(arraylist2);
//将ArrayList转化为HashSet去掉重复元素，再将HashSet转换为ArrayList
arraylist = new ArrayList<String>(new HashSet<String>(arraylist));
return arraylist;
}
/**
* 迭代求出最终的候选频繁集
*
* @param C
* 完成初始化的项目集
* @param L
* 完成初始化的候选集
* @return 最终的候选频繁集
*/
public static HashMap<ArrayList<String>, Integer> iteration(HashMap<ArrayList<String>, Integer> C,HashMap<ArrayList<String>, Integer> L) {
HashMap<ArrayList<String>, Integer> L_temp = new HashMap<ArrayList<String>, Integer>();// 用于判断是否结束剪枝的临时变量
String str = null;
int t = 1;// 迭代次数
while (L.size() > 0) {// 一旦被剪枝后剪干净，剪枝之前则是最终要求的结果。
t++;
L_temp.clear();
L_temp.putAll(L);
// 一、连接步
C.clear();
// 1.将L中的项以一定的规则两两匹配
ArrayList<ArrayList<String>> L_key = new ArrayList<ArrayList<String>>(L.keySet());
for (int i = 0; i < L_key.size(); i++) {
for (int j = i + 1; j < L_key.size(); j++) {
ArrayList<String> C_item = new ArrayList<String>();
C_item = new ArrayList<String>(arrayListUnion(L_key.get(i),
L_key.get(j)));
if (C_item.size() == t) {
C.put(C_item, 0);// 频繁项集的所有非空子集都必须是频繁的
}
}
}
// 2.通过扫描D，计算此项的支持度
for (ArrayList<String> key : C.keySet()) {
for (int i = 0; i < D.size(); i++) {
if (D.get(i).containsAll(key)) {
C.put(key, C.get(key) + 1 );
}
}
}
str = C.toString();
System.out.println("候选"+t+"项集：C: \n"+C);
// 二、剪枝步
pruning(C, L);
System.out.println("频繁"+t+"项集：L: \n"+L+"\n");
str = L.toString();
//System.out.println("===");
L_ALL.putAll(L);
}
return L_temp;
}
// 求一个集合的所有子集
public static ArrayList<ArrayList<String>> getSubset(ArrayList<String> L) {
if (L.size() > 0) {
ArrayList<ArrayList<String>> result = new ArrayList<ArrayList<String>>();
for (int i = 0; i < Math.pow(2, L.size()); i++) {// 集合子集个数=2的该集合长度的乘方
ArrayList<String> subSet = new ArrayList<String>();
int index = i;// 索引从0一直到2的集合长度的乘方-1
for (int j = 0; j < L.size(); j++) {
// 通过逐一位移，判断索引那一位是1，如果是，再添加此项
if ((index & 1) == 1) {// 位与运算，判断最后一位是否为1
subSet.add(L.get(j));
}
index >>= 1;// 索引右移一位
}
result.add(subSet); // 把子集存储起来
}
return result;
} else {
return null;
}
}
// 判断两个集合相交是否为空
public static boolean intersectionIsNull(ArrayList<String> l1,
ArrayList<String> l2) {
Set<String> s1 = new HashSet<String>(l1);
Set<String> s2 = new HashSet<String>(l2);
s1.retainAll(s2);
if (s1.size() > 0) {
return false;
} else {
return true;
}
}
/**
* 根据最终的关联集，根据公式计算出各个关联事件
*/
public static void connection() {
for (ArrayList<String> key : L_ALL.keySet()) {// 对最终的关联集各个事件进行判断
ArrayList<ArrayList<String>> key_allSubset = getSubset(key);
//得到所有频繁集中每个集合的子集
// System.out.println(key_allSubset);
for (int i = 0; i < key_allSubset.size(); i++) {
ArrayList<String> item_pre = key_allSubset.get(i);//得到一个真子集
if (0 < item_pre.size() && item_pre.size() < key.size()) {// 判断是否是非空真子集
// 各个非空互补真子集之间形成关联事件
double item_pre_support = L_ALL.get(item_pre);//得到真子集的支持度度
//System.out.println("itempre="+item_pre_support);
for (int j = 0; j < key_allSubset.size(); j++) {
ArrayList<String> item_post = key_allSubset.get(j);
if (0 < item_post.size()
&& item_post.size() < key.size()
&& arrayListUnion(item_pre, item_post).equals(key)
&& intersectionIsNull(item_pre, item_post))
//不相交的两个非空真子集，相并为频繁项集
{
double d = L_ALL.get(arrayListUnion(item_pre, item_post));
//double item_post_support = L_ALL.get(item_post);// 互补真子集的支持度比则是事件的置信度
//System.out.println("item_post="+item_post_support);
double confident = d
/ item_pre_support; // 事件的置信度
if (confident > min_confident) {// 如果事件的置信度大于最小置信度
System.out.print(item_pre + "==>" + item_post );// 则是一个关联事件
System.out.println("==>" + confident);
}
}
}
}
}
}
}
public static void main(String[] args) {
DataDeal apriori = new DataDeal();
apriori.init();
System.out.println("候选1项集：C：\n"+apriori.C);
System.out.println("频繁1项集：L：\n"+apriori.L+"\n");
apriori.L = apriori.iteration(apriori.C, apriori.L);
System.out.println("关联规则如下：");
apriori.connection();
}
}

135
src/main/java/com/sztzjy/linkCommerce/util/algorithm/DescriptiveStatisticsUtil.java
Unescape Escape View File

@ -0,0 +1,135 @@
package com.sztzjy.linkCommerce.util.algorithm;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import java.util.*;
public class DescriptiveStatisticsUtil {
// 计算平均数
public double getMean(List<Double> data) {
DescriptiveStatistics stats = new DescriptiveStatistics();
for (double value : data) {
stats.addValue(value);
}
return stats.getMean();
}
// 计算中位数
public double getMedian(List<Double> data) {
Collections.sort(data);
int size = data.size();
if (size % 2 == 0) {
return (data.get(size / 2 - 1) + data.get(size / 2)) / 2.0;
} else {
return data.get(size / 2);
}
}
// 计算众数
public Double getMode(List<Double> data) {
HashMap map = new HashMap();
double imode = 0;
for (int i = 0; i < data.size(); i++) {
double x = data.get(i);
if (map.containsKey(String.valueOf(x))) {
// 如果出现多次，取出以前的计数，然后加1
int len = Integer.parseInt(map.get(String.valueOf(x)).toString());
map.put(String.valueOf(x), String.valueOf(len + 1));
imode = Math.max(imode, len + 1);
} else {
// 如果是第一次出现，计数为1
map.put(String.valueOf(x), String.valueOf(1));
imode = Math.max(imode, 1);
}
}
Iterator iter = map.keySet().iterator();
ArrayList lst = new ArrayList();
while (iter.hasNext()) {
Object key = iter.next();
Object v = map.get(key);
if (Integer.parseInt(v.toString()) == imode) {
lst.add(key);
}
}
Object object = lst.get(0);
Double v = Double.valueOf(object.toString());
return v;
}
// 计算标准差
public double getStandardDeviation(List<Double> data) {
DescriptiveStatistics stats = new DescriptiveStatistics();
for (double value : data) {
stats.addValue(value);
}
return stats.getStandardDeviation();
}
// 计算方差
public double getVariance(List<Double> data) {
DescriptiveStatistics stats = new DescriptiveStatistics();
for (double value : data) {
stats.addValue(value);
}
return stats.getVariance();
}
// 计算标准误差
public double getStandardError(List<Double> data) {
double standardDeviation = getStandardDeviation(data);
int count = getCount(data);
return standardDeviation / Math.sqrt(count);
}
// 计算峰度
public double getKurtosis(List<Double> data) {
DescriptiveStatistics stats = new DescriptiveStatistics();
for (double value : data) {
stats.addValue(value);
}
return stats.getKurtosis();
}
// 计算偏度
public double getSkewness(List<Double> data) {
DescriptiveStatistics stats = new DescriptiveStatistics();
for (double value : data) {
stats.addValue(value);
}
double skewness = stats.getSkewness();
return skewness;
}
// 获取最大值
public double getMax(List<Double> data) {
return Collections.max(data);
}
// 获取最小值
public double getMin(List<Double> data) {
return Collections.min(data);
}
// 计算总和
public double getSum(List<Double> data) {
double sum = 0;
for (double value : data) {
sum += value;
}
return sum;
}
// 获取观测数
public int getCount(List<Double> data) {
return data.size();
}
}

182
src/main/java/com/sztzjy/linkCommerce/util/algorithm/KMeans.java
Unescape Escape View File

@ -0,0 +1,182 @@
package com.sztzjy.linkCommerce.util.algorithm;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class KMeans {
private int k; // 簇数
private int maxIterations; // 最大迭代次数
private List<Point> centroids; // 质心
private List<Point> points; // 数据集
public KMeans(int k, int maxIterations, List<Point> points) {
this.k = k;
this.maxIterations = maxIterations;
this.points = points;
this.centroids = initializeCentroids();
}
public List<Point> getCentroids() {
return centroids;
}
// 初始化质心
private List<Point> initializeCentroids() {
List<Point> centroids = new ArrayList<>();
Random random = new Random();
for (int i = 0; i < k; i++) {
int randomIndex = random.nextInt(points.size());
centroids.add(points.get(randomIndex));
}
return centroids;
}
// 计算点到质心的距离
private double distance(Point point1, Point point2) {
return Math.sqrt(Math.pow(point1.getX() - point2.getX(), 2) + Math.pow(point1.getY() - point2.getY(), 2));
}
// 找到每个点所属的最近质心
private int closestCentroid(Point point) {
double minDistance = Double.MAX_VALUE;
int closestCentroidIndex = -1;
for (int i = 0; i < centroids.size(); i++) {
double dist = distance(point, centroids.get(i));
if (dist < minDistance) {
minDistance = dist;
closestCentroidIndex = i;
}
}
return closestCentroidIndex;
}
// 更新质心位置为每个簇的平均值
private void updateCentroids(List<List<Point>> clusters) {
for (int i = 0; i < k; i++) {
double sumX = 0;
double sumY = 0;
List<Point> cluster = clusters.get(i);
int clusterSize = cluster.size();
for (Point point : cluster) {
sumX += point.getX();
sumY += point.getY();
}
if (clusterSize > 0) {
centroids.get(i).setX(sumX / clusterSize);
centroids.get(i).setY(sumY / clusterSize);
}
}
}
// K-means聚类算法
public List<List<Point>> cluster() {
List<List<Point>> clusters=null;
for (int iter = 0; iter < maxIterations; iter++) {
clusters = new ArrayList<>(k);
for (int i = 0; i < k; i++) {
clusters.add(new ArrayList<>());
}
// 分配每个点到最近的质心
for (Point point : points) {
int closestIndex = closestCentroid(point);
clusters.get(closestIndex).add(point);
}
// 更新质心位置
updateCentroids(clusters);
}
// 返回每个簇的点集合
return clusters;
}
// 示例中的数据点类
public static class Point {
private double x;
private double y;
public Point(double x, double y) {
this.x = x;
this.y = y;
}
public double getX() {
return x;
}
public double getY() {
return y;
}
public void setX(double x) {
this.x = x;
}
public void setY(double y) {
this.y = y;
}
@Override
public String toString() {
return "(" + x + ", " + y + ")";
}
}
public static void main(String[] args) {
// List<KMeans.Point> irisData = readIrisData("D:\\home\\linkCommerce\\Iris.txt");
// int k = 3; // 假设聚为3类
// int maxIterations = 100;
// KMeans kMeans = new KMeans(k, maxIterations, irisData);
// List<List<KMeans.Point>> clusters = kMeans.cluster();
//
// // 打印每个簇的结果
// for (int i = 0; i < clusters.size(); i++) {
// System.out.println("Cluster " + i + ": " + clusters.get(i));
// }
//
// // 获取每个簇的中心点坐标
// List<Point> centroids = kMeans.getCentroids();
// for (int i = 0; i < centroids.size(); i++) {
// System.out.println("Centroid for cluster " + i + ": " + centroids.get(i));
// }
}
public static List<String> convertToDelimitedStringList(List<Map<String, Object>> nestedList) {
List<String> flatList = new ArrayList<>();
for (Map<String, Object> innerMap : nestedList) {
StringBuilder sb = new StringBuilder();
for (Object value : innerMap.values()) {
if (sb.length() > 0) {
sb.append(",");
}
sb.append(value);
}
flatList.add(sb.toString());
}
return flatList;
}
public static List<Point> readIrisData(List<String> data) {
List<Point> points = new ArrayList<>();
for (String line : data) {
String[] values = line.split(",");
double x = Double.parseDouble(values[0]);
double y = Double.parseDouble(values[1]);
points.add(new Point(x, y));
}
return points;
}
}

288
src/main/java/com/sztzjy/linkCommerce/util/algorithm/KMeansResult.java
Unescape Escape View File

@ -0,0 +1,288 @@
package com.sztzjy.linkCommerce.util.algorithm;
import com.sztzjy.linkCommerce.entity.dto.KmeansDTO;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* @author tz
* @date 2024/7/2 17:37
*/
public class KMeansResult {
// 记录迭代的次数
static int count = 1;
// 文件所在路径
// static String filePath = "D:\\home\\linkCommerce\\Segmentation_dataset(1).csv";
// 储存从文件中读取的数据
static ArrayList<ArrayList<Float>> table = new ArrayList<ArrayList<Float>>();
// 储存分类一的结果
static ArrayList<ArrayList<Float>> alist = new ArrayList<ArrayList<Float>>();
// 储存分类二的结果
static ArrayList<ArrayList<Float>> blist = new ArrayList<ArrayList<Float>>();
// 储存分类三的结果
static ArrayList<ArrayList<Float>> clist = new ArrayList<ArrayList<Float>>();
// 记录初始随机产生的3个聚类中心
static ArrayList<ArrayList<Float>> randomList = new ArrayList<ArrayList<Float>>();
// 读取文件中的数据，储存到集合中
public static ArrayList<ArrayList<Float>> readTable(List<String> data) {
table.clear();
for (String line : data) {
ArrayList<Float> d = new ArrayList<Float>();
String[] str1 = line.split(",");
for (String str2 : str1) {
d.add(Float.parseFloat(str2));
}
table.add(d);
}
return table;
}
// public static ArrayList<ArrayList<Float>> readTable(MultipartFile file){
// table.clear();
// ArrayList<Float> d = null;
//
// try (BufferedReader br = new BufferedReader(new InputStreamReader(file.getInputStream()))) {
//
//
// // 跳过标题行
// br.readLine();
//
// String str = null;
// while((str = br.readLine()) != null) {
// d = new ArrayList<Float>();
// String[] str1 = str.split(",");
// for(int i = 0; i < str1.length ; i++) {
// d.add(Float.parseFloat(str1[i]));
// }
// table.add(d);
// }
//// System.out.println(table);
//
// } catch (Exception e) {
// e.printStackTrace();
// System.out.println("文件不存在！");
// }
// return table;
// }
// 随机产生k个初始聚类中心
public static ArrayList<ArrayList<Float>> randomList(Integer k,ArrayList<ArrayList<Float>> randomList) {
ArrayList<ArrayList<Float>> centerList=new ArrayList<>();
int[] list = new int[k];
// 生成k个不同的随机数
int size = table.size(); //生成的随机数不能大于table中元素的个数
do {
for (int i = 0; i < k; i++) {
list[i] = (int)(Math.random() * size);
}
} while (hasDuplicates(list));
// 将数据集中的前k个元素作为初始集群中心
for (int i = 0; i < k; i++) {
centerList.add(table.get(list[i]));
}
return centerList;
}
// helper方法来检查数组中是否包含重复项
private static boolean hasDuplicates(int[] arr) {
Set<Integer> set = new HashSet<>();
for (int val : arr) {
if (!set.add(val)) {
return true;
}
}
return false;
}
//比较两个数的大小，并返回其中较小的数
public static double minNumber(double x, double y) {
if(x < y) {
return x;
}
return y;
}
// 计算各个数据到中心点的距离，然后分成k类
public static void eudistance(ArrayList<ArrayList<Float>> list, int k) {
ArrayList<ArrayList<ArrayList<Float>>> clusters = new ArrayList<>(k);
for (int i = 0; i < k; i++) {
clusters.add(new ArrayList<>());
}
for (int i = 0; i < table.size(); i++) {
double[] distances = new double[k];
for (int j = 0; j < k; j++) {
distances[j] = calculateEuclideanDistance(table.get(i), list.get(j));
}
int closestCluster = findClosestCluster(distances);
clusters.get(closestCluster).add(table.get(i));
}
System.out.println("第" + count + "次迭代:");
for (int i = 0; i < k; i++) {
System.out.println("质心 " + i + ": " + clusters.get(i));
}
System.out.println();
count++;
}
//计算两个数据点之间的欧几里得距离
private static double calculateEuclideanDistance(ArrayList<Float> point1, ArrayList<Float> point2) {
double sum = 0;
for (int i = 1; i < point1.size(); i++) {
if(i<=point2.size()){
sum += Math.pow(point1.get(i) - point2.get(i), 2);
}
}
return Math.sqrt(sum);
}
//根据距离数组找到最近的簇的索引
private static int findClosestCluster(double[] distances) {
double minDistance = Double.MAX_VALUE;
int closestCluster = 0;
for (int i = 0; i < distances.length; i++) {
if (distances[i] < minDistance) {
minDistance = distances[i];
closestCluster = i;
}
}
return closestCluster;
}
//最终聚类结果
public static KmeansDTO kmeans(int k, Integer t, ArrayList<ArrayList<Float>> centroids) {
ArrayList<ArrayList<ArrayList<Float>>> clusters = new ArrayList<>(k);
for (int i = 0; i < k; i++) {
clusters.add(new ArrayList<>());
}
boolean converged = false;
int iterations = 0;
while (!converged && iterations < t) {
for (int i = 0; i < table.size(); i++) {
ArrayList<Float> point = table.get(i);
int closestCluster = findClosestCluster(point, centroids);
clusters.get(closestCluster).add(point);
}
ArrayList<ArrayList<Float>> newCentroids = new ArrayList<>(k);
for (int i = 0; i < k; i++) {
newCentroids.add(calculateCentroid(clusters.get(i)));
}
converged = checkConvergence(centroids, newCentroids);
if(!converged){
centroids = newCentroids;
for (int i = 0; i < k; i++) {
clusters.get(i).clear();
}
iterations++;
continue;
}else {
break;
}
}
// System.out.println("最终聚类结果:");
// for (int i = 0; i < k; i++) {
// System.out.println("簇 " + i + ": " + clusters.get(i));
// }
KmeansDTO kmeansDTO=new KmeansDTO();
kmeansDTO.setClusters(clusters);
kmeansDTO.setIterations(iterations);
return kmeansDTO;
}
//计算一个簇的质心
private static ArrayList<Float> calculateCentroid(ArrayList<ArrayList<Float>> cluster) {
float sum1 = 0, sum2 = 0, sum3 = 0, sum4 = 0;
for (ArrayList<Float> point : cluster) {
for (int i = 0; i < point.size(); i++) {
if(i==0){
sum1 += point.get(0);
}
if(i==1){
sum2 += point.get(1);
}
if(i==2){
sum3 += point.get(2);
}
if(i==3){
sum4 += point.get(3);
}
}
}
int a=0;
for (ArrayList<Float> point : cluster) {
a=point.size();
}
float size = cluster.size();
ArrayList<Float> centroid = new ArrayList<>();
for (int i = 0; i < a; i++) {
if(i==0){
centroid.add(sum1 / size);
}
if(i==1){
centroid.add(sum2 / size);
}
if(i==2){
centroid.add(sum3 / size);
}
if(i==3){
centroid.add(sum4 / size);
}
}
return centroid;
}
//检查是否收敛，如果没有收敛,则用新质心替换旧质心,并清空所有簇,准备进行下一轮迭代
private static boolean checkConvergence(ArrayList<ArrayList<Float>> oldCentroids, ArrayList<ArrayList<Float>> newCentroids) {
for (int i = 0; i < oldCentroids.size(); i++) {
if (!oldCentroids.get(i).equals(newCentroids.get(i))) {
return false;
}
}
return true;
}
private static int findClosestCluster(ArrayList<Float> point, ArrayList<ArrayList<Float>> centroids) {
double minDistance = Double.MAX_VALUE;
int closestCluster = 0;
for (int i = 0; i < centroids.size(); i++) {
double distance = calculateEuclideanDistance(point, centroids.get(i));
if (distance < minDistance) {
minDistance = distance;
closestCluster = i;
}
}
return closestCluster;
}
public static void main(String[] args) {
// ArrayList<ArrayList<Float>> rlist = new ArrayList<ArrayList<Float>>();
// readTable(filePath);
// rlist = randomList();
// eudistance(rlist);
// kmeans();
}
}

96
src/main/java/com/sztzjy/linkCommerce/util/algorithm/LinearRegression.java
Unescape Escape View File

@ -0,0 +1,96 @@
package com.sztzjy.linkCommerce.util.algorithm;
import java.text.DecimalFormat;
public class LinearRegression {
private double intercept;
private double[] slopes;
// 线性回归的训练方法
public void fit(double[][] x, double[] y) {
int n = y.length;
int m = x[0].length;
double[] sumX = new double[m];
double sumY = 0.0;
double[] sumXY = new double[m];
double[] sumX2 = new double[m];
for (int i = 0; i < n; i++) {
for (int j = 0; j < m; j++) {
sumX[j] += x[i][j];
sumXY[j] += x[i][j] * y[i];
sumX2[j] += x[i][j] * x[i][j];
}
sumY += y[i];
}
double[] xMean = new double[m];
for (int j = 0; j < m; j++) {
xMean[j] = sumX[j] / n;
}
double yMean = sumY / n;
DecimalFormat df = new DecimalFormat("#.00");
this.slopes = new double[m];
for (int j = 0; j < m; j++) {
this.slopes[j] = Double.parseDouble(df.format((sumXY[j] - n * xMean[j] * yMean) / (sumX2[j] - n * xMean[j] * xMean[j])));
}
this.intercept = Double.parseDouble(df.format(yMean - dotProduct(this.slopes, xMean)));
}
// 辅助方法：计算向量的点积
private double dotProduct(double[] a, double[] b) {
double result = 0.0;
for (int i = 0; i < a.length; i++) {
result += a[i] * b[i];
}
return result;
}
// 预测方法
public double predict(double[] x,double intercept,double[] slopes) {
if (x.length != slopes.length) {
throw new IllegalArgumentException("输入的x维度与模型不匹配");
}
return intercept + dotProduct(slopes, x);
}
// 获取截距
public double getIntercept() {
return this.intercept;
}
// 获取斜率
public double[] getSlopes() {
return this.slopes;
}
// public static void main(String[] args) {
// // 示例数据
// double[][] x = {{1, 19}, {1, 21}, {2, 20}, {2, 23}, {2, 31}};
// double[] y = {15, 15, 16, 16, 17};
//
// // 创建线性回归模型
// LinearRegression lr = new LinearRegression();
// lr.fit(x, y);
//
// DecimalFormat df = new DecimalFormat("#.0");
//
// List<String> list=new ArrayList<>();
// for (int i = 0; i < lr.getSlopes().length; i++) {
// String format = df.format(lr.getSlopes()[i]);
// list.add(format);
// }
// // 输出模型参数
// System.out.println("常数项值: " + df.format(lr.getIntercept()));
// System.out.println("第一个特征系数值: " + list.get(0));
//
// // 预测新数据
// double[] newX = {2, 88};
// double predictedY = lr.predict(newX);
// System.out.println("输入x: " + java.util.Arrays.toString(newX) + "预测y=" + df.format(predictedY));
// }
}

57
src/main/java/com/sztzjy/linkCommerce/util/algorithm/LogisticRegression.java
Unescape Escape View File

@ -0,0 +1,57 @@
package com.sztzjy.linkCommerce.util.algorithm;
public class LogisticRegression {
private double[] x;
private double[] y;
public LogisticRegression(double[] x, double[] y) {
this.x = x;
this.y = y;
}
// 计算线性回归参数
public double[] fit() {
if (x.length != y.length || x.length == 0) {
throw new IllegalArgumentException("Invalid input data");
}
int n = x.length;
double sumX = 0;
double sumY = 0;
double sumXY = 0;
double sumXX = 0;
// 计算各项和
for (int i = 0; i < n; i++) {
sumX += x[i];
sumY += y[i];
sumXY += x[i] * y[i];
sumXX += x[i] * x[i];
}
// 计算斜率和截距
double slope = (n * sumXY - sumX * sumY) / (n * sumXX - sumX * sumX);
double intercept = (sumY - slope * sumX) / n;
return new double[] { intercept, slope };
}
public double[][] getCoordinatesOnLine() {
double[] parameters = fit();
double intercept = parameters[0];
double slope = parameters[1];
double[][] coordinates = new double[x.length][2];
for (int i = 0; i < x.length; i++) {
coordinates[i][0] = x[i];
coordinates[i][1] = slope * x[i] + intercept;
}
return coordinates;
}
}

23
src/main/java/com/sztzjy/linkCommerce/util/algorithm/ResourceFileIoAdapter.java
Unescape Escape View File

@ -0,0 +1,23 @@
package com.sztzjy.linkCommerce.util.algorithm;
import cn.hutool.core.io.resource.ClassPathResource;
import com.hankcs.hanlp.corpus.io.IIOAdapter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
public class ResourceFileIoAdapter implements IIOAdapter {
@Override
public InputStream open(String path) throws IOException {
ClassPathResource resource = new ClassPathResource(path);
return Files.newInputStream(resource.getFile().toPath());
}
@Override
public OutputStream create(String path) throws IOException {
ClassPathResource resource = new ClassPathResource(path);
return Files.newOutputStream(resource.getFile().toPath());
}
}

50
src/main/java/com/sztzjy/linkCommerce/util/algorithm/TrainDataSet.java
Unescape Escape View File

@ -0,0 +1,50 @@
package com.sztzjy.linkCommerce.util.algorithm;
import java.util.ArrayList;
import java.util.List;
public class TrainDataSet {
/**
* 特征集合
**/
public List<double[]> features = new ArrayList<>();
/**
* 标签集合
**/
public List<Double> labels = new ArrayList<>();
/**
* 特征向量维度
**/
public int featureDim;
public int size() {
return labels.size();
}
public double[] getFeature(int index) {
return features.get(index);
}
public double getLabel(int index) {
return labels.get(index);
}
public void addData(double[] feature, double label) {
if (features.isEmpty()) {
featureDim = feature.length;
} else {
if (featureDim != feature.length) {
throwDimensionMismatchException(feature.length);
}
}
features.add(feature);
labels.add(label);
}
public void throwDimensionMismatchException(int errorLen) {
throw new RuntimeException("DimensionMismatchError: 你应该传入维度为 " + featureDim + " 的特征向量 , 但你传入了维度为 " + errorLen + " 的特征向量");
}
}

183
src/main/java/com/sztzjy/linkCommerce/util/excel/DataListener.java
Unescape Escape View File

@ -0,0 +1,183 @@
package com.sztzjy.linkCommerce.util.excel;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.metadata.data.ReadCellData;
import com.alibaba.excel.read.listener.ReadListener;
import com.alibaba.excel.util.ConverterUtils;
import com.alibaba.excel.util.ListUtils;
import com.alibaba.fastjson.JSON;
import com.sztzjy.linkCommerce.entity.HyxqfxData;
import com.sztzjy.linkCommerce.entity.StuUploadExcelUser;
import com.sztzjy.linkCommerce.mapper.HyxqfxDataMapper;
import com.sztzjy.linkCommerce.mapper.StuUploadExcelUserMapper;
import lombok.extern.slf4j.Slf4j;
import java.lang.reflect.Field;
import java.util.*;
// 有个很重要的点 DemoDataListener 不能被spring管理，要每次读取excel都要new,然后里面用到spring可以构造方法传进去
@Slf4j
public class DataListener implements ReadListener<HyxqfxData> {
/**
* 每隔5条存储数据库，实际使用中可以100条，然后清理list ，方便内存回收
*
*/
private static final int BATCH_COUNT = 150;
/**
* 缓存的数据
*/
private List<HyxqfxData> cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
/**
* 假设这个是一个DAO，当然有业务逻辑这个也可以是一个service。当然如果不用存储这个对象没用。
*/
private HyxqfxDataMapper demoDAO;
private String userId;
private String name;
List<String> list = new ArrayList<String>();
/**
* 如果使用了spring,请使用这个构造方法。每次创建Listener的时候需要把spring管理的类传进来
*
* @param demoDAO
*/
public DataListener(HyxqfxDataMapper demoDAO, String userId, String name) {
this.name = name;
this.userId = userId;
this.demoDAO = demoDAO;
}
/**
* 这里会一行行的返回头
*
* @param headMap
* @param context
*/
@Override
public void invokeHead(Map<Integer, ReadCellData<?>> headMap, AnalysisContext context) {
// log.info("解析到一条头数据:{}", JSON.toJSONString(headMap));
Map<Integer, String> integerStringMap = ConverterUtils.convertToStringMap(headMap, context);
//log.info("转换解析到的一条头数据:{}", integerStringMap);
Set<Integer> integers = integerStringMap.keySet();
for (Integer integer : integers) {
list.add(integerStringMap.get(integer));
}
// 如果想转成成 Map<Integer,String>
// 方案1： 不要implements ReadListener 而是 extends AnalysisEventListener
// 方案2： 调用 ConverterUtils.convertToStringMap(headMap, context) 自动会转换
}
/**
* 这个每一条数据解析都会来调用
*
* @param data one row value. Is is same as {@link AnalysisContext#readRowHolder()}
* @param context
*/
@Override
public void invoke(HyxqfxData data, AnalysisContext context) {
//log.info("解析到一条数据:{}", JSON.toJSONString(data));
cachedDataList.add(data);
// 达到BATCH_COUNT了，需要去存储一次数据库，防止数据几万条数据在内存，容易OOM
if (cachedDataList.size() >= BATCH_COUNT) {
saveData();
// 存储完成清理 list
cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
}
}
/**
* 所有数据解析完成了 都会来调用
*
* @param context
*/
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
saveData();
log.info("所有数据解析完成！"+ "，当前线程：" + Thread.currentThread().getName());
}
/**
* 加上存储数据库
*/
private void saveData() {
Map<String, String> map = new HashMap<>();
HyxqfxData stuUploadExcelUser = cachedDataList.get(0);
// 获取StuUploadExcelUser类的所有字段（包括私有字段）
Field[] fields = stuUploadExcelUser.getClass().getDeclaredFields();
// 遍历每个字段
for (Field field : fields) {
// 设置字段可访问（如果它是私有的）
field.setAccessible(true);
try {
// 获取字段的值
Object value = field.get(stuUploadExcelUser);
if (value==null)
{
continue;
}
map.put(field.getName(),(String) value);
} catch (IllegalAccessException e) {
// 如果字段不可访问（理论上不应该发生，因为我们已经设置了setAccessible(true)）
e.printStackTrace();
}
}
List<String> rightList = new ArrayList<>();
// 遍历Map的entrySet
for (Map.Entry<String, String> entry : map.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
if (value.contains(",")){
value = value.replaceAll(",","");
}
try {
// 尝试将字符串转换为 int
int intValue = Integer.parseInt(value);
rightList.add(key);
} catch (NumberFormatException e1) {
try {
// 如果不能转换为 int，尝试将字符串转换为 double
double doubleValue = Double.parseDouble(value);
rightList.add(key);
} catch (NumberFormatException e2) {
// 如果都不能转换，忽略这个字符串
}
}
}
String jsonString = JSON.toJSONString(rightList);
cachedDataList.forEach(item->{
item.setUserId(userId);
item.setModule(name);
// item.setStepOneA(JSON.toJSONString(list));
// item.setStepElevenC(jsonString);
});
// demoDAO.insertBatch(cachedDataList);
//log.info("存储数据库成功！");
}
}

29
src/main/resources/mappers/HyxqfxStuAnswerMapper.xml
Unescape Escape View File

@ -6,6 +6,7 @@
<result column="module" jdbcType="VARCHAR" property="module" /> <result column="module" jdbcType="VARCHAR" property="module" />
<result column="time" jdbcType="VARCHAR" property="time" /> <result column="time" jdbcType="VARCHAR" property="time" />
<result column="user_id" jdbcType="VARCHAR" property="userId" /> <result column="user_id" jdbcType="VARCHAR" property="userId" />
<result column="error_number" jdbcType="INTEGER" property="errorNumber" />
</resultMap> </resultMap>
<resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.sztzjy.linkCommerce.entity.HyxqfxStuAnswer"> <resultMap extends="BaseResultMap" id="ResultMapWithBLOBs" type="com.sztzjy.linkCommerce.entity.HyxqfxStuAnswer">
<result column="answer" jdbcType="LONGVARCHAR" property="answer" /> <result column="answer" jdbcType="LONGVARCHAR" property="answer" />
@ -69,7 +70,7 @@
</where> </where>
</sql> </sql>
<sql id="Base_Column_List"> <sql id="Base_Column_List">
id, module, time, user_id id, module, time, user_id, error_number
</sql> </sql>
<sql id="Blob_Column_List"> <sql id="Blob_Column_List">
answer answer
@ -124,9 +125,11 @@
</delete> </delete>
<insert id="insert" parameterType="com.sztzjy.linkCommerce.entity.HyxqfxStuAnswer"> <insert id="insert" parameterType="com.sztzjy.linkCommerce.entity.HyxqfxStuAnswer">
insert into hyxqfx_stu_answer (id, module, time, insert into hyxqfx_stu_answer (id, module, time,
user_id, answer) user_id, error_number, answer
)
values (#{id,jdbcType=VARCHAR}, #{module,jdbcType=VARCHAR}, #{time,jdbcType=VARCHAR}, values (#{id,jdbcType=VARCHAR}, #{module,jdbcType=VARCHAR}, #{time,jdbcType=VARCHAR},
#{userId,jdbcType=VARCHAR}, #{answer,jdbcType=LONGVARCHAR}) #{userId,jdbcType=VARCHAR}, #{errorNumber,jdbcType=INTEGER}, #{answer,jdbcType=LONGVARCHAR}
)
</insert> </insert>
<insert id="insertSelective" parameterType="com.sztzjy.linkCommerce.entity.HyxqfxStuAnswer"> <insert id="insertSelective" parameterType="com.sztzjy.linkCommerce.entity.HyxqfxStuAnswer">
insert into hyxqfx_stu_answer insert into hyxqfx_stu_answer
@ -143,6 +146,9 @@
<if test="userId != null"> <if test="userId != null">
user_id, user_id,
</if> </if>
<if test="errorNumber != null">
error_number,
</if>
<if test="answer != null"> <if test="answer != null">
answer, answer,
</if> </if>
@ -160,6 +166,9 @@
<if test="userId != null"> <if test="userId != null">
#{userId,jdbcType=VARCHAR}, #{userId,jdbcType=VARCHAR},
</if> </if>
<if test="errorNumber != null">
#{errorNumber,jdbcType=INTEGER},
</if>
<if test="answer != null"> <if test="answer != null">
#{answer,jdbcType=LONGVARCHAR}, #{answer,jdbcType=LONGVARCHAR},
</if> </if>
@ -186,6 +195,9 @@
<if test="record.userId != null"> <if test="record.userId != null">
user_id = #{record.userId,jdbcType=VARCHAR}, user_id = #{record.userId,jdbcType=VARCHAR},
</if> </if>
<if test="record.errorNumber != null">
error_number = #{record.errorNumber,jdbcType=INTEGER},
</if>
<if test="record.answer != null"> <if test="record.answer != null">
answer = #{record.answer,jdbcType=LONGVARCHAR}, answer = #{record.answer,jdbcType=LONGVARCHAR},
</if> </if>
@ -200,6 +212,7 @@
module = #{record.module,jdbcType=VARCHAR}, module = #{record.module,jdbcType=VARCHAR},
time = #{record.time,jdbcType=VARCHAR}, time = #{record.time,jdbcType=VARCHAR},
user_id = #{record.userId,jdbcType=VARCHAR}, user_id = #{record.userId,jdbcType=VARCHAR},
error_number = #{record.errorNumber,jdbcType=INTEGER},
answer = #{record.answer,jdbcType=LONGVARCHAR} answer = #{record.answer,jdbcType=LONGVARCHAR}
<if test="_parameter != null"> <if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" /> <include refid="Update_By_Example_Where_Clause" />
@ -210,7 +223,8 @@
set id = #{record.id,jdbcType=VARCHAR}, set id = #{record.id,jdbcType=VARCHAR},
module = #{record.module,jdbcType=VARCHAR}, module = #{record.module,jdbcType=VARCHAR},
time = #{record.time,jdbcType=VARCHAR}, time = #{record.time,jdbcType=VARCHAR},
user_id = #{record.userId,jdbcType=VARCHAR} user_id = #{record.userId,jdbcType=VARCHAR},
error_number = #{record.errorNumber,jdbcType=INTEGER}
<if test="_parameter != null"> <if test="_parameter != null">
<include refid="Update_By_Example_Where_Clause" /> <include refid="Update_By_Example_Where_Clause" />
</if> </if>
@ -227,6 +241,9 @@
<if test="userId != null"> <if test="userId != null">
user_id = #{userId,jdbcType=VARCHAR}, user_id = #{userId,jdbcType=VARCHAR},
</if> </if>
<if test="errorNumber != null">
error_number = #{errorNumber,jdbcType=INTEGER},
</if>
<if test="answer != null"> <if test="answer != null">
answer = #{answer,jdbcType=LONGVARCHAR}, answer = #{answer,jdbcType=LONGVARCHAR},
</if> </if>
@ -238,6 +255,7 @@
set module = #{module,jdbcType=VARCHAR}, set module = #{module,jdbcType=VARCHAR},
time = #{time,jdbcType=VARCHAR}, time = #{time,jdbcType=VARCHAR},
user_id = #{userId,jdbcType=VARCHAR}, user_id = #{userId,jdbcType=VARCHAR},
error_number = #{errorNumber,jdbcType=INTEGER},
answer = #{answer,jdbcType=LONGVARCHAR} answer = #{answer,jdbcType=LONGVARCHAR}
where id = #{id,jdbcType=VARCHAR} where id = #{id,jdbcType=VARCHAR}
</update> </update>
@ -245,7 +263,8 @@
update hyxqfx_stu_answer update hyxqfx_stu_answer
set module = #{module,jdbcType=VARCHAR}, set module = #{module,jdbcType=VARCHAR},
time = #{time,jdbcType=VARCHAR}, time = #{time,jdbcType=VARCHAR},
user_id = #{userId,jdbcType=VARCHAR} user_id = #{userId,jdbcType=VARCHAR},
error_number = #{errorNumber,jdbcType=INTEGER}
where id = #{id,jdbcType=VARCHAR} where id = #{id,jdbcType=VARCHAR}
</update> </update>
</mapper> </mapper>
Write Preview
Loading…
Cancel
Save