From e58f6a79675046566897da5029c62c1323e5cbea Mon Sep 17 00:00:00 2001 From: whb <17803890193@163.com> Date: Thu, 1 Aug 2024 13:56:46 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9Ejupyterhub=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../financial_bigdata/config/Constant.java | 5 + .../stu/JupyterHubTokenManager.java | 237 ++++++++++++++++++ .../controller/stu/JupyterhubController.java | 212 ++++++++++++++++ .../util/file/IFileUtil.java | 2 + .../util/file/LocalFileUtil.java | 32 ++- 5 files changed, 485 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/sztzjy/financial_bigdata/controller/stu/JupyterHubTokenManager.java create mode 100644 src/main/java/com/sztzjy/financial_bigdata/controller/stu/JupyterhubController.java diff --git a/src/main/java/com/sztzjy/financial_bigdata/config/Constant.java b/src/main/java/com/sztzjy/financial_bigdata/config/Constant.java index cd9f7c4..05f9ca8 100644 --- a/src/main/java/com/sztzjy/financial_bigdata/config/Constant.java +++ b/src/main/java/com/sztzjy/financial_bigdata/config/Constant.java @@ -11,6 +11,11 @@ public class Constant { * 请求头 */ public static final String AUTHORIZATION = "Authorization"; + + public static final String JUPYTERHUB_API_URL = "http://120.78.220.29:8000/hub/api"; + public static final String ADMIN_TOKEN = "170bed30b34242cfb3fda3171e1a111d"; // 替换为你的JupyterHub管理员API令牌 + + /** * Basic 请求头 */ diff --git a/src/main/java/com/sztzjy/financial_bigdata/controller/stu/JupyterHubTokenManager.java b/src/main/java/com/sztzjy/financial_bigdata/controller/stu/JupyterHubTokenManager.java new file mode 100644 index 0000000..7db73b6 --- /dev/null +++ b/src/main/java/com/sztzjy/financial_bigdata/controller/stu/JupyterHubTokenManager.java @@ -0,0 +1,237 @@ +package com.sztzjy.financial_bigdata.controller.stu; + +import cn.hutool.core.util.IdUtil; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; + +import com.sztzjy.financial_bigdata.config.Constant; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; + +public class JupyterHubTokenManager { + + + public static void createUser(String username) throws Exception { + // 创建 HTTP 客户端 + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + // 创建 POST 请求 + HttpPost postRequest = new HttpPost(Constant.JUPYTERHUB_API_URL + "/users/" + username); + postRequest.setHeader("Authorization", "token " + Constant.ADMIN_TOKEN); + + // 发送请求 + try (CloseableHttpResponse response = httpClient.execute(postRequest)) { + int statusCode = response.getStatusLine().getStatusCode(); + String responseBody = EntityUtils.toString(response.getEntity()); + + if (statusCode == 201) { + System.out.println("User " + username + " 创建成功！"); + + // 准备 chpasswd 命令的输入内容 + String passwordEntry = username + ":123qwe"; + + try { + System.out.println("开始执行密码命令"); + String[] command = {"chpasswd"}; + // 创建一个新的进程来执行 chpasswd 命令 + Process process = Runtime.getRuntime().exec(command); + + // 将用户名和密码对写入进程的标准输入 + try (OutputStream os = process.getOutputStream()) { + os.write(passwordEntry.getBytes()); + os.flush(); + } + + // 获取进程的输入流 + BufferedReader inputStream = new BufferedReader(new InputStreamReader(process.getInputStream())); + // 获取进程的错误流 + BufferedReader errorStream = new BufferedReader(new InputStreamReader(process.getErrorStream())); + + // 读取命令的输出 + String line; + StringBuilder output = new StringBuilder(); + while ((line = inputStream.readLine()) != null) { + output.append(line).append("\n"); + } + + // 读取命令的错误信息 + StringBuilder errors = new StringBuilder(); + while ((line = errorStream.readLine()) != null) { + errors.append(line).append("\n"); + } + + System.out.println("等待进程执行完成"); + // 等待进程执行完成 + int exitCode = process.waitFor(); + + if (exitCode == 0) { + // 执行成功 + System.out.println("密码创建成功！"); + } else { + // 执行失败，输出错误信息 + System.err.println("密码创建失败！\n" + errors.toString()); + } + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + } else { + System.out.println("失败的创建用户 " + username + ". Status code: " + statusCode + ", Response: " + responseBody); + } + } + } + } + + + // 获取登录 token + public static String generateToken(String username) throws IOException { + CloseableHttpClient client = HttpClients.createDefault(); + HttpPost request = new HttpPost(Constant.JUPYTERHUB_API_URL + "/users/" + username + "/tokens"); + request.addHeader("Authorization", "Bearer " + Constant.ADMIN_TOKEN); + request.addHeader("Content-Type", "application/json"); + + String json = "{\"note\": \"" + username + "\"}"; + StringEntity entity = new StringEntity(json); + request.setEntity(entity); + + HttpResponse response = client.execute(request); + HttpEntity responseEntity = response.getEntity(); + String responseBody = EntityUtils.toString(responseEntity); + + if (response.getStatusLine().getStatusCode() == 201) { + + // 解析 JSON 响应体 + JSONObject jsonObject = JSON.parseObject(responseBody); + // 提取生成的令牌 + String token = jsonObject.getString("token"); + + + System.out.println("Token : " + token); + + String url = "http://jrdsj.sztzjy.com:8000/user/" + username + "/?token=" + token; + + return url; + } else { + System.err.println("Failed to generate token. Status code: " + response.getStatusLine().getStatusCode()); + System.err.println("Response: " + responseBody); + return null; + } + } + + // 启动用户的 Jupyter 服务器 + public static void startServer(String username,String caseName) throws IOException { + + // 启动服务器后将需要的文件复制到刚创建的容器内 模拟实现文件挂载 + CloseableHttpClient client = HttpClients.createDefault(); + HttpPost request = new HttpPost(Constant.JUPYTERHUB_API_URL + "/users/" + username + "/server"); + request.addHeader("Authorization", "Bearer " + Constant.ADMIN_TOKEN); + request.addHeader("Content-Type", "application/json"); + + HttpResponse response = client.execute(request); + HttpEntity responseEntity = response.getEntity(); + String responseBody = EntityUtils.toString(responseEntity); + + if (response.getStatusLine().getStatusCode() == 200 || response.getStatusLine().getStatusCode() == 201) { + + + System.out.println("服务成功的启动: " + username); + + if (caseName == null) + { + System.out.println("服务成功的启动: " + username+"不需要挂载文件！"); + return; + } + //需要挂载的文件 + String path = "/etc/jupyterhub/data/"+caseName+"/."; + + + + try { + + String dockerName = "jupyter-"+username; + + String[] command = {"docker", "cp", path ,dockerName+":/home/jovyan/work"}; + // 创建一个新的进程来执行Python代码 + Process process = Runtime.getRuntime().exec(command); + + + // 获取进程的输入流 + BufferedReader inputStream = new BufferedReader(new InputStreamReader(process.getInputStream())); + + // 获取进程的输出流 + BufferedReader errorStream = new BufferedReader(new InputStreamReader(process.getErrorStream())); + + + // 读取Python代码的输出 + String line; + StringBuilder output = new StringBuilder(); + while ((line = inputStream.readLine()) != null) { + output.append(line).append("\n"); + } + + // 读取Python代码的错误信息 + StringBuilder errors = new StringBuilder(); + while ((line = errorStream.readLine()) != null) { + errors.append(line).append("\n"); + } + + + // 等待进程执行完成 + int exitCode = process.waitFor(); + + if (exitCode == 0) { + // 执行成功，输出Python代码的结果 + System.out.println("文件挂载成功！"); + + + } else { + // 执行失败，输出错误信息 + System.err.println("文件挂载失败了！\n"+ errors.toString()); + } + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + + } else { + System.err.println("Failed to start server for user " + username + ". Status code: " + response.getStatusLine().getStatusCode()); + System.err.println("Response: " + responseBody); + } + + + + client.close(); + } + + public static void main(String[] args) throws Exception { + + String s = IdUtil.fastSimpleUUID(); + System.out.println(s); + +// String username = "wang1"; +// createUser(username); +// try { +// // 创建用户 +// // createUser(username); +//// 启动用户 +// startServer(username); +// // 生成 token +// String token = generateToken(username); +// if (token != null) { +// System.out.println("Access your Jupyter environment at:"+ token); +// } +// +// +// } catch (IOException e) { +// System.err.println("Error managing JupyterHub user: " + e.getMessage()); +// } + } +} diff --git a/src/main/java/com/sztzjy/financial_bigdata/controller/stu/JupyterhubController.java b/src/main/java/com/sztzjy/financial_bigdata/controller/stu/JupyterhubController.java new file mode 100644 index 0000000..7ce305e --- /dev/null +++ b/src/main/java/com/sztzjy/financial_bigdata/controller/stu/JupyterhubController.java @@ -0,0 +1,212 @@ +package com.sztzjy.financial_bigdata.controller.stu; + +import cn.hutool.core.util.IdUtil; + +import com.sztzjy.financial_bigdata.annotation.AnonymousAccess; +import com.sztzjy.financial_bigdata.util.ResultEntity; +import com.sztzjy.financial_bigdata.util.file.IFileUtil; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.List; + +/** + * @author 17803 + * @date 2024-07-30 15:23 + */ + +@Api(tags = "jupyterhub的Api接口") +@RequestMapping("api/jupyterhub") +@RestController +public class JupyterhubController { + + + @Autowired + IFileUtil iFileUtil; + + @ApiOperation("创建容器") + @GetMapping("/jumpJupyerHub") + @AnonymousAccess + public ResultEntity jumpJupyerHub(@ApiParam("案例名")String caseName) { + + String username = IdUtil.fastSimpleUUID(); + String substring = username.substring(2,9); + String name = "zy" +substring; + + try { + JupyterHubTokenManager.createUser(name); + System.out.println("创建用户成功！"); + JupyterHubTokenManager.startServer(name,caseName); + System.out.println("启动服务！"); + String url = JupyterHubTokenManager.generateToken(name); + return new ResultEntity<>(HttpStatus.OK,"容器创建成功！",url); + + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + + /** + * + * 1.进入页面首先发送请求判断数据是否存在 + * 2.不存在直接调用上传接口，将数据传输过来 格式 文件夹-（ipynb,data/数据集） + * + * 教师端上传案例直接上传到对应文件夹下， 教师端管理时候 + * + */ + + @ApiOperation("判断数据集是否存在") + @GetMapping("/fileExistState") + @AnonymousAccess + public ResultEntity fileExistState(@ApiParam("案例名")String caseName) { + //存储文件路径 + String path = "/etc/jupyterhub/data/"; + String newCreatFile = path + caseName; + + File dataFile = new File(newCreatFile); + + //判断文件夹是否存在 + if (!dataFile.exists()) { + //创建文件夹 + dataFile.mkdir(); + + + //判断有无ipynb文件，进行迁移 + // 使用File类表示这个文件 + String fileResource = path+caseName+".ipynb"; + + + File myFile = new File(fileResource); + + // 判断文件是否存在 + if (myFile.exists()) { + System.out.println("文件存在: " + fileResource); + + //文件移动到对应的文件夹内 + try { + String[] command = {"mv", fileResource, newCreatFile}; + // 创建一个新的进程来执行Python代码 + Process process = Runtime.getRuntime().exec(command); + + + // 获取进程的输入流 + BufferedReader inputStream = new BufferedReader(new InputStreamReader(process.getInputStream())); + + // 获取进程的输出流 + BufferedReader errorStream = new BufferedReader(new InputStreamReader(process.getErrorStream())); + + + // 读取Python代码的输出 + String line; + StringBuilder output = new StringBuilder(); + while ((line = inputStream.readLine()) != null) { + output.append(line).append("\n"); + } + + // 读取Python代码的错误信息 + StringBuilder errors = new StringBuilder(); + while ((line = errorStream.readLine()) != null) { + errors.append(line).append("\n"); + } + + // 等待进程执行完成 + int exitCode = process.waitFor(); + + if (exitCode == 0) { + // 执行成功，输出Python代码的结果 + System.out.println("文件迁移成功！"); + return new ResultEntity<>(HttpStatus.OK,"数据不存在！",false); + } else { + // 执行失败，输出错误信息 + System.err.println("文件迁移失败了！\n"+ errors.toString()); + return new ResultEntity<>(HttpStatus.OK,"数据不存在！",false); + } + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + + } else { + return new ResultEntity<>(HttpStatus.OK,"数据不存在！",false); + } + + + return new ResultEntity<>(HttpStatus.OK,"数据不存在！",false); + + }else { + + //判断子目录是否有data文件 + + String sonPath = newCreatFile +"/data"; + File sonFile = new File(sonPath); + if (sonFile.exists()){ + return new ResultEntity<>(HttpStatus.OK,"数据存在！",true); + }else { + return new ResultEntity<>(HttpStatus.OK,"数据不存在！",false); + } + } + + } + + + @ApiOperation("数据集不存在文件上传") + @PostMapping("/uploadFile") + @AnonymousAccess + public ResultEntity uploadFile(@ApiParam("文件")List dataFile,@ApiParam("案例名")String caseName) { + + + String filepath = "/etc/jupyterhub/data/"+caseName+"/data/"; + + + //设置数据文件 + if (dataFile != null && !dataFile.isEmpty()) { + + for (MultipartFile file : dataFile) { + + iFileUtil.uploadResource(file,filepath,null); + } + + } + return new ResultEntity<>(HttpStatus.OK,"上传成功"); + + } + + + /** + * 上传ipynb文件 创建文件夹 /文件名 到金融大数据服务器上 + * 上传html 上传到 资源中心服务器 html + * 创建案例 跳转 （文件名创建 不存在 ） + */ + + + + @ApiOperation("超管端或者教师端上传ipynb文件") + @PostMapping("/uploadFileByIpynb") + @AnonymousAccess + public ResultEntity uploadFileByIpynb(@RequestPart @ApiParam("文件")MultipartFile file, @ApiParam("案例名")String caseName) { + + + String filepath = "/etc/jupyterhub/data/"+caseName; + + + iFileUtil.uploadResource(file,filepath,caseName); + + + + return new ResultEntity<>(HttpStatus.OK,"上传成功"); + + } + + + +} diff --git a/src/main/java/com/sztzjy/financial_bigdata/util/file/IFileUtil.java b/src/main/java/com/sztzjy/financial_bigdata/util/file/IFileUtil.java index cd871f5..0e4d513 100644 --- a/src/main/java/com/sztzjy/financial_bigdata/util/file/IFileUtil.java +++ b/src/main/java/com/sztzjy/financial_bigdata/util/file/IFileUtil.java @@ -89,4 +89,6 @@ public interface IFileUtil { void uploadResource(MultipartFile file,String filePath); + + void uploadResource(MultipartFile file, String filepath,String caseName); } diff --git a/src/main/java/com/sztzjy/financial_bigdata/util/file/LocalFileUtil.java b/src/main/java/com/sztzjy/financial_bigdata/util/file/LocalFileUtil.java index 7e1a51b..9190560 100644 --- a/src/main/java/com/sztzjy/financial_bigdata/util/file/LocalFileUtil.java +++ b/src/main/java/com/sztzjy/financial_bigdata/util/file/LocalFileUtil.java @@ -15,7 +15,7 @@ import java.util.Calendar; import java.util.Date; import java.util.List; -public class LocalFileUtil implements IFileUtil{ +public class LocalFileUtil implements IFileUtil { private final static List excludeSp = Arrays.asList("exe", "bin", "sh"); @@ -176,7 +176,7 @@ public class LocalFileUtil implements IFileUtil{ @Override - public void downloadByOtherFile(HttpServletResponse response, String name,String relativePath) { + public void downloadByOtherFile(HttpServletResponse response, String name, String relativePath) { Assert.hasText(relativePath, "路径为空"); System.out.println("------------------------------------" + relativePath); File file = new File(relativePath); @@ -228,5 +228,31 @@ public class LocalFileUtil implements IFileUtil{ } catch (IOException e) { throw new IllegalArgumentException("上传文件失败,IO错误", e); } -} + } + + @Override + public void uploadResource(MultipartFile file, String relativePath,String caseName) { + Assert.isTrue(!file.isEmpty(), "文件不存在"); + + try { + // 创建目录如果不存在 + File directory = new File(relativePath); + if (!directory.exists()) { + directory.mkdirs(); + } + // 设置目标文件路径 + String result = caseName == null ? file.getOriginalFilename() : caseName; + + int i = file.getOriginalFilename().lastIndexOf("."); + String substring = file.getOriginalFilename().substring(i); + + File destFile = new File(relativePath + File.separator + result+substring); + + file.transferTo(destFile); + } catch (FileNotFoundException e) { + throw new IllegalArgumentException("上传文件失败,FileNotFound错误", e); + } catch (IOException e) { + throw new IllegalArgumentException("上传文件失败,IO错误", e); + } + } }