Merge branch 'dev-lzy' of http://47.108.150.237:10000/www/taais into develop

script/sql/postgresql/postgresql-V1.0.0.sql

@@ -1989,7 +1989,7 @@ CREATE TABLE public.sys_dict_data
     create_time timestamp without time zone,
     update_by   bigint,
     update_time timestamp without time zone,
-    remark      character varying(500) DEFAULT NULL::character varying
+    remark      character varying(2000) DEFAULT NULL::character varying
 );
 
 

taais-modules/taais-biz/src/main/java/com/taais/biz/controller/DataController.java

@@ -2,11 +2,15 @@ package com.taais.biz.controller;
 
 import cn.dev33.satoken.annotation.SaCheckPermission;
 import com.taais.biz.domain.vo.BatchDataResult;
+import cn.hutool.core.util.IdUtil;
 import com.taais.biz.domain.Data;
 import com.taais.biz.domain.bo.DataBo;
+import com.taais.biz.domain.dto.DataAmplifyDto;
+import com.taais.biz.domain.vo.DataExcelVo;
 import com.taais.biz.domain.vo.DataSelectVo;
 import com.taais.biz.domain.vo.DataVo;
 import com.taais.biz.service.IDataService;
+import com.taais.common.core.config.TaaisConfig;
 import com.taais.common.core.core.domain.CommonResult;
 import com.taais.common.core.core.page.PageResult;
 import com.taais.common.excel.utils.ExcelUtil;
@@ -16,8 +20,12 @@ import com.taais.common.web.annotation.RepeatSubmit;
 import com.taais.common.web.core.BaseController;
 import com.taais.system.config.ServerConfig;
 import jakarta.annotation.Resource;
+import jakarta.servlet.ServletOutputStream;
 import jakarta.servlet.http.HttpServletResponse;
 import lombok.RequiredArgsConstructor;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -25,8 +33,16 @@ import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 /**
  * 数据管理Controller
@@ -78,8 +94,84 @@ public class DataController extends BaseController {
     @Log(title = "数据管理", businessType = BusinessType.EXPORT)
     @PostMapping("/export")
     public void export(HttpServletResponse response, DataBo dataBo) {
+        // 查询数据列表
         List<DataVo> list = dataService.selectList(dataBo);
-        ExcelUtil.exportExcel(list, "数据管理", DataVo.class, response);
+
+        // 创建一个临时目录用于存放文件
+        String tempDirPath = System.getProperty("java.io.tmpdir") + "/export_" + System.currentTimeMillis();
+        File tempDir = new File(tempDirPath);
+        if (!tempDir.exists()) {
+            log.info("创建临时目录:{}", tempDir.mkdirs());
+        }
+        String profile = TaaisConfig.getProfile();
+        // 拷贝文件到临时目录
+        for (DataVo dataVo : list) {
+            File srcFile = new File(profile + dataVo.getUrl());
+            File destFile = new File(tempDir, srcFile.getName());
+            try {
+                if (srcFile.exists()) {
+                    FileUtils.copyFile(srcFile, destFile);
+                } else {
+                    log.error("图片源文件不存在：{}", srcFile.getName());
+                }
+                if (!StringUtils.isEmpty(dataVo.getLabelurl())) {
+                    File labelurlFile = new File(profile + dataVo.getLabelurl());
+                    File labelurlDestFile = new File(tempDir, labelurlFile.getName());
+                    if (labelurlFile.exists()) {
+                        FileUtils.copyFile(labelurlFile, labelurlDestFile);
+                    } else {
+                        log.error("标注源文件不存在：{}", labelurlFile.getName());
+                    }
+                }
+            } catch (IOException e) {
+                throw new RuntimeException("文件拷贝失败：" + srcFile.getName(), e);
+            }
+        }
+
+        List<DataExcelVo> dataExcelVoList = new ArrayList<>();
+        if (!list.isEmpty()) {
+            dataExcelVoList = list.stream().map(info -> DataExcelVo.builder().id(info.getId()).name(info.getName())
+                            .dataType(info.getDataType()).fileType(info.getFileType()).objectType(info.getObjectType())
+                            .objectSubtype(info.getObjectSubtype()).batchNum(info.getBatchNum()).scene(info.getScene())
+                            .dataSource(info.getDataSource()).gatherTime(info.getGatherTime()).gatherSpot(info.getGatherSpot())
+                            .increment(info.getIncrement()).build())
+                    .toList();
+        }
+        // 生成 Excel 文件并保存到临时目录
+        File excelFile = new File(tempDir, "数据管理.xlsx");
+        try (FileOutputStream fos = new FileOutputStream(excelFile)) {
+            ExcelUtil.exportExcel(dataExcelVoList, "数据管理", DataExcelVo.class, fos);
+        } catch (IOException e) {
+            throw new RuntimeException("生成Excel文件失败", e);
+        }
+        // 将临时目录打包为ZIP文件
+        File zipFile = new File(tempDir.getParent(), "export_" + System.currentTimeMillis() + ".zip");
+        try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
+            for (File file : Objects.requireNonNull(tempDir.listFiles())) {
+                zos.putNextEntry(new ZipEntry(file.getName()));
+                try (FileInputStream fis = new FileInputStream(file)) {
+                    IOUtils.copy(fis, zos);
+                }
+                zos.closeEntry();
+            }
+        } catch (IOException e) {
+            throw new RuntimeException("打包ZIP文件失败", e);
+        }
+
+        // 设置响应头，返回ZIP文件给客户端
+        response.setContentType("application/zip");
+        response.setHeader("Content-Disposition", "attachment; filename=" + zipFile.getName());
+
+        try (ServletOutputStream os = response.getOutputStream()) {
+            FileUtils.copyFile(zipFile, os);
+            os.flush();
+        } catch (IOException e) {
+            throw new RuntimeException("返回ZIP文件失败", e);
+        } finally {
+            // 删除临时文件
+            FileUtils.deleteQuietly(tempDir);
+            FileUtils.deleteQuietly(zipFile);
+        }
     }
 
     /**
@@ -148,4 +240,9 @@ public class DataController extends BaseController {
         }
         return dataService.uploadDataInfo(file, dataInfo);
     }
+
+    @PostMapping("/amplify")
+    public CommonResult<Boolean> dataAmplify(@RequestBody DataAmplifyDto dataAmplifyDto) {
+        return dataService.dataAmplify(dataAmplifyDto);
+    }
 }

taais-modules/taais-biz/src/main/java/com/taais/biz/domain/dto/DataAmplifyDto.java

@@ -0,0 +1,37 @@
+package com.taais.biz.domain.dto;
+
+import jakarta.validation.constraints.NotEmpty;
+import jakarta.validation.constraints.NotNull;
+import lombok.*;
+import lombok.experimental.Accessors;
+
+import java.util.Map;
+
+
+/**
+ * @author Eureka
+ */
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Accessors(chain = true)
+public class DataAmplifyDto {
+
+    @NotEmpty(message = "批次不能为空")
+    private String batchNum;
+
+    @NotEmpty(message = "任务名称")
+    private String taskName;
+
+    @NotEmpty(message = "扩充方式不能为空")
+    private String augmentationType;
+
+    private String inputImagePath;
+
+    private String outputImagePath;
+
+    @NotNull(message = "扩充算法配置不能为空")
+    private Map<String, String> otherParams;
+}

taais-modules/taais-biz/src/main/java/com/taais/biz/domain/vo/DataExcelVo.java

@@ -0,0 +1,74 @@
+package com.taais.biz.domain.vo;
+
+import com.alibaba.excel.annotation.ExcelIgnoreUnannotated;
+import com.alibaba.excel.annotation.ExcelProperty;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.taais.biz.domain.Data;
+import io.github.linpeilie.annotations.AutoMapper;
+import lombok.*;
+import lombok.experimental.Accessors;
+
+import java.util.Date;
+
+/**
+ * @author king
+ * @date 2024年08月23日 上午9:43
+ */
+@Getter
+@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Accessors(chain = true)
+@ExcelIgnoreUnannotated
+@AutoMapper(target = Data.class)
+public class DataExcelVo {
+
+    @ExcelProperty(value = "编号")
+    private Long id;
+
+    /** 名称 */
+    @ExcelProperty(value = "名称")
+    private String name;
+
+    /** 数据类型 */
+    @ExcelProperty(value = "数据类型")
+    private String dataType;
+
+    /** 文件类型 */
+    @ExcelProperty(value = "文件类型")
+    private String fileType;
+
+    /** 目标类型 */
+    @ExcelProperty(value = "目标类型")
+    private String objectType;
+
+    /** 目标子类型 */
+    @ExcelProperty(value = "目标子类型")
+    private String objectSubtype;
+
+    /** 批次号 */
+    @ExcelProperty(value = "批次号")
+    private String batchNum;
+
+    /** 场景 */
+    @ExcelProperty(value = "场景")
+    private String scene;
+
+    /** 数据源 */
+    @ExcelProperty(value = "数据源")
+    private String dataSource;
+
+    /** 采集时间 */
+    @ExcelProperty(value = "采集时间")
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
+    private Date gatherTime;
+
+    /** 采集地点 */
+    @ExcelProperty(value = "采集地点")
+    private String gatherSpot;
+
+    /** 扩增方式 */
+    @ExcelProperty(value = "扩增方式")
+    private String increment;
+}

taais-modules/taais-biz/src/main/java/com/taais/biz/service/IDataService.java

@@ -6,6 +6,7 @@ import java.util.Map;
 
 import com.taais.biz.domain.Data;
 import com.taais.biz.domain.vo.BatchDataResult;
+import com.taais.biz.domain.dto.DataAmplifyDto;
 import com.taais.biz.domain.vo.DataSelectVo;
 import com.taais.biz.domain.vo.DataVo;
 import com.taais.biz.domain.bo.DataBo;
@@ -53,6 +54,14 @@ public interface IDataService extends IBaseService<Data> {
      */
     CommonResult<Boolean> uploadDataInfo(MultipartFile file, Data dataInfo);
 
+    /**
+     * 数据扩增
+     *
+     * @param dataAmplifyDto 数据表单
+     * @return 数据管理集合
+     */
+    CommonResult<Boolean> dataAmplify(DataAmplifyDto dataAmplifyDto);
+
     /**
      * 分页查询数据管理列表
      *

taais-modules/taais-biz/src/main/java/com/taais/biz/service/impl/DataServiceImpl.java

@@ -1,11 +1,17 @@
 package com.taais.biz.service.impl;
 
 import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.http.HttpRequest;
+import cn.hutool.http.HttpUtil;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.mybatisflex.core.paginate.Page;
 import com.mybatisflex.core.query.QueryWrapper;
 import com.taais.biz.domain.Data;
 import com.taais.biz.domain.bo.DataBo;
 import com.taais.biz.domain.vo.BatchDataResult;
+import com.taais.biz.domain.dto.DataAmplifyDto;
 import com.taais.biz.domain.vo.DataSelectVo;
 import com.taais.biz.domain.vo.DataVo;
 import com.taais.biz.mapper.DataMapper;
@@ -18,6 +24,7 @@ import com.taais.common.core.utils.StringUtils;
 import com.taais.common.core.utils.file.FileUploadUtils;
 import com.taais.common.core.utils.file.FileUtils;
 import com.taais.common.core.utils.file.UnPackedUtil;
+import com.taais.common.json.utils.JsonUtils;
 import com.taais.common.orm.core.page.PageQuery;
 import com.taais.common.orm.core.service.impl.BaseServiceImpl;
 import com.taais.common.redis.utils.RedisUtils;
@@ -28,6 +35,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.File;
@@ -37,11 +45,11 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 import static com.taais.biz.domain.table.DataTableDef.DATA;
 
@@ -60,6 +68,10 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
     private static final String RAR = ".rar";
     private static final String TXT = ".txt";
     private static final String[] VALID_EXTENSIONS = {".jpg", ".jpeg", ".png"};
+    private static final String PYTHON_DATA_AMPLIFY_API = "http://127.0.0.1:9096/api/data/amplify";
+    private static final String RESULT_CODE = "status";
+    private static final String RESULT_STATUS = "200";
+    private static final String AMPLIFY = "/AMPLIFY/";
     @Resource
     private DataMapper dataMapper;
 
@@ -256,6 +268,63 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
         return CommonResult.success("数据集上传成功！");
     }
 
+    @Override
+    @Transactional
+    public CommonResult<Boolean> dataAmplify(DataAmplifyDto dataAmplifyDto) {
+        //根据批次号获取该批次的所有文件数据
+        QueryWrapper query = query();
+        query.eq(Data::getBatchNum, dataAmplifyDto.getBatchNum());
+        List<Data> dataList = dataMapper.selectListByQuery(query);
+        if (dataList.isEmpty()) {
+            return CommonResult.fail("该批次下没有文件数据，请重新选择批次！");
+        }
+        //TODO: 此处需要定义任务开始，把相关任务信息添加上（任务名称、任务开始时间、任务类型），然后再处理文件。
+
+        List<Data> dataListInfo = dataList.stream().filter(data -> !StringUtils.isEmpty(data.getUrl())).toList();
+        if (dataListInfo.isEmpty()) {
+            return CommonResult.fail("该批次下没有文件数据，请重新选择批次！");
+        }
+        String filePath = TaaisConfig.getUploadPath();
+        LocalDate currentDate = LocalDate.now();
+        // 定义日期格式器
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy/MM/dd");
+        String formattedDate = currentDate.format(formatter);
+        filePath = filePath + File.separator + formattedDate;
+        String finalFilePath = filePath;
+        dataListInfo.forEach(dataInfo -> {
+            try {
+                //循环调用Python扩增接口
+                Map<String, Object> bodyJson = new HashMap<>();
+                bodyJson.put("augmentationType", dataAmplifyDto.getAugmentationType());
+                bodyJson.put("inputImagePath", dataInfo.getUrl());
+                String outputImagePath = finalFilePath + AMPLIFY + System.currentTimeMillis();
+                File desc = new File(outputImagePath);
+                if (!desc.exists()) {
+                    log.info("创建文件目录: {}", desc.mkdirs());
+                }
+                bodyJson.put("outputImagePath", outputImagePath);
+                bodyJson.put("otherParams", dataAmplifyDto.getOtherParams());
+                //实际请求接口，接口未提供，暂且注释
+//                String response = HttpRequest.post(PYTHON_DATA_AMPLIFY_API)
+//                        .body(JsonUtils.toJsonString(bodyJson))
+//                        .execute().body();
+                String response = "{\"status\":200,\"msg\":\"扩增成功\"}";
+                ObjectMapper objectMapper = new ObjectMapper();
+                JsonNode rootNode = objectMapper.readTree(response);
+                String resultCode = rootNode.path(RESULT_CODE).asText();
+                //判断接口是否响应成功
+                if (!RESULT_STATUS.equals(resultCode)) {
+                    throw new RuntimeException("调用Python接口返回扩增失败");
+                }
+                //处理当前目录文件，并进行入库
+                saveDataInfo(outputImagePath, dataInfo);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+        return CommonResult.fail("该批次下没有文件数据，请重新选择批次！");
+    }
+
     private void initFileInfo(String dest, List<File> extractedImagesFileList, boolean directory, String fileName) {
         if (!directory) {
             String fileHeaderSuffix = StringUtils.substring(fileName, fileName.lastIndexOf("."), fileName.length());
@@ -368,4 +437,84 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
     }
 
 
+    /**
+     * 解析目标目录所有文件，进行文件更名并入库
+     *
+     * @param directoryPath：目录地址
+     * @param dataInfo：深拷贝对象
+     */
+    public void saveDataInfo(String directoryPath, Data dataInfo) {
+        try {
+            // 获取指定目录下所有文件
+            File directory = new File(directoryPath);
+            List<File> extractedImagesFileList = new ArrayList<>();
+            if (directory.exists() && directory.isDirectory()) {
+                File[] files = directory.listFiles();
+                if (files != null) {
+                    for (File file : files) {
+                        initFileInfo(directoryPath, extractedImagesFileList, file.isDirectory(), file.getName());
+                    }
+                }
+            }
+
+            // 获取ID集合
+            List<Long> ids = dataMapper.getIds(extractedImagesFileList.size());
+            if (ids.isEmpty()) {
+                return;
+            }
+
+            List<Data> dataList = new ArrayList<>();
+            AtomicInteger countSize = new AtomicInteger();
+            extractedImagesFileList.forEach(fileInfo -> {
+                Long id = ids.get(countSize.get());
+                Data data = new Data();
+                BeanUtils.copyProperties(dataInfo, data);
+
+                if (checkLabeled(fileInfo.getPath())) {
+                    data.setLabeled(Boolean.TRUE);
+                } else {
+                    data.setLabeled(Boolean.FALSE);
+                }
+
+                try {
+                    Path path = Paths.get(fileInfo.getAbsolutePath());
+                    BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class);
+                    Instant creationTime = attrs.lastModifiedTime().toInstant();
+                    Date date = Date.from(creationTime);
+
+                    data.setId(id);
+                    data.setGatherTime(date);
+                    data.setName(fileInfo.getName());
+
+                    // 更改图片文件名称
+                    String fileHeaderSuffix = StringUtils.substring(fileInfo.getName(), fileInfo.getName().lastIndexOf("."), fileInfo.getName().length());
+                    String destInfo = fileInfo.getPath().replaceAll(fileInfo.getName(), "");
+                    File newFile = new File(destInfo, id + fileHeaderSuffix);
+                    File oldFile = new File(destInfo, fileInfo.getName());
+                    log.info("saveDataInfo更改用户上传图片文件名称:{}", oldFile.renameTo(newFile));
+
+                    String imagePath = FileUploadUtils.getPathFileName(destInfo, id + fileHeaderSuffix);
+                    data.setUrl(imagePath);
+
+                    if (data.getLabeled()) {
+                        String labeledPath = fileInfo.getPath().replaceFirst("[.][^.]+$", "") + ".txt";
+                        File labeledNewFile = new File(destInfo, id + ".txt");
+                        File labeledOldFile = new File(labeledPath);
+                        log.info("saveDataInfo更改用户上传标注文件名称:{}", labeledOldFile.renameTo(labeledNewFile));
+                        String labelUrl = FileUploadUtils.getPathFileName(destInfo, id + ".txt");
+                        data.setLabelurl(labelUrl);
+                    }
+                    dataList.add(data);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+                countSize.getAndIncrement();
+            });
+
+            dataMapper.insertBatch(dataList);
+        } catch (Exception e) {
+            log.error("[saveDataInfo]数据集处理出现未知异常.e:", e);
+        }
+    }
+
 }