feat:添加新功能

taais-modules/taais-biz/src/main/java/com/taais/biz/controller/DataController.java

@@ -1,14 +1,8 @@
 package com.taais.biz.controller;
 
 import cn.dev33.satoken.annotation.SaCheckPermission;
-import com.mybatisflex.core.query.QueryWrapper;
-import com.taais.common.core.config.TaaisConfig;
 import com.taais.common.core.core.domain.CommonResult;
 import com.taais.common.core.core.page.PageResult;
-import com.taais.common.core.utils.StringUtils;
-import com.taais.common.core.utils.file.FileUploadUtils;
-import com.taais.common.core.utils.file.FileUtils;
-import com.taais.common.core.utils.file.UnPackedUtil;
 import com.taais.common.excel.utils.ExcelUtil;
 import com.taais.common.log.annotation.Log;
 import com.taais.common.log.enums.BusinessType;
@@ -19,29 +13,18 @@ import com.taais.biz.domain.bo.DataBo;
 import com.taais.biz.domain.vo.DataVo;
 import com.taais.biz.service.IDataService;
 import com.taais.system.config.ServerConfig;
-import com.taais.system.domain.vo.SysOssUploadVo;
 import jakarta.annotation.Resource;
 import jakarta.servlet.http.HttpServletResponse;
 import lombok.RequiredArgsConstructor;
-import net.lingala.zip4j.model.FileHeader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * 数据管理Controller
@@ -141,143 +124,11 @@ public class DataController extends BaseController {
         ExcelUtil.exportExcel(new ArrayList<>(), "用户数据", DataVo.class, response);
     }
 
-    /**
-     * todo Eureka
-     * 代码放在service层
-     * 1 检查是否有重复的批次号
-     * 2 解压图片并检查
-     * 2.1 解压缩包，并获取所有图片
-     * 2.2 如果选择了已标注，检查所有图片是否都有标注
-     * 3 入库前逻辑
-     * 3.1 根据图片个数获取主键序列
-     * 3.2 循环图片
-     * 3.2.1 深克隆dataInfo数据，把图片名称保存在clone.name里，把主键序列放入id、更新图片和标注名称和主键id保持一致，把图片的文件路径存入url，把标注的文件路径存入labelurl
-     * 3.2.2 获取当前图片的创建时间 存入clone.gatherTime，如果该图片有标注的txt 设置labeled为true，否则为false
-     * 4 入库
-     * @param file
-     * @param dataInfo
-     * @return
-     * @throws Exception
-     */
     @PostMapping("/zip/upload")
-    public CommonResult<SysOssUploadVo> uploadZipFile(@RequestParam("file") MultipartFile file, Data dataInfo) throws Exception {
-        String originalFilename = file.getOriginalFilename();
-        assert originalFilename != null;
-        String suffix = StringUtils.substring(originalFilename, originalFilename.lastIndexOf("."), originalFilename.length());
-        // 解压
-        String basedir = TaaisConfig.getUploadPath();
-        // 解压目录
-        File zipOrRarTemp = FileUploadUtils.getAbsoluteFile(basedir, FileUploadUtils.extractFilename2(file));
-        String destZip = zipOrRarTemp.getAbsolutePath();
-
-        file.transferTo(Paths.get(destZip));
-        String dest = zipOrRarTemp.getParent();
-
-        // todo Eureka 改成使用sequence获取id，使用id作为图片的名称和data的id，此处的逻辑在并发场景会获取相同的idMax会出现异常
-        //取出最大值
-        Integer idMax = dataService.getIdMax();
-        if (idMax == null) {
-            idMax = 1;
-        } else {
-            idMax = idMax + 1;
-        }
-
-        List<Data> dataList = new ArrayList<>();
-        if (suffix.equals(".zip")) {
-            List<FileHeader> fileheaders = UnPackedUtil.unPackZip(zipOrRarTemp, dest);
-            //添加解压目录
-            for (FileHeader fileHeader : fileheaders) {
-                boolean isOk = initFileInfo(dest, idMax, fileHeader.isDirectory(), fileHeader.getFileName(), dataList, dataInfo);
-                if (isOk) {
-                    idMax++;
-                }
-            }
-        } else if (suffix.equals(".rar")) {
-            List<com.github.junrar.rarfile.FileHeader> fileheaders = UnPackedUtil.unPackRar(zipOrRarTemp, dest);
-            //添加解压目录
-            for (com.github.junrar.rarfile.FileHeader fileHeader : fileheaders) {
-                boolean isOk = initFileInfo(dest, idMax, fileHeader.isDirectory(), fileHeader.getFileName(), dataList, dataInfo);
-                if (isOk) {
-                    idMax++;
-                }
-            }
-        }
-
-        if (dataInfo.getLabeled()) {
-            List<Data> unmarkedData = dataList.stream()
-                    .filter(data -> Boolean.FALSE.equals(data.getLabeled()))
-                    .toList();
-
-            // 计算已标注和未标注的数据数量
-            long markedCount = dataList.stream()
-                    .filter(data -> Boolean.TRUE.equals(data.getLabeled()))
-                    .count();
-            long unmarkedCount = unmarkedData.size();
-
-            // 如果存在未标注的数据，则返回错误信息
-            if (unmarkedCount > 0) {
-                String format = String.format("错误: 已标注文件 %d 个，未标注文件 %d 个", markedCount, unmarkedCount);
-                return CommonResult.fail(format);
-            }
+    public CommonResult<Boolean> uploadZipFile(@RequestParam("file") MultipartFile file, Data dataInfo) throws Exception {
+        if (file == null) {
+            return CommonResult.fail("请上传.zip、.rar压缩文件。");
         }
-
-        // todo Eureka 不用循环检查，所有批次都是一样的，这个检查放在最开始
-        AtomicBoolean isTrue = new AtomicBoolean(false);
-        dataList.forEach(data -> {
-            QueryWrapper query = dataService.query();
-            query.eq(Data::getBatchNum, data.getBatchNum());
-            long count = dataService.getMapper().selectCountByQuery(query);
-            if (count > 0) {
-                isTrue.set(true);
-            }
-        });
-
-        if (isTrue.get()) {
-            return CommonResult.fail("文件检测出重复批次号，请仔细检查后重新导入！");
-        }
-        dataService.saveBatch(dataList);
-        //删除压缩文件
-        FileUtils.deleteFile(destZip);
-        return CommonResult.success("数据集上传成功！");
-    }
-
-    private boolean initFileInfo(String dest, Integer idMax, boolean directory, String fileName, List<Data> dataList, Data dataInfo) throws IOException {
-        if (!directory) {
-            String fileHeaderSuffix = StringUtils.substring(fileName, fileName.lastIndexOf("."), fileName.length());
-            // todo Eureka add bpm
-            if (!fileHeaderSuffix.equals(".jpg") && !fileHeaderSuffix.equals(".jpeg") && !fileHeaderSuffix.equals(".png")) {
-                return false;
-            }
-            File file = new File(dest, idMax + fileHeaderSuffix);
-            File odlFile = new File(dest, fileName);
-            log.info("更改用户上传文件名称:{}", odlFile.renameTo(file));
-            String pathFileName = FileUploadUtils.getPathFileName(dest, idMax + fileHeaderSuffix);
-            // 检查是否已标准抓住
-            Data data = new Data();
-            BeanUtils.copyProperties(dataInfo, data);
-            if (checkLabeled(pathFileName)) {
-                data.setLabeled(Boolean.TRUE);
-            } else {
-                data.setLabeled(Boolean.FALSE);
-            }
-            int lastSlashIndex = fileName.lastIndexOf('/');
-            if (lastSlashIndex != -1) {
-                fileName = fileName.substring(lastSlashIndex + 1);
-            }
-            data.setName(fileName);
-            data.setUrl(pathFileName);
-            dataList.add(data);
-            return true;
-        }
-        return false;
+        return dataService.uploadDataInfo(file,dataInfo);
     }
-
-    // 检查有没有标注信息有的话把标注数据置为已标注
-    private boolean checkLabeled(String pathFileName) {
-        Path newPath = FileUtils.getTxtPath(pathFileName);
-
-        // 检查替换后的文件是否存在
-        return Files.exists(newPath);
-    }
-
 }

taais-modules/taais-biz/src/main/java/com/taais/biz/domain/Data.java

@@ -26,7 +26,6 @@ public class Data {
     /** 名称 */
     private String name;
 
-
     /** 数据类型 */
     private String dataType;
 
@@ -68,4 +67,5 @@ public class Data {
 //    @Column(onInsertValue = "false")
     private Boolean labeled;
 
+
 }

taais-modules/taais-biz/src/main/java/com/taais/biz/mapper/DataMapper.java

@@ -3,6 +3,9 @@ package com.taais.biz.mapper;
 import com.mybatisflex.core.BaseMapper;
 import org.apache.ibatis.annotations.Mapper;
 import com.taais.biz.domain.Data;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
 
 /**
  * 数据管理Mapper接口
@@ -13,5 +16,5 @@ import com.taais.biz.domain.Data;
 @Mapper
 public interface DataMapper extends BaseMapper<Data> {
 
-    Integer getIdMax();
+    List<Long> getIds(@Param("size") Integer size);
 }

taais-modules/taais-biz/src/main/java/com/taais/biz/service/IDataService.java

@@ -33,8 +33,6 @@ public interface IDataService extends IBaseService<Data> {
      */
     List<DataVo> selectList(DataBo dataBo);
 
-    Integer getIdMax();
-
     /**
      * 上传数据集
      *

taais-modules/taais-biz/src/main/java/com/taais/biz/service/impl/DataServiceImpl.java

@@ -256,11 +256,6 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
         return exists;
     }
 
-    @Override
-    public Integer getIdMax() {
-        return dataMapper.getIdMax();
-    }
-
     /**
      * 分页查询数据管理列表
      *

taais-modules/taais-biz/src/main/resources/mapper/biz/DataMapper.xml

@@ -4,7 +4,9 @@
     "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="com.taais.biz.mapper.DataMapper">
 
-    <select id="getIdMax" resultType="java.lang.Integer">
-        select max(id) from data
+    <select id="getIds" resultType="java.lang.Long">
+        SELECT nextval('public.data_id_seq') AS sequence_value
+        FROM
+            generate_series(1, #{size})
     </select>
 </mapper>