Merge remote-tracking branch 'origin/dev-lzy' into dev-lzy

# Conflicts:
#	taais-modules/taais-biz/src/main/java/com/taais/biz/controller/DataController.java
#	taais-modules/taais-biz/src/main/java/com/taais/biz/mapper/DataMapper.java
#	taais-modules/taais-biz/src/main/resources/mapper/biz/DataMapper.xml

taais-admin/src/main/resources/application-dev.yml

@@ -35,7 +35,7 @@ mybatis-flex:
       #      password: Root@369
       #postgresql数据库
       driver-class-name: org.postgresql.Driver
-      url: jdbc:postgresql://192.168.101.34:5432/taais?useUnicode=true&characterEncoding=utf8&useSSL=true&autoReconnect=true&reWriteBatchedInserts=true
+      url: jdbc:postgresql://110.41.34.83:5432/taais?useUnicode=true&characterEncoding=utf8&useSSL=true&autoReconnect=true&reWriteBatchedInserts=true
       username: postgres
       password: P3x0LG8jzyHRX59l
 
@@ -58,13 +58,13 @@ mybatis-flex:
 spring.data:
   redis:
     # 地址
-    host: localhost
+    host: 110.41.34.83
     # 端口，默认为6379
     port: 6379
     # 数据库索引
     database: 0
     # 密码(如没有密码请注释掉)
-    password: CWwYsGjafmv8Sr7g
+    password: Z;G4AS:Vor'YF#p?
     # 连接超时时间
     timeout: 10s
     # 是否开启ssl

taais-modules/taais-biz/src/main/java/com/taais/biz/controller/DataController.java

@@ -1,8 +1,14 @@
 package com.taais.biz.controller;
 
 import cn.dev33.satoken.annotation.SaCheckPermission;
+import com.mybatisflex.core.query.QueryWrapper;
+import com.taais.common.core.config.TaaisConfig;
 import com.taais.common.core.core.domain.CommonResult;
 import com.taais.common.core.core.page.PageResult;
+import com.taais.common.core.utils.StringUtils;
+import com.taais.common.core.utils.file.FileUploadUtils;
+import com.taais.common.core.utils.file.FileUtils;
+import com.taais.common.core.utils.file.UnPackedUtil;
 import com.taais.common.excel.utils.ExcelUtil;
 import com.taais.common.log.annotation.Log;
 import com.taais.common.log.enums.BusinessType;
@@ -13,18 +19,29 @@ import com.taais.biz.domain.bo.DataBo;
 import com.taais.biz.domain.vo.DataVo;
 import com.taais.biz.service.IDataService;
 import com.taais.system.config.ServerConfig;
+import com.taais.system.domain.vo.SysOssUploadVo;
 import jakarta.annotation.Resource;
 import jakarta.servlet.http.HttpServletResponse;
 import lombok.RequiredArgsConstructor;
+import net.lingala.zip4j.model.FileHeader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * 数据管理Controller
@@ -124,11 +141,143 @@ public class DataController extends BaseController {
         ExcelUtil.exportExcel(new ArrayList<>(), "用户数据", DataVo.class, response);
     }
 
+    /**
+     * todo Eureka
+     * 代码放在service层
+     * 1 检查是否有重复的批次号
+     * 2 解压图片并检查
+     * 2.1 解压缩包，并获取所有图片
+     * 2.2 如果选择了已标注，检查所有图片是否都有标注
+     * 3 入库前逻辑
+     * 3.1 根据图片个数获取主键序列
+     * 3.2 循环图片
+     * 3.2.1 深克隆dataInfo数据，把图片名称保存在clone.name里，把主键序列放入id、更新图片和标注名称和主键id保持一致，把图片的文件路径存入url，把标注的文件路径存入labelurl
+     * 3.2.2 获取当前图片的创建时间 存入clone.gatherTime，如果该图片有标注的txt 设置labeled为true，否则为false
+     * 4 入库
+     * @param file
+     * @param dataInfo
+     * @return
+     * @throws Exception
+     */
     @PostMapping("/zip/upload")
-    public CommonResult<Boolean> uploadZipFile(@RequestParam("file") MultipartFile file, Data dataInfo) throws Exception {
-        if (file == null) {
-            return CommonResult.fail("请上传.zip、.rar压缩文件。");
+    public CommonResult<SysOssUploadVo> uploadZipFile(@RequestParam("file") MultipartFile file, Data dataInfo) throws Exception {
+        String originalFilename = file.getOriginalFilename();
+        assert originalFilename != null;
+        String suffix = StringUtils.substring(originalFilename, originalFilename.lastIndexOf("."), originalFilename.length());
+        // 解压
+        String basedir = TaaisConfig.getUploadPath();
+        // 解压目录
+        File zipOrRarTemp = FileUploadUtils.getAbsoluteFile(basedir, FileUploadUtils.extractFilename2(file));
+        String destZip = zipOrRarTemp.getAbsolutePath();
+
+        file.transferTo(Paths.get(destZip));
+        String dest = zipOrRarTemp.getParent();
+
+        // todo Eureka 改成使用sequence获取id，使用id作为图片的名称和data的id，此处的逻辑在并发场景会获取相同的idMax会出现异常
+        //取出最大值
+        Integer idMax = dataService.getIdMax();
+        if (idMax == null) {
+            idMax = 1;
+        } else {
+            idMax = idMax + 1;
+        }
+
+        List<Data> dataList = new ArrayList<>();
+        if (suffix.equals(".zip")) {
+            List<FileHeader> fileheaders = UnPackedUtil.unPackZip(zipOrRarTemp, dest);
+            //添加解压目录
+            for (FileHeader fileHeader : fileheaders) {
+                boolean isOk = initFileInfo(dest, idMax, fileHeader.isDirectory(), fileHeader.getFileName(), dataList, dataInfo);
+                if (isOk) {
+                    idMax++;
+                }
+            }
+        } else if (suffix.equals(".rar")) {
+            List<com.github.junrar.rarfile.FileHeader> fileheaders = UnPackedUtil.unPackRar(zipOrRarTemp, dest);
+            //添加解压目录
+            for (com.github.junrar.rarfile.FileHeader fileHeader : fileheaders) {
+                boolean isOk = initFileInfo(dest, idMax, fileHeader.isDirectory(), fileHeader.getFileName(), dataList, dataInfo);
+                if (isOk) {
+                    idMax++;
+                }
+            }
+        }
+
+        if (dataInfo.getLabeled()) {
+            List<Data> unmarkedData = dataList.stream()
+                    .filter(data -> Boolean.FALSE.equals(data.getLabeled()))
+                    .toList();
+
+            // 计算已标注和未标注的数据数量
+            long markedCount = dataList.stream()
+                    .filter(data -> Boolean.TRUE.equals(data.getLabeled()))
+                    .count();
+            long unmarkedCount = unmarkedData.size();
+
+            // 如果存在未标注的数据，则返回错误信息
+            if (unmarkedCount > 0) {
+                String format = String.format("错误: 已标注文件 %d 个，未标注文件 %d 个", markedCount, unmarkedCount);
+                return CommonResult.fail(format);
+            }
         }
-        return dataService.uploadDataInfo(file,dataInfo);
+
+        // todo Eureka 不用循环检查，所有批次都是一样的，这个检查放在最开始
+        AtomicBoolean isTrue = new AtomicBoolean(false);
+        dataList.forEach(data -> {
+            QueryWrapper query = dataService.query();
+            query.eq(Data::getBatchNum, data.getBatchNum());
+            long count = dataService.getMapper().selectCountByQuery(query);
+            if (count > 0) {
+                isTrue.set(true);
+            }
+        });
+
+        if (isTrue.get()) {
+            return CommonResult.fail("文件检测出重复批次号，请仔细检查后重新导入！");
+        }
+        dataService.saveBatch(dataList);
+        //删除压缩文件
+        FileUtils.deleteFile(destZip);
+        return CommonResult.success("数据集上传成功！");
+    }
+
+    private boolean initFileInfo(String dest, Integer idMax, boolean directory, String fileName, List<Data> dataList, Data dataInfo) throws IOException {
+        if (!directory) {
+            String fileHeaderSuffix = StringUtils.substring(fileName, fileName.lastIndexOf("."), fileName.length());
+            // todo Eureka add bpm
+            if (!fileHeaderSuffix.equals(".jpg") && !fileHeaderSuffix.equals(".jpeg") && !fileHeaderSuffix.equals(".png")) {
+                return false;
+            }
+            File file = new File(dest, idMax + fileHeaderSuffix);
+            File odlFile = new File(dest, fileName);
+            log.info("更改用户上传文件名称:{}", odlFile.renameTo(file));
+            String pathFileName = FileUploadUtils.getPathFileName(dest, idMax + fileHeaderSuffix);
+            // 检查是否已标准抓住
+            Data data = new Data();
+            BeanUtils.copyProperties(dataInfo, data);
+            if (checkLabeled(pathFileName)) {
+                data.setLabeled(Boolean.TRUE);
+            } else {
+                data.setLabeled(Boolean.FALSE);
+            }
+            int lastSlashIndex = fileName.lastIndexOf('/');
+            if (lastSlashIndex != -1) {
+                fileName = fileName.substring(lastSlashIndex + 1);
+            }
+            data.setName(fileName);
+            data.setUrl(pathFileName);
+            dataList.add(data);
+            return true;
+        }
+        return false;
     }
+
+    // 检查有没有标注信息有的话把标注数据置为已标注
+    private boolean checkLabeled(String pathFileName) {
+        Path newPath = FileUtils.getTxtPath(pathFileName);
+
+        // 检查替换后的文件是否存在
+        return Files.exists(newPath);
+    }
+
 }

taais-modules/taais-biz/src/main/java/com/taais/biz/domain/Data.java

@@ -26,6 +26,7 @@ public class Data {
     /** 名称 */
     private String name;
 
+
     /** 数据类型 */
     private String dataType;
 
@@ -67,5 +68,4 @@ public class Data {
 //    @Column(onInsertValue = "false")
     private Boolean labeled;
 
-
 }

taais-modules/taais-biz/src/main/java/com/taais/biz/mapper/DataMapper.java

@@ -3,9 +3,6 @@ package com.taais.biz.mapper;
 import com.mybatisflex.core.BaseMapper;
 import org.apache.ibatis.annotations.Mapper;
 import com.taais.biz.domain.Data;
-import org.apache.ibatis.annotations.Param;
-
-import java.util.List;
 
 /**
  * 数据管理Mapper接口
@@ -16,5 +13,5 @@ import java.util.List;
 @Mapper
 public interface DataMapper extends BaseMapper<Data> {
 
-    List<Long> getIds(@Param("size") Integer size);
+    Integer getIdMax();
 }

taais-modules/taais-biz/src/main/java/com/taais/biz/service/IDataService.java

@@ -33,6 +33,8 @@ public interface IDataService extends IBaseService<Data> {
      */
     List<DataVo> selectList(DataBo dataBo);
 
+    Integer getIdMax();
+
     /**
      * 上传数据集
      *

taais-modules/taais-biz/src/main/java/com/taais/biz/service/impl/DataServiceImpl.java

@@ -256,6 +256,11 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
         return exists;
     }
 
+    @Override
+    public Integer getIdMax() {
+        return dataMapper.getIdMax();
+    }
+
     /**
      * 分页查询数据管理列表
      *

taais-modules/taais-biz/src/main/resources/mapper/biz/DataMapper.xml

@@ -4,9 +4,7 @@
     "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="com.taais.biz.mapper.DataMapper">
 
-    <select id="getIds" resultType="java.lang.Long">
-        SELECT nextval('public.data_id_seq') AS sequence_value
-        FROM
-            generate_series(1, #{size})
+    <select id="getIdMax" resultType="java.lang.Integer">
+        select max(id) from data
     </select>
 </mapper>