feat:添加新功能

taais-modules/taais-biz/src/main/java/com/taais/biz/controller/DataController.java

@@ -1,15 +1,8 @@
 package com.taais.biz.controller;
 
 import cn.dev33.satoken.annotation.SaCheckPermission;
-import com.taais.common.core.config.TaaisConfig;
-import com.taais.common.core.constant.Constants;
 import com.taais.common.core.core.domain.CommonResult;
 import com.taais.common.core.core.page.PageResult;
-import com.taais.common.core.utils.MapstructUtils;
-import com.taais.common.core.utils.StringUtils;
-import com.taais.common.core.utils.file.FileUploadUtils;
-import com.taais.common.core.utils.file.FileUtils;
-import com.taais.common.core.utils.file.UnPackedUtil;
 import com.taais.common.excel.utils.ExcelUtil;
 import com.taais.common.log.annotation.Log;
 import com.taais.common.log.enums.BusinessType;
@@ -20,24 +13,18 @@ import com.taais.biz.domain.bo.DataBo;
 import com.taais.biz.domain.vo.DataVo;
 import com.taais.biz.service.IDataService;
 import com.taais.system.config.ServerConfig;
-import com.taais.system.domain.vo.SysOssUploadVo;
 import jakarta.annotation.Resource;
 import jakarta.servlet.http.HttpServletResponse;
 import lombok.RequiredArgsConstructor;
-import net.lingala.zip4j.model.FileHeader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
-import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 /**
  * 数据管理Controller
@@ -50,6 +37,7 @@ import java.util.Map;
 @RestController
 @RequestMapping("/demo/data")
 public class DataController extends BaseController {
+    private static final Logger log = LoggerFactory.getLogger(DataController.class);
     @Resource
     private IDataService dataService;
     @Autowired
@@ -137,90 +125,10 @@ public class DataController extends BaseController {
     }
 
     @PostMapping("/zip/upload")
-    public CommonResult<SysOssUploadVo> uploadZipFile(@RequestParam("file") MultipartFile file, @RequestParam("excelFile") MultipartFile excelFile) throws Exception {
-
-        List<DataVo> dataVos = ExcelUtil.importExcel(excelFile.getInputStream(),DataVo.class);
-        Map<String, Data> picName2Obj = new HashMap<>();
-        //为插入数据库做准备
-        for(DataVo dataVo:dataVos){
-            picName2Obj.put(dataVo.getName(), MapstructUtils.convert(dataVo, Data.class));
+    public CommonResult<Boolean> uploadZipFile(@RequestParam("file") MultipartFile file, Data dataInfo) throws Exception {
+        if (file == null) {
+            return CommonResult.fail("请上传.zip、.rar压缩文件。");
         }
-        String  originalFilename = file.getOriginalFilename();
-        assert originalFilename != null;
-        String suffix = StringUtils.substring(originalFilename, originalFilename.lastIndexOf("."), originalFilename.length());
-
-
-        // 解压
-        String basedir = TaaisConfig.getUploadPath();
-        File zipOrRarTemp = FileUploadUtils.getAbsoluteFile(basedir,FileUploadUtils.extractFilename2(file));// 解压目录
-        String destZip = zipOrRarTemp.getAbsolutePath();
-
-        file.transferTo(Paths.get(destZip));
-        String dest = zipOrRarTemp.getParent();
-
-        List<File> extractedFileList = new ArrayList<File>();
-        if(suffix.equals(".zip")){
-            List<FileHeader> fileheaders = UnPackedUtil.unPackZip(zipOrRarTemp,dest);
-            //添加解压目录
-            for(FileHeader fileHeader : fileheaders) {
-                if (!fileHeader.isDirectory()) {
-                    String fileHeaderName = fileHeader.getFileName();
-                    String fileHeaderSuffix = StringUtils.substring(fileHeaderName, fileHeaderName.lastIndexOf("."), fileHeaderName.length());
-                    if(!fileHeaderSuffix.equals(".jpg")&&!fileHeaderSuffix.equals(".jpeg")
-                        &&!fileHeaderSuffix.equals(".png")){
-                        continue;
-                    }
-                    extractedFileList.add(new File(dest,fileHeader.getFileName()));
-                }
-            }
-
-
-
-        }else if(suffix.equals(".rar")){
-            List<com.github.junrar.rarfile.FileHeader> fileheaders = UnPackedUtil.unPackRar(zipOrRarTemp,dest);
-
-
-            //添加解压目录
-            for(com.github.junrar.rarfile.FileHeader fileHeader : fileheaders) {
-
-                if (!fileHeader.isDirectory()) {
-                    String fileHeaderName = fileHeader.getFileName();
-                    String fileHeaderSuffix = StringUtils.substring(fileHeaderName, fileHeaderName.lastIndexOf("."), fileHeaderName.length());
-                    if(!fileHeaderSuffix.equals(".jpg")&&!fileHeaderSuffix.equals(".jpeg")
-                        &&!fileHeaderSuffix.equals(".png")){
-                        continue;
-                    }
-                    extractedFileList.add(new File(dest,fileHeader.getFileName()));
-                }
-            }
-        }
-
-        for(File efile:extractedFileList) {
-            String pathFileName = FileUploadUtils.getPathFileName(dest,efile.getName());
-            Data mydata = picName2Obj.get(efile.getName());
-
-            // 检查是否已标准抓住
-            if(checkLabeled(pathFileName)){
-                mydata.setLabeled(Boolean.TRUE);
-            } else {
-                mydata.setLabeled(Boolean.FALSE);
-            }
-            mydata.setUrl(pathFileName);
-            dataService.save(mydata);
-        }
-        //删除压缩文件
-        FileUtils.deleteFile(destZip);
-
-
-        return CommonResult.success("数据集上传成功！");
+        return dataService.uploadDataInfo(file,dataInfo);
     }
-
-    // 检查有没有标注信息有的话把标注数据置为已标注
-    private boolean checkLabeled(String pathFileName) {
-        Path newPath = FileUtils.getTxtPath(pathFileName);
-
-        // 检查替换后的文件是否存在
-        return Files.exists(newPath);
-    }
-
 }

taais-modules/taais-biz/src/main/java/com/taais/biz/domain/Data.java

@@ -21,8 +21,6 @@ public class Data {
 //    @Serial
 //    private static final long serialVersionUID = 1L;
 
-    /** id */
-    @Id(value = "id",keyType = KeyType.Auto)
     private Long id;
 
     /** 名称 */
@@ -41,7 +39,7 @@ public class Data {
     private String objectSubtype;
 
     /** 批次号 */
-    private Long batchNum;
+    private String batchNum;
 
     /** 场景 */
     private String scene;

taais-modules/taais-biz/src/main/java/com/taais/biz/mapper/DataMapper.java

@@ -3,6 +3,9 @@ package com.taais.biz.mapper;
 import com.mybatisflex.core.BaseMapper;
 import org.apache.ibatis.annotations.Mapper;
 import com.taais.biz.domain.Data;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
 
 /**
  * 数据管理Mapper接口
@@ -13,4 +16,5 @@ import com.taais.biz.domain.Data;
 @Mapper
 public interface DataMapper extends BaseMapper<Data> {
 
+    List<Long> getIds(@Param("size") Integer size);
 }

taais-modules/taais-biz/src/main/java/com/taais/biz/service/IDataService.java

@@ -5,8 +5,10 @@ import java.util.List;
 import com.taais.biz.domain.Data;
 import com.taais.biz.domain.vo.DataVo;
 import com.taais.biz.domain.bo.DataBo;
+import com.taais.common.core.core.domain.CommonResult;
 import com.taais.common.orm.core.service.IBaseService;
 import com.taais.common.core.core.page.PageResult;
+import org.springframework.web.multipart.MultipartFile;
 
 /**
  * 数据管理Service接口
@@ -31,6 +33,15 @@ public interface IDataService extends IBaseService<Data> {
      */
     List<DataVo> selectList(DataBo dataBo);
 
+    /**
+     * 上传数据集
+     *
+     * @param file 文件
+     * @param dataInfo 数据表单
+     * @return 数据管理集合
+     */
+    CommonResult<Boolean> uploadDataInfo(MultipartFile file, Data dataInfo);
+
     /**
      * 分页查询数据管理列表
      *

taais-modules/taais-biz/src/main/java/com/taais/biz/service/impl/DataServiceImpl.java

@@ -8,15 +8,33 @@ import com.taais.biz.domain.bo.DataBo;
 import com.taais.biz.domain.vo.DataVo;
 import com.taais.biz.mapper.DataMapper;
 import com.taais.biz.service.IDataService;
+import com.taais.common.core.config.TaaisConfig;
+import com.taais.common.core.core.domain.CommonResult;
 import com.taais.common.core.core.page.PageResult;
 import com.taais.common.core.utils.MapstructUtils;
+import com.taais.common.core.utils.StringUtils;
+import com.taais.common.core.utils.file.FileUploadUtils;
+import com.taais.common.core.utils.file.FileUtils;
+import com.taais.common.core.utils.file.UnPackedUtil;
 import com.taais.common.orm.core.page.PageQuery;
 import com.taais.common.orm.core.service.impl.BaseServiceImpl;
 import jakarta.annotation.Resource;
+import net.lingala.zip4j.model.FileHeader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
 
-import java.util.Arrays;
-import java.util.List;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Instant;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static com.taais.biz.domain.table.DataTableDef.DATA;
 
@@ -28,6 +46,11 @@ import static com.taais.biz.domain.table.DataTableDef.DATA;
  */
 @Service
 public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implements IDataService {
+    private static final Logger log = LoggerFactory.getLogger(DataServiceImpl.class);
+    private static final String ZIP = ".zip";
+    private static final String RAR = ".rar";
+    private static final String TXT = ".txt";
+    private static final String[] VALID_EXTENSIONS = {".jpg", ".jpeg", ".png"};
     @Resource
     private DataMapper dataMapper;
 
@@ -39,33 +62,33 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
     private QueryWrapper buildQueryWrapper(DataBo dataBo) {
         QueryWrapper queryWrapper = super.buildBaseQueryWrapper();
         queryWrapper.and(DATA.NAME.like
-        (dataBo.getName()));
+                (dataBo.getName()));
         queryWrapper.and(DATA.DATA_TYPE.eq
-        (dataBo.getDataType()));
+                (dataBo.getDataType()));
         queryWrapper.and(DATA.FILE_TYPE.eq
-        (dataBo.getFileType()));
+                (dataBo.getFileType()));
         queryWrapper.and(DATA.OBJECT_TYPE.eq
-        (dataBo.getObjectType()));
+                (dataBo.getObjectType()));
         queryWrapper.and(DATA.OBJECT_SUBTYPE.eq
-        (dataBo.getObjectSubtype()));
+                (dataBo.getObjectSubtype()));
         queryWrapper.and(DATA.BATCH_NUM.eq
-        (dataBo.getBatchNum()));
+                (dataBo.getBatchNum()));
         queryWrapper.and(DATA.SCENE.eq
-        (dataBo.getScene()));
+                (dataBo.getScene()));
         queryWrapper.and(DATA.DATA_SOURCE.eq
-        (dataBo.getDataSource()));
+                (dataBo.getDataSource()));
         queryWrapper.and(DATA.GATHER_TIME.eq
-        (dataBo.getGatherTime()));
+                (dataBo.getGatherTime()));
         queryWrapper.and(DATA.GATHER_SPOT.eq
-        (dataBo.getGatherSpot()));
+                (dataBo.getGatherSpot()));
         queryWrapper.and(DATA.URL.eq
-        (dataBo.getUrl()));
+                (dataBo.getUrl()));
         queryWrapper.and(DATA.LABELURL.eq
-        (dataBo.getLabelurl()));
+                (dataBo.getLabelurl()));
         queryWrapper.and(DATA.INCREMENT.eq
-        (dataBo.getIncrement()));
+                (dataBo.getIncrement()));
         queryWrapper.and(DATA.LABELED.eq
-        (dataBo.getLabeled()));
+                (dataBo.getLabeled()));
 
         return queryWrapper;
     }
@@ -78,7 +101,7 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
      */
     @Override
     public DataVo selectById(Long id) {
-            return this.getOneAs(query().where(DATA.ID.eq(id)), DataVo.class);
+        return this.getOneAs(query().where(DATA.ID.eq(id)), DataVo.class);
 
     }
 
@@ -91,7 +114,146 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
     @Override
     public List<DataVo> selectList(DataBo dataBo) {
         QueryWrapper queryWrapper = buildQueryWrapper(dataBo);
-            return this.listAs(queryWrapper, DataVo.class);
+        return this.listAs(queryWrapper, DataVo.class);
+    }
+
+    @Override
+    public CommonResult<Boolean> uploadDataInfo(MultipartFile file, Data dataInfo) {
+        //1.检测是否有重复的批次号
+        QueryWrapper query = query();
+        query.eq(Data::getBatchNum, dataInfo.getBatchNum());
+        long count = dataMapper.selectCountByQuery(query);
+        if (count > 0) {
+            return CommonResult.fail("检测出重复批次号，请仔细检查后重新导入！");
+        }
+        try {
+            Boolean labeled = dataInfo.getLabeled();
+            //2.解压图片并检查
+            String originalFilename = file.getOriginalFilename();
+            assert originalFilename != null;
+            String suffix = StringUtils.substring(originalFilename, originalFilename.lastIndexOf("."), originalFilename.length());
+            // 解压
+            String basedir = TaaisConfig.getUploadPath();
+            // 解压目录
+            File zipOrRarTemp = FileUploadUtils.getAbsoluteFile(basedir, FileUploadUtils.extractFilename2(file));
+            String destZip = zipOrRarTemp.getAbsolutePath();
+            file.transferTo(Paths.get(destZip));
+            String dest = zipOrRarTemp.getParent();
+            // 解压图片集合
+            List<File> extractedImagesFileList = new ArrayList<>();
+            if (ZIP.equals(suffix)) {
+                //添加解压目录
+                for (FileHeader fileHeader : UnPackedUtil.unPackZip(zipOrRarTemp, dest)) {
+                    initFileInfo(dest, extractedImagesFileList, fileHeader.isDirectory(), fileHeader.getFileName());
+                }
+            } else if (RAR.equals(suffix)) {
+                //添加解压目录
+                for (com.github.junrar.rarfile.FileHeader fileHeader : UnPackedUtil.unPackRar(zipOrRarTemp, dest)) {
+                    initFileInfo(dest, extractedImagesFileList, fileHeader.isDirectory(), fileHeader.getFileName());
+                }
+            } else {
+                return CommonResult.fail("请上传.zip、.rar压缩文件。！");
+            }
+
+            //检测图片文件是否为空
+            if (extractedImagesFileList.isEmpty()) {
+                return CommonResult.fail("压缩文件图片为空，请检查后重新上传！");
+            }
+
+            //3获取ID集合
+            List<Long> ids = dataMapper.getIds(extractedImagesFileList.size());
+            if (ids.isEmpty()) {
+                return CommonResult.fail("系统异常！");
+            }
+
+            List<Boolean> labeledList = new ArrayList<>();
+            List<Data> dataList = new ArrayList<>();
+            AtomicInteger countSize = new AtomicInteger();
+            extractedImagesFileList.forEach(fileInfo -> {
+                //获取ID
+                Long id = ids.get(countSize.get());
+                Data data = new Data();
+                //拷贝dataInfo
+                BeanUtils.copyProperties(dataInfo, data);
+                //检测是否标注
+                if (checkLabeled(fileInfo.getPath())) {
+                    labeledList.add(Boolean.TRUE);
+                    data.setLabeled(Boolean.TRUE);
+                } else {
+                    labeledList.add(Boolean.FALSE);
+                    data.setLabeled(Boolean.FALSE);
+                }
+                try {
+                    Path path = Paths.get(fileInfo.getAbsolutePath());
+                    // 使用Files类的readAttributes方法获取文件的基本属性
+                    BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class);
+                    // 获取文件的创建时间
+                    Instant creationTime = attrs.creationTime().toInstant();
+                    // 将Instant转换为Date
+                    Date date = Date.from(creationTime);
+                    //设置文件创建时间
+                    data.setId(id);
+                    data.setGatherTime(date);
+                    data.setName(fileInfo.getName());
+                    //更改图片文件名称
+                    String fileHeaderSuffix = StringUtils.substring(fileInfo.getName(), fileInfo.getName().lastIndexOf("."), fileInfo.getName().length());
+                    String destInfo = fileInfo.getPath().replaceAll(fileInfo.getName(), "");
+                    File newFile = new File(destInfo, id + fileHeaderSuffix);
+                    File odlFile = new File(destInfo, fileInfo.getName());
+                    log.info("更改用户上传图片文件名称:{}", odlFile.renameTo(newFile));
+                    String imagePath = FileUploadUtils.getPathFileName(destInfo, id + fileHeaderSuffix);
+                    data.setUrl(imagePath);
+                    //该图片有标注，更改标注文件名称
+                    if (data.getLabeled()) {
+                        String labeledPath = fileInfo.getPath().replaceFirst("[.][^.]+$", "") + ".txt";
+                        File labeledNewFile = new File(destInfo, id + ".txt");
+                        File labeledOdlFile = new File(labeledPath);
+                        log.info("更改用户上传标注文件名称:{}", labeledOdlFile.renameTo(labeledNewFile));
+                        String labelUrl = FileUploadUtils.getPathFileName(destInfo, id + ".txt");
+                        data.setLabelurl(labelUrl);
+                    }
+                    dataList.add(data);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+                countSize.getAndIncrement();
+            });
+            //是否选择已标注，如果已标注则需要检测所有图片是否标注
+            if (!labeledList.isEmpty() && labeled) {
+                // 已标注数量
+                long unmarkedCount = labeledList.stream().filter(Boolean.FALSE::equals).count();
+                // 未标注数量
+                long markedCount = labeledList.stream().filter(Boolean.TRUE::equals).count();
+                // 如果存在未标注的数据，则返回错误信息
+                if (unmarkedCount > 0) {
+                    String format = String.format("错误: 已标注文件 %d 个，未标注文件 %d 个", markedCount, unmarkedCount);
+                    return CommonResult.fail(format);
+                }
+            }
+            dataMapper.insertBatch(dataList);
+            FileUtils.deleteFile(destZip);
+        } catch (Exception e) {
+            log.error("[uploadDataInfo]数据集处理出现未知异常.e:", e);
+            return CommonResult.fail("系统异常！");
+        }
+        return CommonResult.success("数据集上传成功！");
+    }
+
+    private void initFileInfo(String dest, List<File> extractedImagesFileList, boolean directory, String fileName) {
+        if (!directory) {
+            String fileHeaderSuffix = StringUtils.substring(fileName, fileName.lastIndexOf("."), fileName.length());
+            if (Arrays.asList(VALID_EXTENSIONS).contains(fileHeaderSuffix)) {
+                extractedImagesFileList.add(new File(dest, fileName));
+            }
+        }
+    }
+
+    private boolean checkLabeled(String pathFileName) {
+        Path newPath = Paths.get(pathFileName.replaceFirst("[.][^.]+$", "") + ".txt");
+        // 检查替换后的文件是否存在
+        boolean exists = Files.exists(newPath);
+        log.info("[checkLabeled]图片文件路径名称:{},标注文件路径名称:{},是否标注:{}", pathFileName, newPath, exists);
+        return exists;
     }
 
     /**
@@ -103,7 +265,7 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
     @Override
     public PageResult<DataVo> selectPage(DataBo dataBo) {
         QueryWrapper queryWrapper = buildQueryWrapper(dataBo);
-            Page<DataVo> page = this.pageAs(PageQuery.build(), queryWrapper, DataVo.class);
+        Page<DataVo> page = this.pageAs(PageQuery.build(), queryWrapper, DataVo.class);
         return PageResult.build(page);
     }
 
@@ -128,10 +290,10 @@ public class DataServiceImpl extends BaseServiceImpl<DataMapper, Data> implement
      */
     @Override
     public boolean update(DataBo dataBo) {
-        Data data =MapstructUtils.convert(dataBo, Data. class);
-        if (ObjectUtil.isNotNull(data) && ObjectUtil.isNotNull(data.getId())){
+        Data data = MapstructUtils.convert(dataBo, Data.class);
+        if (ObjectUtil.isNotNull(data) && ObjectUtil.isNotNull(data.getId())) {
             boolean updated = this.updateById(data);
-                return updated;
+            return updated;
         }
         return false;
     }

taais-modules/taais-biz/src/main/resources/mapper/biz/DataMapper.xml

@@ -4,4 +4,9 @@
     "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
 <mapper namespace="com.taais.biz.mapper.DataMapper">
 
+    <select id="getIds" resultType="java.lang.Long">
+        SELECT nextval('public.data_id_seq') AS sequence_value
+        FROM
+            generate_series(1, #{size})
+    </select>
 </mapper>