|
|
@@ -0,0 +1,339 @@
|
|
|
+package org.eco.als.utils;
|
|
|
+
|
|
|
+import cn.hutool.core.collection.CollectionUtil;
|
|
|
+import cn.hutool.core.io.FileUtil;
|
|
|
+import cn.hutool.core.map.MapUtil;
|
|
|
+import cn.hutool.core.text.csv.CsvData;
|
|
|
+import cn.hutool.core.text.csv.CsvReader;
|
|
|
+import cn.hutool.core.text.csv.CsvRow;
|
|
|
+import cn.hutool.core.text.csv.CsvUtil;
|
|
|
+import cn.hutool.core.util.CharsetUtil;
|
|
|
+import cn.hutool.core.util.StrUtil;
|
|
|
+import cn.hutool.json.JSONArray;
|
|
|
+import cn.hutool.json.JSONObject;
|
|
|
+import cn.hutool.json.JSONUtil;
|
|
|
+import cn.hutool.poi.excel.ExcelReader;
|
|
|
+import cn.hutool.poi.excel.ExcelUtil;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.eco.common.core.config.EcoConfig;
|
|
|
+import org.eco.common.core.utils.DateUtils;
|
|
|
+import org.eco.common.core.utils.StringUtils;
|
|
|
+import org.eco.common.core.utils.uuid.Seq;
|
|
|
+
|
|
|
+import java.io.File;
|
|
|
+import java.io.InputStream;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Map;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+
|
|
|
+/**
|
|
|
+ * @Description: CsvUtils
|
|
|
+ * @Author: GaoKun Wang
|
|
|
+ * @Date: 2024/8/1
|
|
|
+ */
|
|
|
+@Slf4j
|
|
|
+public class CsvUtils {
|
|
|
+ public static final String CSV_TYPE = "csv";
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 编码文件名
|
|
|
+ */
|
|
|
+ public static final String extractFilename(String fileName) {
|
|
|
+ return StringUtils.format("{}/{}_{}.{}", DateUtils.datePath(), fileName, Seq.getId(Seq.uploadSeqType),
|
|
|
+ CSV_TYPE);
|
|
|
+ }
|
|
|
+
|
|
|
+ public static final String getFilename(String fileName) {
|
|
|
+ return StringUtils.format("{}/{}.{}", DateUtils.datePath(), fileName, CSV_TYPE);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * json 转 csv数据
|
|
|
+ *
|
|
|
+ * @param jsonArray 数组
|
|
|
+ * @return csv 结果
|
|
|
+ */
|
|
|
+ public static List<String> jsonArrayToCsv(JSONArray jsonArray) {
|
|
|
+ // 获取CSV的表头
|
|
|
+ List<String> headers = new ArrayList<>(jsonArray.getJSONObject(0).keySet());
|
|
|
+ // 构造CSV行列表
|
|
|
+ List<String> csvLines = jsonArray.stream().map(jsonObject -> {
|
|
|
+ StringBuilder line = new StringBuilder();
|
|
|
+ for (String header : headers) {
|
|
|
+ // 根据表头获取对应的值，并追加到CSV行中
|
|
|
+ line.append(((JSONObject) jsonObject).get(header)).append(",");
|
|
|
+ }
|
|
|
+ // 移除末尾的逗号
|
|
|
+ return line.substring(0, line.length() - 1);
|
|
|
+ }).collect(Collectors.toList());
|
|
|
+ // 在CSV行列表开头添加表头
|
|
|
+ csvLines.add(0, String.join(",", headers));
|
|
|
+ return csvLines;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * json转csv文件
|
|
|
+ *
|
|
|
+ * @param jsonString data
|
|
|
+ * @param csvFilePath path
|
|
|
+ */
|
|
|
+ public static File jsonToFileCsv(String jsonString, String csvFilePath) {
|
|
|
+ try {
|
|
|
+ // 将JSON字符串转换为JSON数组
|
|
|
+ JSONArray jsonArray = JSONUtil.parseArray(jsonString);
|
|
|
+ // 将JSON数组写入CSV文件
|
|
|
+ return jsonToFileCsvByJsonArray(jsonArray, csvFilePath);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("json转csv文件错误：{}", e.getMessage());
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * json转csv文件
|
|
|
+ *
|
|
|
+ * @param jsonArray data
|
|
|
+ * @param csvFilePath path
|
|
|
+ */
|
|
|
+ public static File jsonToFileCsvByJsonArray(JSONArray jsonArray, String csvFilePath) {
|
|
|
+ try {
|
|
|
+ // 将JSON数组写入CSV文件
|
|
|
+ return FileUtil.writeLines(jsonArrayToCsv(jsonArray), csvFilePath, CharsetUtil.CHARSET_UTF_8);
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("json转csv文件错误：{}", e.getMessage());
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * csv文件转Json
|
|
|
+ *
|
|
|
+ * @param path path
|
|
|
+ */
|
|
|
+ public static JSONArray fileCsvToJson(String path) {
|
|
|
+ String csvFilePath = EcoConfig.getUploadPath() + path;
|
|
|
+ List<CsvRow> rows = getCsvRowList(csvFilePath);
|
|
|
+ // 获取CSV表头，即第一行数据
|
|
|
+ assert rows != null;
|
|
|
+ List<String> headers = rows.get(0).getRawList();
|
|
|
+ // 去除第一行，保留后续数据
|
|
|
+ List<CsvRow> dataLines = rows.subList(1, rows.size());
|
|
|
+ JSONArray jsonArray = new JSONArray();
|
|
|
+ for (CsvRow line : dataLines) {
|
|
|
+ JSONObject jsonObject = new JSONObject();
|
|
|
+ for (int i = 0; i < headers.size(); i++) {
|
|
|
+ jsonObject.putOnce(headers.get(i), line.getRawList().get(i));
|
|
|
+ }
|
|
|
+ jsonArray.add(jsonObject);
|
|
|
+ }
|
|
|
+ return jsonArray;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 处理异常值
|
|
|
+ *
|
|
|
+ * @param path path
|
|
|
+ */
|
|
|
+ public static JSONArray abnormal(String path) {
|
|
|
+ JSONArray jsonArray = fileCsvToJson(path);
|
|
|
+ for (int i = 0; i < jsonArray.size(); i++) {
|
|
|
+ // 获取JSONArray中的JSONObject
|
|
|
+ JSONObject jsonObject = jsonArray.getJSONObject(i);
|
|
|
+ // 用于存储要删除的 key
|
|
|
+ List<String> keysToRemove = new ArrayList<>();
|
|
|
+ // 获取JSONObject的所有key
|
|
|
+ for (String key : jsonObject.keySet()) {
|
|
|
+ if (StrUtil.contains(key, "_是否异常") && StrUtil.equals("1", jsonObject.get(key).toString())) {
|
|
|
+ jsonObject.set(StrUtil.subBefore(key, "_是否异常", true), "");
|
|
|
+ }
|
|
|
+ if (StrUtil.contains(key, "_是否异常")) {
|
|
|
+ // 添加要删除的 key 到列表
|
|
|
+ keysToRemove.add(key);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 移除要删除的 key
|
|
|
+ for (String keyToRemove : keysToRemove) {
|
|
|
+ jsonObject.remove(keyToRemove);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return jsonArray;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 获取csv文件中 参数名称
|
|
|
+ *
|
|
|
+ * @param csvFilePath csv文件地址
|
|
|
+ * @return 参数名称列表
|
|
|
+ */
|
|
|
+ public static List<String> getCsvHeaders(String csvFilePath) {
|
|
|
+ List<CsvRow> rows = getCsvRowList(EcoConfig.getUploadPath() + csvFilePath);
|
|
|
+ assert rows != null;
|
|
|
+ return rows.get(0).getRawList();
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 根据头名称(参数名称)获取csv文件中参数数据
|
|
|
+ *
|
|
|
+ * @param csvFilePath csv文件地址
|
|
|
+ * @param headerNames 参数名称
|
|
|
+ * @return 参数名称列表
|
|
|
+ */
|
|
|
+ public static JSONArray getCsvDataByHeaders(String csvFilePath, List<String> headerNames, Integer step) {
|
|
|
+ List<CsvRow> rows = getCsvRowList(EcoConfig.getUploadPath() + csvFilePath);
|
|
|
+ // 获取CSV表头，即第一行数据
|
|
|
+ List<String> headers = rows.get(0).getRawList();
|
|
|
+ List<Integer> indexList = new ArrayList<>();
|
|
|
+ headerNames.forEach(name -> {
|
|
|
+ int index = headers.indexOf(name);
|
|
|
+ if (index != -1) {
|
|
|
+ indexList.add(index);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ if (CollectionUtil.isEmpty(indexList)) {
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ // 去除第一行，保留后续数据
|
|
|
+ List<CsvRow> dataLines = rows.subList(1, rows.size());
|
|
|
+ JSONArray jsonArray = new JSONArray();
|
|
|
+ for (CsvRow line : dataLines) {
|
|
|
+ JSONObject jsonObject = new JSONObject();
|
|
|
+ for (int i = 0; i < headers.size(); i++) {
|
|
|
+ // 参数索引位置
|
|
|
+ if (indexList.contains(i)) {
|
|
|
+ jsonObject.putOnce(headers.get(i), line.getRawList().get(i));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ jsonArray.add(jsonObject);
|
|
|
+ }
|
|
|
+ if (step != null) {
|
|
|
+ jsonArray = reduceFrameRate(jsonArray, 10, step);
|
|
|
+ }
|
|
|
+ return jsonArray;
|
|
|
+ }
|
|
|
+
|
|
|
+ public static JSONObject getPlaybackByHeaders(String csvFilePath, List<String> headerNames, Integer step) {
|
|
|
+ List<CsvRow> rows = getCsvRowList(EcoConfig.getUploadPath() + csvFilePath);
|
|
|
+ // 获取CSV表头，即第一行数据
|
|
|
+ List<String> headers = rows.get(0).getRawList();
|
|
|
+ List<Integer> indexList = new ArrayList<>();
|
|
|
+ headerNames.forEach(name -> {
|
|
|
+ int index = headers.indexOf(name);
|
|
|
+ if (index != -1) {
|
|
|
+ indexList.add(index);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ if (CollectionUtil.isEmpty(indexList)) {
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ // 去除第一行，保留后续数据
|
|
|
+ List<CsvRow> dataLines = rows.subList(1, rows.size());
|
|
|
+ JSONObject jsonObject = new JSONObject();
|
|
|
+ for (int i = 0; i < headers.size(); i++) {
|
|
|
+ JSONArray jsonArray = new JSONArray();
|
|
|
+ for (CsvRow line : dataLines) {
|
|
|
+ jsonArray.add(line.getRawList().get(i));
|
|
|
+ }
|
|
|
+ if (step != null) {
|
|
|
+ jsonArray = reduceFrameRate(jsonArray, 10, step);
|
|
|
+ }
|
|
|
+ // 参数索引位置
|
|
|
+ if (indexList.contains(i)) {
|
|
|
+ jsonObject.putOnce(headers.get(i), jsonArray);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return jsonObject;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 处理数据帧
|
|
|
+ *
|
|
|
+ * @param originalData 原始数据
|
|
|
+ * @param originalFrameRate 原始帧率
|
|
|
+ * @param targetFrameRate 目标帧率
|
|
|
+ * @return 降采样后的数据
|
|
|
+ */
|
|
|
+ public static JSONArray reduceFrameRate(JSONArray originalData, int originalFrameRate, int targetFrameRate) {
|
|
|
+ int originalFrameCount = originalData.size();
|
|
|
+ int targetFrameCount = (int) Math.floor((double) originalFrameCount / originalFrameRate * targetFrameRate);
|
|
|
+ JSONArray result = new JSONArray();
|
|
|
+ double ratio = (double) originalFrameCount / targetFrameCount;
|
|
|
+ for (int i = 0; i < targetFrameCount; i++) {
|
|
|
+ int index = (int) Math.floor(i * ratio);
|
|
|
+ result.add(originalData.get(index));
|
|
|
+ }
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static List<CsvRow> getCsvRowList(String csvFilePath) {
|
|
|
+ try {
|
|
|
+ CsvReader reader = CsvUtil.getReader();
|
|
|
+ // 从文件中读取CSV数据
|
|
|
+ CsvData data = reader.read(FileUtil.file(csvFilePath), CharsetUtil.CHARSET_UTF_8);
|
|
|
+ return data.getRows();
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error(e.getMessage());
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * json转csv文件
|
|
|
+ *
|
|
|
+ * @param excelFilePath path
|
|
|
+ * @param csvFilePath path
|
|
|
+ */
|
|
|
+ public static void excelToFileCsv(String excelFilePath, String csvFilePath) {
|
|
|
+ // 读取Excel文件，获取ExcelReader
|
|
|
+ ExcelReader reader = ExcelUtil.getReader(excelFilePath);
|
|
|
+ // 通过ExcelReader将Excel文件读取为List<Map>
|
|
|
+ List<Map<String, Object>> readAll = reader.readAll();
|
|
|
+ String json = JSONUtil.toJsonStr(readAll);
|
|
|
+ jsonToFileCsv(json, csvFilePath);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * json转csv文件
|
|
|
+ *
|
|
|
+ * @param excelFilePath path
|
|
|
+ * @param columnNames names
|
|
|
+ */
|
|
|
+ public static File excelToFileCsvByColumns(String excelFilePath, String[] columnNames) {
|
|
|
+ return getFile(ExcelUtil.getReader(excelFilePath), columnNames, excelFilePath);
|
|
|
+ }
|
|
|
+
|
|
|
+ public static File excelToFileCsvByColumns(InputStream stream, String[] columnNames) {
|
|
|
+ String tempPath = "模型参数.csv";
|
|
|
+ return getFile(ExcelUtil.getReader(stream), columnNames, tempPath);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static File getFile(ExcelReader reader, String[] columnNames, String tempPath) {
|
|
|
+ // 通过ExcelReader将Excel文件读取为List<Map>
|
|
|
+ List<Map<String, Object>> readAll = filterColumns(reader.readAll(), columnNames);
|
|
|
+ String json = JSONUtil.toJsonStr(readAll);
|
|
|
+ return jsonToFileCsv(json, tempPath);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * filterColumns 根据列名称过滤数据
|
|
|
+ *
|
|
|
+ * @param originalData 原数据
|
|
|
+ * @param columnNames 列表名
|
|
|
+ * @return java.util.List<java.util.Map < java.lang.String, java.lang.Object>>
|
|
|
+ **/
|
|
|
+ private static List<Map<String, Object>> filterColumns(List<Map<String, Object>> originalData, String[] columnNames) {
|
|
|
+ List<Map<String, Object>> filteredData = new ArrayList<>();
|
|
|
+ for (Map<String, Object> originalRow : originalData) {
|
|
|
+ // 使用列名构建新映射
|
|
|
+ Map<String, Object> filteredRow = MapUtil.newHashMap();
|
|
|
+ for (String columnName : columnNames) {
|
|
|
+ Object value = originalRow.get(columnName.trim());
|
|
|
+ if (value != null) {
|
|
|
+ filteredRow.put(columnName.trim(), value);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ filteredData.add(filteredRow);
|
|
|
+ }
|
|
|
+ return filteredData;
|
|
|
+ }
|
|
|
+}
|
