|
|
|
|
@ -5,26 +5,35 @@ import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.controller.admin.datas.vo.DatasEnhance;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.controller.admin.datas.vo.DatasPageReqVO;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.controller.admin.datas.vo.DatasSaveReqVO;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.controller.admin.types.vo.TypesSaveReqVO;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.dal.dataobject.datas.DatasDO;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.dal.dataobject.dataset.DatasetDO;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.dal.dataobject.mark.MarkDO;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.dal.dataobject.mark.MarkInfoDO;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.dal.mysql.datas.DatasMapper;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.dal.mysql.mark.MarkInfoMapper;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.service.ImageService;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.service.dataset.DatasetService;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.service.mark.MarkService;
|
|
|
|
|
import cn.iocoder.yudao.module.annotation.service.types.TypesService;
|
|
|
|
|
import cn.iocoder.yudao.module.system.dal.dataobject.dict.DictDataDO;
|
|
|
|
|
import cn.iocoder.yudao.module.system.service.dict.DictDataService;
|
|
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
|
|
|
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
|
|
|
|
|
import jakarta.annotation.Resource;
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
|
import org.springframework.validation.annotation.Validated;
|
|
|
|
|
|
|
|
|
|
import java.io.File;
|
|
|
|
|
import java.io.IOException;
|
|
|
|
|
import java.nio.file.Files;
|
|
|
|
|
import java.nio.file.StandardCopyOption;
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.Collections;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
import java.util.concurrent.CompletableFuture;
|
|
|
|
|
import java.util.stream.Collectors;
|
|
|
|
|
//import static cn.iocoder.yudao.module.annotation.enums.ErrorCodeConstants.*;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
@ -32,6 +41,7 @@ import java.util.stream.Collectors;
|
|
|
|
|
*
|
|
|
|
|
* @author 管理员
|
|
|
|
|
*/
|
|
|
|
|
@Slf4j
|
|
|
|
|
@Service
|
|
|
|
|
@Validated
|
|
|
|
|
public class DatasServiceImpl extends ServiceImpl<DatasMapper, DatasDO> implements DatasService{
|
|
|
|
|
@ -44,89 +54,43 @@ public class DatasServiceImpl extends ServiceImpl<DatasMapper, DatasDO> impleme
|
|
|
|
|
|
|
|
|
|
@Resource
|
|
|
|
|
private DictDataService dictDataService;
|
|
|
|
|
/**
|
|
|
|
|
* 递归检查路径是否存在，并获取其中所有图片文件名称（包括子目录中的图片）
|
|
|
|
|
* @param path 要检查的路径字符串
|
|
|
|
|
* @return 图片文件名称列表
|
|
|
|
|
*/
|
|
|
|
|
public List<String> getImageNamesFromPath(String superiorPath) {
|
|
|
|
|
DictDataDO basePath = dictDataService.parseDictData("visual_annotation_conf","base_path");
|
|
|
|
|
String path = basePath.getValue() + superiorPath;
|
|
|
|
|
|
|
|
|
|
List<String> imageNames = new ArrayList<>();
|
|
|
|
|
@Resource
|
|
|
|
|
private DatasetService datasetService;
|
|
|
|
|
|
|
|
|
|
// 检查路径是否为空
|
|
|
|
|
if (path == null || path.isEmpty()) {
|
|
|
|
|
return imageNames;
|
|
|
|
|
}
|
|
|
|
|
@Resource
|
|
|
|
|
private MarkInfoMapper markInfoMapper;
|
|
|
|
|
|
|
|
|
|
File directory = new File(path);
|
|
|
|
|
File rootDirectory = directory; // 保存根目录引用
|
|
|
|
|
@Resource
|
|
|
|
|
private ImageService imageService;
|
|
|
|
|
|
|
|
|
|
// 检查路径是否存在且为目录
|
|
|
|
|
/**
|
|
|
|
|
* 递归获取指定目录下的所有图片文件路径
|
|
|
|
|
* @param directory 目录
|
|
|
|
|
* @return 图片文件路径列表
|
|
|
|
|
*/
|
|
|
|
|
private List<String> getAllImageFiles(File directory) {
|
|
|
|
|
List<String> imageFiles = new ArrayList<>();
|
|
|
|
|
if (!directory.exists() || !directory.isDirectory()) {
|
|
|
|
|
return imageNames;
|
|
|
|
|
return imageFiles;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 递归遍历目录，传递根目录用于计算相对路径
|
|
|
|
|
traverseDirectory(directory, rootDirectory, imageNames);
|
|
|
|
|
imageNames = imageNames.stream()
|
|
|
|
|
.map(v->{return superiorPath+"/"+v;})
|
|
|
|
|
.collect(Collectors.toList());
|
|
|
|
|
|
|
|
|
|
return imageNames;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 递归遍历目录查找图片文件
|
|
|
|
|
* @param directory 要遍历的目录
|
|
|
|
|
* @param rootDirectory 根目录
|
|
|
|
|
* @param imageNames 图片文件名称收集列表
|
|
|
|
|
*/
|
|
|
|
|
private void traverseDirectory(File directory, File rootDirectory, List<String> imageNames) {
|
|
|
|
|
File[] files = directory.listFiles();
|
|
|
|
|
if (files != null) {
|
|
|
|
|
for (File file : files) {
|
|
|
|
|
if (file.isDirectory()) {
|
|
|
|
|
// 递归处理子目录，保持根目录不变
|
|
|
|
|
traverseDirectory(file, rootDirectory, imageNames);
|
|
|
|
|
imageFiles.addAll(getAllImageFiles(file));
|
|
|
|
|
} else if (isImageFile(file)) {
|
|
|
|
|
// 添加图片文件（包含相对于根目录的路径）
|
|
|
|
|
String relativePath = getRelativePath(file, rootDirectory);
|
|
|
|
|
imageNames.add(relativePath);
|
|
|
|
|
imageFiles.add(file.getAbsolutePath());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 获取文件相对于根目录的路径
|
|
|
|
|
* @param file 文件
|
|
|
|
|
* @param rootDirectory 根目录
|
|
|
|
|
* @return 相对路径
|
|
|
|
|
*/
|
|
|
|
|
private String getRelativePath(File file, File rootDirectory) {
|
|
|
|
|
// 使用 Path API 来正确计算相对路径
|
|
|
|
|
try {
|
|
|
|
|
return rootDirectory.toPath().relativize(file.toPath()).toString();
|
|
|
|
|
} catch (IllegalArgumentException e) {
|
|
|
|
|
// 如果无法计算相对路径，则返回文件名
|
|
|
|
|
return file.getName();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public static void main(String[] args) {
|
|
|
|
|
DatasServiceImpl datasService = new DatasServiceImpl();
|
|
|
|
|
List<String> imageNames = datasService.getImageNamesFromPath("D:\\PycharmProjects\\yolo\\runs\\detect\\11");
|
|
|
|
|
System.out.println(imageNames);
|
|
|
|
|
return imageFiles;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 判断文件是否为图片文件
|
|
|
|
|
* @param file 要判断的文件
|
|
|
|
|
* @param file 文件
|
|
|
|
|
* @return 是否为图片文件
|
|
|
|
|
*/
|
|
|
|
|
private boolean isImageFile(File file) {
|
|
|
|
|
@ -142,22 +106,351 @@ public class DatasServiceImpl extends ServiceImpl<DatasMapper, DatasDO> impleme
|
|
|
|
|
fileName.endsWith(".bmp") ||
|
|
|
|
|
fileName.endsWith(".webp");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 复制文件
|
|
|
|
|
* @param source 源文件
|
|
|
|
|
* @param dest 目标文件
|
|
|
|
|
*/
|
|
|
|
|
private void copyFile(File source, File dest) throws IOException {
|
|
|
|
|
Files.copy(source.toPath(), dest.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 处理数据集，根据任务类型生成相应的数据集结构
|
|
|
|
|
* @param datasDO 数据对象
|
|
|
|
|
* @return 生成的图片信息列表，每个Map包含：0=url路径, 1=文件路径, 2=标注路径（分类没有）
|
|
|
|
|
*/
|
|
|
|
|
private List<Map<Integer, String>> processDatasets(DatasDO datasDO) {
|
|
|
|
|
List<Map<Integer, String>> imagePaths = new ArrayList<>();
|
|
|
|
|
|
|
|
|
|
if (datasDO.getDatasets() == null || datasDO.getDatasets().isEmpty()) {
|
|
|
|
|
return imagePaths;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 获取字典配置
|
|
|
|
|
Map<String, DictDataDO> baseConf = dictDataService.getDictDataList("base_conf");
|
|
|
|
|
DictDataDO basePathDO = baseConf.get("base_path");
|
|
|
|
|
DictDataDO dataApiPathDO = baseConf.get("data_api_path");
|
|
|
|
|
|
|
|
|
|
if (basePathDO == null) {
|
|
|
|
|
log.error("base_conf.base_path 配置不存在");
|
|
|
|
|
return imagePaths;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
String basePath = basePathDO.getValue();
|
|
|
|
|
String dataApiPath = dataApiPathDO != null ? dataApiPathDO.getValue() : "";
|
|
|
|
|
|
|
|
|
|
// 创建YOLO训练目录
|
|
|
|
|
String yoloProjectDir = basePath + File.separator + "yolo" + File.separator + datasDO.getId();
|
|
|
|
|
File yoloDir = new File(yoloProjectDir);
|
|
|
|
|
if (!yoloDir.exists()) {
|
|
|
|
|
yoloDir.mkdirs();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 根据任务类型创建目录结构
|
|
|
|
|
String projectType = datasDO.getType();
|
|
|
|
|
// 在字典表的visual_type里面
|
|
|
|
|
if ("2".equals(projectType)) {
|
|
|
|
|
// 分类任务：data/train/class1/, data/val/class1/
|
|
|
|
|
processClassificationDatasets(datasDO, yoloDir, basePath, dataApiPath, imagePaths);
|
|
|
|
|
} else if ("1".equals(projectType)) {
|
|
|
|
|
// 检测任务：data/images/train/, data/labels/train/
|
|
|
|
|
processDetectionDatasets(datasDO, yoloDir, basePath, dataApiPath, imagePaths);
|
|
|
|
|
} else {
|
|
|
|
|
log.error("未知的任务类型: {}", projectType);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return imagePaths;
|
|
|
|
|
}
|
|
|
|
|
@Resource
|
|
|
|
|
TypesService typesService;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 处理分类任务数据集
|
|
|
|
|
* 目录结构：
|
|
|
|
|
* data/
|
|
|
|
|
* ├── train/
|
|
|
|
|
* │ ├── class1/
|
|
|
|
|
* │ └── class2/
|
|
|
|
|
* └── val/
|
|
|
|
|
* ├── class1/
|
|
|
|
|
* └── class2/
|
|
|
|
|
*/
|
|
|
|
|
private void processClassificationDatasets(DatasDO datasDO, File yoloDir, String basePath,
|
|
|
|
|
String dataApiPath, List<Map<Integer, String>> imagePaths) {
|
|
|
|
|
// 将所有图片保存到默认路径下，并且创建默认分类
|
|
|
|
|
TypesSaveReqVO typesSaveReqVO = new TypesSaveReqVO();
|
|
|
|
|
typesSaveReqVO.setDataId(datasDO.getId());
|
|
|
|
|
typesSaveReqVO.setName("default");
|
|
|
|
|
typesSaveReqVO.setColor("#FF6B6B");
|
|
|
|
|
typesService.createTypes(typesSaveReqVO);
|
|
|
|
|
// 解析数据集ID
|
|
|
|
|
String[] datasetIds = datasDO.getDatasets().split(",");
|
|
|
|
|
List<DatasetDO> datasetList = new ArrayList<>();
|
|
|
|
|
for (String datasetIdStr : datasetIds) {
|
|
|
|
|
try {
|
|
|
|
|
Integer datasetId = Integer.parseInt(datasetIdStr.trim());
|
|
|
|
|
DatasetDO dataset = datasetService.getDataset(datasetId);
|
|
|
|
|
if (dataset != null) {
|
|
|
|
|
datasetList.add(dataset);
|
|
|
|
|
}
|
|
|
|
|
} catch (NumberFormatException e) {
|
|
|
|
|
log.warn("无效的数据集ID: {}", datasetIdStr);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 收集所有图片
|
|
|
|
|
List<String> allImages = new ArrayList<>();
|
|
|
|
|
for (DatasetDO dataset : datasetList) {
|
|
|
|
|
String datasetPath = dataset.getPath();
|
|
|
|
|
if (datasetPath != null && !datasetPath.isEmpty()) {
|
|
|
|
|
File datasetDir = new File(datasetPath);
|
|
|
|
|
if (datasetDir.exists()) {
|
|
|
|
|
allImages.addAll(getAllImageFiles(datasetDir));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 根据比例分配图片
|
|
|
|
|
int totalImages = allImages.size();
|
|
|
|
|
int trainCount = (int) (totalImages * datasDO.getTrainRatio() / 100.0);
|
|
|
|
|
int valCount = (int) (totalImages * datasDO.getValRatio() / 100.0);
|
|
|
|
|
int testCount = totalImages - trainCount - valCount;
|
|
|
|
|
|
|
|
|
|
Collections.shuffle(allImages);
|
|
|
|
|
|
|
|
|
|
// 创建目录结构
|
|
|
|
|
File dataDir = yoloDir;
|
|
|
|
|
File trainDir = new File(dataDir, "train");
|
|
|
|
|
File valDir = new File(dataDir, "val");
|
|
|
|
|
|
|
|
|
|
if (!trainDir.exists()) trainDir.mkdirs();
|
|
|
|
|
if (!valDir.exists()) valDir.mkdirs();
|
|
|
|
|
|
|
|
|
|
// 创建默认类别目录（由于分类任务需要类别，这里先创建一个默认类别）
|
|
|
|
|
File defaultClassDir = new File(trainDir, "default");
|
|
|
|
|
if (!defaultClassDir.exists()) defaultClassDir.mkdirs();
|
|
|
|
|
|
|
|
|
|
File defaultClassValDir = new File(valDir, "default");
|
|
|
|
|
if (!defaultClassValDir.exists()) defaultClassValDir.mkdirs();
|
|
|
|
|
|
|
|
|
|
// 处理训练集
|
|
|
|
|
for (int i = 0; i < trainCount; i++) {
|
|
|
|
|
String imagePath = allImages.get(i);
|
|
|
|
|
processClassificationImage(imagePath, defaultClassDir, basePath, dataApiPath,
|
|
|
|
|
datasDO.getId(), "train", "default", imagePaths);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 处理验证集
|
|
|
|
|
for (int i = trainCount; i < trainCount + valCount; i++) {
|
|
|
|
|
String imagePath = allImages.get(i);
|
|
|
|
|
processClassificationImage(imagePath, defaultClassValDir, basePath, dataApiPath,
|
|
|
|
|
datasDO.getId(), "val", "default", imagePaths);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 测试集（合并到验证集）
|
|
|
|
|
for (int i = trainCount + valCount; i < totalImages; i++) {
|
|
|
|
|
String imagePath = allImages.get(i);
|
|
|
|
|
processClassificationImage(imagePath, defaultClassValDir, basePath, dataApiPath,
|
|
|
|
|
datasDO.getId(), "val", "default", imagePaths);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 处理单张分类图片
|
|
|
|
|
*/
|
|
|
|
|
private void processClassificationImage(String imagePath, File destDir, String basePath,
|
|
|
|
|
String dataApiPath, Integer projectId, String split,
|
|
|
|
|
String className, List<Map<Integer, String>> imagePaths) {
|
|
|
|
|
File sourceFile = new File(imagePath);
|
|
|
|
|
File destFile = new File(destDir, sourceFile.getName());
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
copyFile(sourceFile, destFile);
|
|
|
|
|
|
|
|
|
|
// 生成相对路径
|
|
|
|
|
String relativePath = split + "/" + className + "/" + destFile.getName();
|
|
|
|
|
|
|
|
|
|
// 生成HTTP路径
|
|
|
|
|
String httpPath = !dataApiPath.isEmpty()
|
|
|
|
|
? dataApiPath + "/yolo/" + projectId + "/" + relativePath
|
|
|
|
|
: "/yolo/" + projectId + "/" + relativePath;
|
|
|
|
|
|
|
|
|
|
// 生成绝对路径
|
|
|
|
|
String absolutePath = destFile.getAbsolutePath();
|
|
|
|
|
|
|
|
|
|
// 构建返回Map：0=url路径, 1=文件路径, 2=标注路径（分类没有标注）
|
|
|
|
|
Map<Integer, String> result = new java.util.HashMap<>();
|
|
|
|
|
result.put(0, httpPath);
|
|
|
|
|
result.put(1, absolutePath);
|
|
|
|
|
result.put(2, ""); // 分类任务没有标注路径
|
|
|
|
|
|
|
|
|
|
imagePaths.add(result);
|
|
|
|
|
} catch (IOException e) {
|
|
|
|
|
log.error("复制文件失败: {}", imagePath, e);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 处理检测任务数据集
|
|
|
|
|
* 目录结构：
|
|
|
|
|
* data/
|
|
|
|
|
* ├── images/
|
|
|
|
|
* │ ├── train/
|
|
|
|
|
* │ └── val/
|
|
|
|
|
* └── labels/
|
|
|
|
|
* ├── train/
|
|
|
|
|
* └── val/
|
|
|
|
|
*/
|
|
|
|
|
private void processDetectionDatasets(DatasDO datasDO, File yoloDir, String basePath,
|
|
|
|
|
String dataApiPath, List<Map<Integer, String>> imagePaths) {
|
|
|
|
|
// 解析数据集ID
|
|
|
|
|
String[] datasetIds = datasDO.getDatasets().split(",");
|
|
|
|
|
List<DatasetDO> datasetList = new ArrayList<>();
|
|
|
|
|
for (String datasetIdStr : datasetIds) {
|
|
|
|
|
try {
|
|
|
|
|
Integer datasetId = Integer.parseInt(datasetIdStr.trim());
|
|
|
|
|
DatasetDO dataset = datasetService.getDataset(datasetId);
|
|
|
|
|
if (dataset != null) {
|
|
|
|
|
datasetList.add(dataset);
|
|
|
|
|
}
|
|
|
|
|
} catch (NumberFormatException e) {
|
|
|
|
|
log.warn("无效的数据集ID: {}", datasetIdStr);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 收集所有图片
|
|
|
|
|
List<String> allImages = new ArrayList<>();
|
|
|
|
|
for (DatasetDO dataset : datasetList) {
|
|
|
|
|
String datasetPath = dataset.getPath();
|
|
|
|
|
if (datasetPath != null && !datasetPath.isEmpty()) {
|
|
|
|
|
File datasetDir = new File(datasetPath);
|
|
|
|
|
if (datasetDir.exists()) {
|
|
|
|
|
allImages.addAll(getAllImageFiles(datasetDir));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 根据比例分配图片
|
|
|
|
|
int totalImages = allImages.size();
|
|
|
|
|
int trainCount = (int) (totalImages * datasDO.getTrainRatio() / 100.0);
|
|
|
|
|
int valCount = (int) (totalImages * datasDO.getValRatio() / 100.0);
|
|
|
|
|
int testCount = totalImages - trainCount - valCount;
|
|
|
|
|
|
|
|
|
|
Collections.shuffle(allImages);
|
|
|
|
|
|
|
|
|
|
// 创建目录结构
|
|
|
|
|
File imagesTrainDir = new File(yoloDir, "images/train");
|
|
|
|
|
File imagesValDir = new File(yoloDir, "images/val");
|
|
|
|
|
File labelsTrainDir = new File(yoloDir, "labels/train");
|
|
|
|
|
File labelsValDir = new File(yoloDir, "labels/val");
|
|
|
|
|
|
|
|
|
|
if (!imagesTrainDir.exists()) imagesTrainDir.mkdirs();
|
|
|
|
|
if (!imagesValDir.exists()) imagesValDir.mkdirs();
|
|
|
|
|
if (!labelsTrainDir.exists()) labelsTrainDir.mkdirs();
|
|
|
|
|
if (!labelsValDir.exists()) labelsValDir.mkdirs();
|
|
|
|
|
|
|
|
|
|
// 处理训练集
|
|
|
|
|
for (int i = 0; i < trainCount; i++) {
|
|
|
|
|
String imagePath = allImages.get(i);
|
|
|
|
|
processDetectionImage(imagePath, imagesTrainDir, labelsTrainDir, basePath, dataApiPath,
|
|
|
|
|
datasDO.getId(), "train", imagePaths);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 处理验证集
|
|
|
|
|
for (int i = trainCount; i < trainCount + valCount; i++) {
|
|
|
|
|
String imagePath = allImages.get(i);
|
|
|
|
|
processDetectionImage(imagePath, imagesValDir, labelsValDir, basePath, dataApiPath,
|
|
|
|
|
datasDO.getId(), "val", imagePaths);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 测试集（合并到验证集）
|
|
|
|
|
for (int i = trainCount + valCount; i < totalImages; i++) {
|
|
|
|
|
String imagePath = allImages.get(i);
|
|
|
|
|
processDetectionImage(imagePath, imagesValDir, labelsValDir, basePath, dataApiPath,
|
|
|
|
|
datasDO.getId(), "val", imagePaths);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 处理单张检测图片
|
|
|
|
|
*/
|
|
|
|
|
private void processDetectionImage(String imagePath, File imagesDir, File labelsDir, String basePath,
|
|
|
|
|
String dataApiPath, Integer projectId, String split,
|
|
|
|
|
List<Map<Integer, String>> imagePaths) {
|
|
|
|
|
File sourceFile = new File(imagePath);
|
|
|
|
|
File destImageFile = new File(imagesDir, sourceFile.getName());
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
copyFile(sourceFile, destImageFile);
|
|
|
|
|
|
|
|
|
|
// 生成标注文件路径
|
|
|
|
|
String labelFileName = sourceFile.getName().replaceAll("\\.[^.]+$", ".txt");
|
|
|
|
|
File labelFile = new File(labelsDir, labelFileName);
|
|
|
|
|
|
|
|
|
|
// 创建空的标注文件
|
|
|
|
|
if (!labelFile.exists()) {
|
|
|
|
|
labelFile.createNewFile();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 生成图片相对路径
|
|
|
|
|
String imageRelativePath = "images/" + split + "/" + destImageFile.getName();
|
|
|
|
|
|
|
|
|
|
// 生成标注文件相对路径
|
|
|
|
|
String labelRelativePath = "labels/" + split + "/" + labelFile.getName();
|
|
|
|
|
|
|
|
|
|
// 生成HTTP路径
|
|
|
|
|
String httpPath = !dataApiPath.isEmpty()
|
|
|
|
|
? dataApiPath + "/yolo/" + projectId + "/" + imageRelativePath
|
|
|
|
|
: "/yolo/" + projectId + "/" + imageRelativePath;
|
|
|
|
|
|
|
|
|
|
// 生成绝对路径
|
|
|
|
|
String absolutePath = destImageFile.getAbsolutePath();
|
|
|
|
|
|
|
|
|
|
// 生成标注文件绝对路径
|
|
|
|
|
String labelAbsolutePath = labelFile.getAbsolutePath();
|
|
|
|
|
|
|
|
|
|
// 构建返回Map：0=url路径, 1=文件路径, 2=标注路径
|
|
|
|
|
Map<Integer, String> result = new java.util.HashMap<>();
|
|
|
|
|
result.put(0, httpPath);
|
|
|
|
|
result.put(1, absolutePath);
|
|
|
|
|
result.put(2, labelAbsolutePath);
|
|
|
|
|
|
|
|
|
|
imagePaths.add(result);
|
|
|
|
|
} catch (IOException e) {
|
|
|
|
|
log.error("复制文件失败: {}", imagePath, e);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public Integer createDatas(DatasSaveReqVO createReqVO) {
|
|
|
|
|
// 插入
|
|
|
|
|
DatasDO datas = BeanUtils.toBean(createReqVO, DatasDO.class);
|
|
|
|
|
// 查看数据集是否存在判断路径是否存在
|
|
|
|
|
List<String> imageNames = getImageNamesFromPath(createReqVO.getPath());
|
|
|
|
|
//默认类型为正在标注
|
|
|
|
|
datas.setStatus("1");
|
|
|
|
|
datas.setCount((long) imageNames.size());
|
|
|
|
|
|
|
|
|
|
// 默认状态为还未标注
|
|
|
|
|
datas.setStatus("0");
|
|
|
|
|
datas.setProgress(0.0);
|
|
|
|
|
for (String imageName : imageNames){
|
|
|
|
|
|
|
|
|
|
// 先插入数据
|
|
|
|
|
datasMapper.insert(datas);
|
|
|
|
|
|
|
|
|
|
// 处理数据集，复制图片并生成YOLO结构
|
|
|
|
|
List<Map<Integer, String>> imagePaths = processDatasets(datas);
|
|
|
|
|
|
|
|
|
|
// 保存图片路径到mark表
|
|
|
|
|
for (Map<Integer, String> imageInfo : imagePaths) {
|
|
|
|
|
markService.save(new MarkDO()
|
|
|
|
|
.setDataId(datas.getId())
|
|
|
|
|
.setPath(imageName));
|
|
|
|
|
.setPath(imageInfo.get(0)) // 保存URL路径
|
|
|
|
|
.setImagePath(imageInfo.get(1)) // 保存图片绝对路径
|
|
|
|
|
.setLabelPath(imageInfo.get(2)) // 保存标注文件路径
|
|
|
|
|
.setStatus(0)); // 默认未标注
|
|
|
|
|
}
|
|
|
|
|
datasMapper.insert(datas);
|
|
|
|
|
|
|
|
|
|
// 更新图片总数
|
|
|
|
|
datas.setCount((long) imagePaths.size());
|
|
|
|
|
datasMapper.updateById(datas);
|
|
|
|
|
|
|
|
|
|
// 返回
|
|
|
|
|
return datas.getId();
|
|
|
|
|
@ -167,8 +460,28 @@ public class DatasServiceImpl extends ServiceImpl<DatasMapper, DatasDO> impleme
|
|
|
|
|
public void updateDatas(DatasSaveReqVO updateReqVO) {
|
|
|
|
|
// 校验存在
|
|
|
|
|
validateDatasExists(updateReqVO.getId());
|
|
|
|
|
// 更新
|
|
|
|
|
|
|
|
|
|
// 删除旧的标记记录
|
|
|
|
|
markService.remove(new QueryWrapper<MarkDO>().eq("data_id", updateReqVO.getId()));
|
|
|
|
|
|
|
|
|
|
// 更新数据
|
|
|
|
|
DatasDO updateObj = BeanUtils.toBean(updateReqVO, DatasDO.class);
|
|
|
|
|
|
|
|
|
|
// 处理数据集，复制图片并生成YOLO结构
|
|
|
|
|
List<Map<Integer, String>> imagePaths = processDatasets(updateObj);
|
|
|
|
|
|
|
|
|
|
// 保存图片路径到mark表
|
|
|
|
|
for (Map<Integer, String> imageInfo : imagePaths) {
|
|
|
|
|
markService.save(new MarkDO()
|
|
|
|
|
.setDataId(updateObj.getId())
|
|
|
|
|
.setPath(imageInfo.get(0)) // 保存URL路径
|
|
|
|
|
.setImagePath(imageInfo.get(1)) // 保存图片绝对路径
|
|
|
|
|
.setLabelPath(imageInfo.get(2)) // 保存标注文件路径
|
|
|
|
|
.setStatus(0));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 更新图片总数
|
|
|
|
|
updateObj.setCount((long) imagePaths.size());
|
|
|
|
|
datasMapper.updateById(updateObj);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -176,10 +489,47 @@ public class DatasServiceImpl extends ServiceImpl<DatasMapper, DatasDO> impleme
|
|
|
|
|
public void deleteDatas(Integer id) {
|
|
|
|
|
// 校验存在
|
|
|
|
|
validateDatasExists(id);
|
|
|
|
|
// 删除
|
|
|
|
|
|
|
|
|
|
// 获取字典配置
|
|
|
|
|
Map<String, DictDataDO> baseConf = dictDataService.getDictDataList("base_conf");
|
|
|
|
|
DictDataDO basePathDO = baseConf.get("base_path");
|
|
|
|
|
|
|
|
|
|
if (basePathDO != null) {
|
|
|
|
|
String basePath = basePathDO.getValue();
|
|
|
|
|
String yoloProjectDir = basePath + File.separator + "yolo" + File.separator + id;
|
|
|
|
|
File yoloDir = new File(yoloProjectDir);
|
|
|
|
|
|
|
|
|
|
// 删除YOLO训练目录
|
|
|
|
|
if (yoloDir.exists()) {
|
|
|
|
|
deleteDirectory(yoloDir);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 删除标记记录
|
|
|
|
|
markService.remove(new QueryWrapper<MarkDO>().eq("data_id", id));
|
|
|
|
|
|
|
|
|
|
// 删除数据
|
|
|
|
|
datasMapper.deleteById(id);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* 递归删除目录
|
|
|
|
|
* @param directory 要删除的目录
|
|
|
|
|
*/
|
|
|
|
|
private void deleteDirectory(File directory) {
|
|
|
|
|
File[] files = directory.listFiles();
|
|
|
|
|
if (files != null) {
|
|
|
|
|
for (File file : files) {
|
|
|
|
|
if (file.isDirectory()) {
|
|
|
|
|
deleteDirectory(file);
|
|
|
|
|
} else {
|
|
|
|
|
file.delete();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
directory.delete();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void validateDatasExists(Integer id) {
|
|
|
|
|
if (datasMapper.selectById(id) == null) {
|
|
|
|
|
// throw exception();
|
|
|
|
|
@ -195,75 +545,25 @@ public class DatasServiceImpl extends ServiceImpl<DatasMapper, DatasDO> impleme
|
|
|
|
|
public PageResult<DatasDO> getDatasPage(DatasPageReqVO pageReqVO) {
|
|
|
|
|
return datasMapper.selectPage(pageReqVO);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void refreshDatas(DatasDO datasDO) {
|
|
|
|
|
// 1. 获取路径中的所有图片文件
|
|
|
|
|
List<String> currentImageNames = getImageNamesFromPath(datasDO.getPath());
|
|
|
|
|
|
|
|
|
|
// 2. 获取数据库中已有的标记记录
|
|
|
|
|
List<MarkDO> existingMarks = markService.list(new QueryWrapper<MarkDO>().eq("data_id", datasDO.getId()));
|
|
|
|
|
|
|
|
|
|
// 3. 找出需要删除的图片（数据库中有但文件系统中没有）
|
|
|
|
|
List<String> existingImageNames = existingMarks.stream()
|
|
|
|
|
.map(MarkDO::getPath)
|
|
|
|
|
.collect(Collectors.toList());
|
|
|
|
|
|
|
|
|
|
List<String> imagesToDelete = existingImageNames.stream()
|
|
|
|
|
.filter(imageName -> !currentImageNames.contains(imageName))
|
|
|
|
|
.collect(Collectors.toList());
|
|
|
|
|
|
|
|
|
|
// 删除不存在的标记记录
|
|
|
|
|
if (!imagesToDelete.isEmpty()) {
|
|
|
|
|
markService.remove(new QueryWrapper<MarkDO>()
|
|
|
|
|
.eq("data_id", datasDO.getId())
|
|
|
|
|
.in("path", imagesToDelete));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 4. 找出需要新增的图片（文件系统中有但数据库中没有）
|
|
|
|
|
List<String> imagesToAdd = currentImageNames.stream()
|
|
|
|
|
.filter(imageName -> !existingImageNames.contains(imageName))
|
|
|
|
|
.collect(Collectors.toList());
|
|
|
|
|
|
|
|
|
|
// 新增图片标记记录
|
|
|
|
|
for (String imageName : imagesToAdd) {
|
|
|
|
|
markService.save(new MarkDO()
|
|
|
|
|
.setDataId(datasDO.getId())
|
|
|
|
|
.setPath(imageName)
|
|
|
|
|
.setStatus(0)); // 默认未标注状态
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 5. 更新数据集统计信息
|
|
|
|
|
List<MarkDO> updatedMarks = markService.list(new QueryWrapper<MarkDO>().eq("data_id", datasDO.getId()));
|
|
|
|
|
|
|
|
|
|
// 计算进度
|
|
|
|
|
long totalImages = updatedMarks.size();
|
|
|
|
|
long completedImages = updatedMarks.stream()
|
|
|
|
|
.filter(mark -> mark.getStatus() != null && mark.getStatus() == 1)
|
|
|
|
|
.count();
|
|
|
|
|
|
|
|
|
|
double progress = totalImages > 0 ? (completedImages * 100.0 / totalImages) : 0.0;
|
|
|
|
|
|
|
|
|
|
// 判断是否全部完成
|
|
|
|
|
String status = (completedImages == totalImages && totalImages > 0) ? "2" : "1"; // 2表示完成，1表示进行中
|
|
|
|
|
|
|
|
|
|
// 更新数据集
|
|
|
|
|
datasDO.setCount(totalImages);
|
|
|
|
|
datasDO.setProgress(progress);
|
|
|
|
|
datasDO.setStatus(status);
|
|
|
|
|
datasDO.setCreator( null);
|
|
|
|
|
datasDO.setUpdater(null);
|
|
|
|
|
datasMapper.updateById(datasDO);
|
|
|
|
|
// 不再使用path字段，此方法需要重新实现
|
|
|
|
|
log.warn("refreshDatas方法已废弃，请使用updateDatas方法更新数据集");
|
|
|
|
|
}
|
|
|
|
|
@Resource
|
|
|
|
|
ImageService imageService;
|
|
|
|
|
@Resource
|
|
|
|
|
MarkInfoMapper markInfoMapper;
|
|
|
|
|
|
|
|
|
|
@Override
|
|
|
|
|
public void enhance(DatasEnhance datasEnhance) {
|
|
|
|
|
DictDataDO basePathDO = dictDataService.parseDictData("visual_annotation_conf", "base_path");
|
|
|
|
|
|
|
|
|
|
if (basePathDO == null) {
|
|
|
|
|
log.error("visual_annotation_conf.base_path 配置不存在");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
String basePath = basePathDO.getValue();
|
|
|
|
|
|
|
|
|
|
DictDataDO basePath = dictDataService.parseDictData("visual_annotation_conf","base_path");
|
|
|
|
|
// 遍历图片，根据选择的类型，新增图片，并且新增他自身的标记记录
|
|
|
|
|
// 遍历图片，根据选择的类型，新增图片，并且新增他自身的标记记录
|
|
|
|
|
List<MarkDO> markDOList = markService.list(new QueryWrapper<MarkDO>()
|
|
|
|
|
.eq("data_id", datasEnhance.getId()));
|
|
|
|
|
|
|
|
|
|
@ -277,7 +577,7 @@ public class DatasServiceImpl extends ServiceImpl<DatasMapper, DatasDO> impleme
|
|
|
|
|
try {
|
|
|
|
|
String path = imageService.enhanceImage(
|
|
|
|
|
markDO.getPath(),
|
|
|
|
|
basePath.getValue(),
|
|
|
|
|
basePath,
|
|
|
|
|
datasEnhance.getEnhancements()[index % datasEnhance.getEnhancements().length]
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
@ -285,20 +585,19 @@ public class DatasServiceImpl extends ServiceImpl<DatasMapper, DatasDO> impleme
|
|
|
|
|
new QueryWrapper<MarkInfoDO>().eq("mark_id", markDO.getId())
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
MarkDO newMarkDO = BeanUtils.toBean(markDO, MarkDO.class);
|
|
|
|
|
|
|
|
|
|
MarkDO newMarkDO = BeanUtils.toBean(markDO, MarkDO.class);
|
|
|
|
|
newMarkDO.setId(null);
|
|
|
|
|
newMarkDO.setPath(path);
|
|
|
|
|
markService.save(newMarkDO);
|
|
|
|
|
|
|
|
|
|
for (MarkInfoDO markInfoDO : markInfoDOList) {
|
|
|
|
|
MarkInfoDO newMarkInfoDO = BeanUtils.toBean(markInfoDO, MarkInfoDO.class);
|
|
|
|
|
MarkInfoDO newMarkInfoDO = BeanUtils.toBean(markInfoDO, MarkInfoDO.class);
|
|
|
|
|
newMarkInfoDO.setId(null);
|
|
|
|
|
newMarkInfoDO.setMarkId(newMarkDO.getId());
|
|
|
|
|
markInfoMapper.insert(newMarkInfoDO);
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
log.error("图像增强处理失败，图片路径: "+markDO.getPath(),e);
|
|
|
|
|
log.error("图像增强处理失败，图片路径: " + markDO.getPath(), e);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
@ -307,8 +606,6 @@ public class DatasServiceImpl extends ServiceImpl<DatasMapper, DatasDO> impleme
|
|
|
|
|
|
|
|
|
|
// 等待所有异步任务完成
|
|
|
|
|
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
LAPTOP-S9HJSOEB\昊天
2 weeks ago