From 3d69f32fae8a5b2d581593a85fccde1e4ccd26e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?LAPTOP-S9HJSOEB=5C=E6=98=8A=E5=A4=A9?= Date: Mon, 20 Apr 2026 11:58:48 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=8C=E7=BB=B4=E7=A0=81?= =?UTF-8?q?=E8=AF=86=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 7 + pom.xml | 40 +- .../controller/CategoryController.java | 2 +- .../controller/QtCodeController.java | 341 ++++++++++++++ .../IndustrialCamera/QrCode/WeChatDeCode.java | 48 +- .../algorithm/ONNXServiceNew.java | 433 +++++++++++++++++- .../camera/lx/config/BoxCountResponse.java | 6 + src/main/resources/application.yml | 11 +- 8 files changed, 875 insertions(+), 13 deletions(-) create mode 100644 src/main/java/com/example/lxcameraapi/controller/QtCodeController.java diff --git a/.gitignore b/.gitignore index 549e00a..485b4ac 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,10 @@ build/ ### VS Code ### .vscode/ + +### Logs ### +logs/ +log/ + +### ONNX models ### +*.onnx diff --git a/pom.xml b/pom.xml index d39b3a6..19dd8f6 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,31 @@ org.springframework.boot spring-boot-starter-web-services - + + org.bytedeco + javacpp + 1.5.7 + windows-x86_64 + + + org.bytedeco + openblas + 0.3.19-1.5.7 + windows-x86_64 + + + org.bytedeco + opencv + 4.5.5-1.5.7 + + + org.bytedeco + opencv + 4.5.5-1.5.7 + windows-x86_64 + + + com.microsoft.onnxruntime onnxruntime 1.16.0 @@ -127,13 +151,13 @@ javase 3.5.0 - - org - opencv - system - 1.0 - ${project.basedir}/src/main/resources/libs/opencv-480.jar - + + + + + + + diff --git a/src/main/java/com/example/lxcameraapi/controller/CategoryController.java b/src/main/java/com/example/lxcameraapi/controller/CategoryController.java index 397de50..77a3d12 100644 --- a/src/main/java/com/example/lxcameraapi/controller/CategoryController.java +++ b/src/main/java/com/example/lxcameraapi/controller/CategoryController.java @@ -57,7 +57,7 @@ public class CategoryController { } boolean success = imageCaptureService.captureImage(ip, path); - List detectResults = onnxServiceNew.detect(path, null); + List detectResults = onnxServiceNew.detect26(path, null); log.info("检测结果: {}", detectResults); boxCountResponse.setImagePath(url); diff --git a/src/main/java/com/example/lxcameraapi/controller/QtCodeController.java b/src/main/java/com/example/lxcameraapi/controller/QtCodeController.java new file mode 100644 index 0000000..ac18873 --- /dev/null +++ b/src/main/java/com/example/lxcameraapi/controller/QtCodeController.java @@ -0,0 +1,341 @@ +package com.example.lxcameraapi.controller; + +import ai.onnxruntime.OrtException; +import com.example.lxcameraapi.conf.AppConfig; +import com.example.lxcameraapi.service.IndustrialCamera.QrCode.WeChatDeCode; +import com.example.lxcameraapi.service.IndustrialCamera.algorithm.ONNXServiceNew; +import com.example.lxcameraapi.service.IndustrialCamera.camera.hik.ImageCaptureService; +import com.example.lxcameraapi.service.IndustrialCamera.camera.lx.config.BoxCountRequest; +import com.example.lxcameraapi.service.IndustrialCamera.camera.lx.config.BoxCountResponse; +import com.example.lxcameraapi.service.IndustrialCamera.opencv.OpencvService; +import com.example.lxcameraapi.service.IndustrialCamera.yolo.BoundingBox; +import com.google.zxing.*; +import com.google.zxing.client.j2se.BufferedImageLuminanceSource; +import com.google.zxing.common.HybridBinarizer; +import com.google.zxing.qrcode.QRCodeReader; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.bytedeco.opencv.opencv_core.Mat; +import org.bytedeco.opencv.opencv_core.StringVector; +import org.bytedeco.opencv.opencv_wechat_qrcode.WeChatQRCode; +import org.opencv.imgcodecs.Imgcodecs; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.*; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +@RestController +@RequestMapping("/api/qrcode") +public class QtCodeController { + @Resource + ONNXServiceNew onnxServiceNew; + + @Resource + HikCaptureController hikCaptureController; + + + @Resource + private ImageCaptureService imageCaptureService; + + @Resource + private AppConfig appConfig; + @Autowired + private OpencvService opencvService; + + /** + * 品规识别 + */ + @PostMapping("/single") + public BoxCountResponse captureSingle(@RequestBody BoxCountRequest request) { + + + BoxCountResponse boxCountResponse = new BoxCountResponse(); + + // 获取当前日期格式为 yyyy-MM-dd + LocalDateTime currentDate = LocalDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + String formattedDate = currentDate.format(formatter); + + String path = appConfig.getPicPath()+ formattedDate + "/" + request.getCameraId() + "/" + System.currentTimeMillis() + ".png"; + String url = appConfig.getPicUrl() + formattedDate + "/" + request.getCameraId() + "/" + System.currentTimeMillis() + ".png"; + try { + + log.info("开始拍照，IP: {}, 路径: {}", request.getCameraId(), path); + String ip = null; + for (AppConfig.Camera camera : appConfig.getHikCamera()) { + if (camera.getId().equals(request.getCameraId())) + ip = camera.getIp(); + } + boolean success = imageCaptureService.captureImage(ip, path); + + List detectResults = onnxServiceNew.detect26(path, "qrCode"); + opencvService.drawBoundingBoxesOnImage(detectResults, path,path+".jpg"); + log.info("检测结果: {}", detectResults); + + // 读取原始图片 + BufferedImage originalImage = ImageIO.read(new File(path)); + if (originalImage == null) { + log.error("无法读取图片: {}", path); + boxCountResponse.setResult("读取图片失败"); + return boxCountResponse; + } + + // 存储解码结果 + List qrCodeResults = new ArrayList<>(); + + for (BoundingBox box : detectResults){ + try { + // 根据 BoundingBox 裁剪图片 + BufferedImage croppedImage = cropBoundingBox(originalImage, box,30); + ImageIO.write(croppedImage, "jpg", new File(path+"_"+box.getIndex()+".jpg")); + + // 使用 ZXing 读取二维码 + String qrCodeContent = WeChatDeCode.deCode(croppedImage); + qrCodeResults.add(qrCodeContent); + if (qrCodeContent != null) { + + log.info("解码成功: {}, 置信度: {}", qrCodeContent, box.getConfidence()); + } else { + log.warn("解码失败，box: {}", box); + } + } catch (Exception e) { + log.error("解码二维码异常，box: {}", box, e); + } + } + + boxCountResponse.setQrCodeResults(qrCodeResults); + + } catch (OrtException e) { + log.error("检测异常", e); + return boxCountResponse; + } catch (Exception e) { + log.error("处理异常", e); + } + if (boxCountResponse.getQrCodeResults().size() > 0){ + + boxCountResponse.setResult(boxCountResponse.getQrCodeResults().get(0).substring(0,8)); + boxCountResponse.setImagePath(url+".jpg"); + }else { + boxCountResponse.setResult("Unknown"); + } + return boxCountResponse; + } + + + /** + * 品规识别 + */ + @PostMapping("/w") + public BoxCountResponse captureSingleWhite(@RequestBody BoxCountRequest request) { + + for (int i = 0; i < 100; i++){ + + + BoxCountResponse boxCountResponse = new BoxCountResponse(); + try { + + // 获取当前日期格式为 yyyy-MM-dd + LocalDateTime currentDate = LocalDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + String formattedDate = currentDate.format(formatter); + + String path = appConfig.getPicPath()+ formattedDate + "/" + request.getCameraId() + "/" + System.currentTimeMillis() + ".png"; + String url = appConfig.getPicUrl() + formattedDate + "/" + request.getCameraId() + "/" + System.currentTimeMillis() + ".png"; + log.info("开始拍照，IP: {}, 路径: {}", request.getCameraId(), path); + String ip = null; + for (AppConfig.Camera camera : appConfig.getHikCamera()) { + if (camera.getId().equals(request.getCameraId())) + ip = camera.getIp(); + } + boolean success = imageCaptureService.captureImage(ip, path); + + List detectResults = onnxServiceNew.detect26(path, "qrCode"); + opencvService.drawBoundingBoxesOnImage(detectResults, path,path+".jpg"); + log.info("检测结果: {}", detectResults); + + // 读取原始图片 + BufferedImage originalImage = ImageIO.read(new File(path)); + if (originalImage == null) { + log.error("无法读取图片: {}", path); + boxCountResponse.setResult("读取图片失败"); + } + + // 存储解码结果 + List qrCodeResults = new ArrayList<>(); + + for (BoundingBox box : detectResults){ + try { + // 根据 BoundingBox 裁剪图片 + BufferedImage croppedImage = cropBoundingBox(originalImage, box,30); + ImageIO.write(croppedImage, "jpg", new File(path+"_"+box.getIndex()+".jpg")); + + // 使用 ZXing 读取二维码 + String qrCodeContent = WeChatDeCode.deCode(croppedImage); + qrCodeResults.add(qrCodeContent); + if (qrCodeContent != null) { + + log.info("解码成功: {}, 置信度: {}", qrCodeContent, box.getConfidence()); + } else { + log.warn("解码失败，box: {}", box); + } + } catch (Exception e) { + log.error("解码二维码异常，box: {}", box, e); + } + } + + boxCountResponse.setQrCodeResults(qrCodeResults); + + } catch (OrtException e) { + log.error("检测异常", e); + } catch (Exception e) { + log.error("处理异常", e); + } + boxCountResponse.setResult("Unknown"); + } + return null; + } +// +// public static void main(String[] args) { +// String imagePath = "D:\\data\\media\\2026-04-17\\1\\QQ20260417-134439.jpg"; +// BufferedImage image = null; +// try { +// image = ImageIO.read(new File(imagePath)); +// } catch (IOException e) { +// throw new RuntimeException(e); +// } +// BufferedImageLuminanceSource source = new BufferedImageLuminanceSource(image); +// HybridBinarizer binarizer = new HybridBinarizer(source); +// BinaryBitmap bitmap = new BinaryBitmap(binarizer); +// MultiFormatReader multiFormatReader = new MultiFormatReader(); +// try { +// System.out.println(multiFormatReader.decode(bitmap).getText()); ; +// } catch (NotFoundException e) { +// throw new RuntimeException("二维码未找到或无法解码"); +// } +// } + +// public static void main(String... args) { +// +// String imagePath = "D:\\data\\media\\2026-04-17\\1\\QQ20260417-134439.jpg"; +// Mat img = Imgcodecs.imread(imagePath); +// System.out.println(deCode(img)); +// } + + private static String deCode(Mat img) { + // 微信二维码对象，要返回二维码坐标前2个参数必传；后2个在二维码小或不清晰时必传。 + WeChatQRCode we = new WeChatQRCode(); +// List points = new ArrayList(); + // 微信二维码引擎解码，返回的valList中存放的是解码后的数据，points中Mat存放的是二维码4个角的坐标 + StringVector stringVector = we.detectAndDecode(img); + if (stringVector.empty()) { + return "0"; + } + return stringVector.get(0).getString(StandardCharsets.UTF_8); + } + + /** + * 根据 BoundingBox 裁剪图片 + * + * @param originalImage 原始图片 + * @param box 边界框（detect26 输出：左上角坐标 x, y 和宽高 w, h，都是绝对像素坐标） + * @return 裁剪后的图片 + */ + private BufferedImage cropBoundingBox(BufferedImage originalImage, BoundingBox box) { + return cropBoundingBox(originalImage, box, 0); + } + + /** + * 根据 BoundingBox 裁剪图片（支持扩展区域） + * + * @param originalImage 原始图片 + * @param box 边界框（detect26 输出：左上角坐标 x, y 和宽高 w, h，都是绝对像素坐标） + * @param expandPixels 扩展像素数（上下左右各扩展的像素值） + * @return 裁剪后的图片 + */ + private BufferedImage cropBoundingBox(BufferedImage originalImage, BoundingBox box, int expandPixels) { + int imageWidth = originalImage.getWidth(); + int imageHeight = originalImage.getHeight(); + + // detect26 输出的 x, y 已经是左上角坐标，w, h 是宽高，都是绝对像素坐标 + int x = (int) box.getX(); + int y = (int) box.getY(); + int width = (int) box.getW(); + int height = (int) box.getH(); + + // 应用扩展：向左上角扩展 + x = x - expandPixels; + y = y - expandPixels; + + // 应用扩展：向右下角扩展（增加宽高） + width = width + (expandPixels * 2); + height = height + (expandPixels * 2); + + // 确保坐标在图片范围内 + x = Math.max(0, Math.min(x, imageWidth - 1)); + y = Math.max(0, Math.min(y, imageHeight - 1)); + + // 确保宽高不超出图片边界 + width = Math.min(width, imageWidth - x); + height = Math.min(height, imageHeight - y); + + // 确保宽高至少为 1 + width = Math.max(1, width); + height = Math.max(1, height); + + log.info("裁剪区域: x={}, y={}, width={}, height={}, 扩展像素={}, 原图尺寸: {}x{}", + x, y, width, height, expandPixels, imageWidth, imageHeight); + + // 裁剪图片 + return originalImage.getSubimage(x, y, width, height); + } + + /** + * 使用 ZXing 读取二维码 + * + * @param image 图片 + * @return 二维码内容，如果解码失败返回 null + */ + private String decodeQRCode(BufferedImage image) { + try { + // 创建一个二进制图像 + LuminanceSource source = new BufferedImageLuminanceSource(image); + BinaryBitmap binaryBitmap = new BinaryBitmap(new HybridBinarizer(source)); + + // 创建解码器 + QRCodeReader reader = new QRCodeReader(); + + // 解码二维码 + Result result = reader.decode(binaryBitmap); + + return result.getText(); + } catch (NotFoundException e) { + log.debug("未找到二维码"); + return null; + } catch (ChecksumException e) { + log.warn("二维码校验失败: {}", e.getMessage()); + return null; + } catch (FormatException e) { + log.warn("二维码格式错误: {}", e.getMessage()); + return null; + } catch (Exception e) { + log.error("解码二维码异常", e); + return null; + } + } + + +} diff --git a/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/QrCode/WeChatDeCode.java b/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/QrCode/WeChatDeCode.java index 6af3722..d93d4ea 100644 --- a/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/QrCode/WeChatDeCode.java +++ b/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/QrCode/WeChatDeCode.java @@ -20,6 +20,7 @@ import java.util.UUID; import javax.imageio.ImageIO; import static org.bytedeco.opencv.global.opencv_imgcodecs.imread; +import static org.opencv.core.CvType.CV_8UC3; /** * @@ -27,7 +28,8 @@ import static org.bytedeco.opencv.global.opencv_imgcodecs.imread; public class WeChatDeCode { public static void main(String... args) { - Mat img = imread("D:\\data\\QQ20260331-175110.jpg"); + String imagePath = "D:\\data\\media\\2026-04-17\\1\\1776408379598.png_0.jpg"; + Mat img = imread(imagePath); System.out.println(deCode(img)); //下载二维码到本地识别 @@ -122,8 +124,50 @@ public class WeChatDeCode { e.printStackTrace(); } } + public static String deCode(BufferedImage croppedImage) { + Mat img = bufferedImageToMat(croppedImage); + try { + return deCode(img); + } finally { + if (img != null) { + img.close(); + } + } + } + + private static Mat bufferedImageToMat(BufferedImage image) { + if (image == null) { + return null; + } + + int width = image.getWidth(); + int height = image.getHeight(); - private static String deCode(Mat img) { + // 创建 Mat 对象 + Mat mat = new Mat(height, width, CV_8UC3); + + // 获取像素数据 + int[] pixels = new int[width * height]; + image.getRGB(0, 0, width, height, pixels, 0, width); + + // 将像素数据复制到 Mat + byte[] data = new byte[width * height * 3]; + for (int i = 0; i < width * height; i++) { + int pixel = pixels[i]; + int r = (pixel >> 16) & 0xFF; + int g = (pixel >> 8) & 0xFF; + int b = pixel & 0xFF; + + // OpenCV 使用 BGR 格式 + data[i * 3] = (byte) b; + data[i * 3 + 1] = (byte) g; + data[i * 3 + 2] = (byte) r; + } + + mat.data().put(data); + return mat; + } + public static String deCode(Mat img) { // 微信二维码对象，要返回二维码坐标前2个参数必传；后2个在二维码小或不清晰时必传。 WeChatQRCode we = new WeChatQRCode(); diff --git a/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/algorithm/ONNXServiceNew.java b/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/algorithm/ONNXServiceNew.java index 708ad93..58e68f8 100644 --- a/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/algorithm/ONNXServiceNew.java +++ b/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/algorithm/ONNXServiceNew.java @@ -664,7 +664,7 @@ public class ONNXServiceNew { detection.setH(h); detection.setConfidence(conf); detection.setIndex(label); - detection.setName(config.getNames()[label]); +// detection.setName(config.getNames()[label]); detections.add(detection); } @@ -686,6 +686,437 @@ public class ONNXServiceNew { return arg; } + /** + * YOLO26 专用检测方法 + * 处理 YOLO26 的两种输出格式： + * 1. [1, 300, 6]: NMS 后的输出，格式为 [x1, y1, x2, y2, conf, class_id] + * 2. [1, 84, 8400]: YOLOv8 原始输出，84 = 4(xywh) + 1(conf) + 79(classes) + * + * @param imagePath 输入图片路径 + * @param type 模型类型 + * @return 检测结果列表 + * @throws OrtException + */ + public List detect26(String imagePath, String type) throws OrtException { + + OrtSession session = getSession(type); + AppConfig.YoloModelConfig config = getConfig(type); + // 处理图像 + float[] imageData = new float[0]; + try { + imageData = processImageFromURL(imagePath, config.getImageSize()); + } catch (IOException ex) { + log.error("处理图像时出错: {}", ex.getMessage(), ex); + return new ArrayList<>(); + } + + // 构建输入张量 + long[] shape = new long[]{1, 3, config.getImageSize(), config.getImageSize()}; + OnnxTensor inputTensor = null; + OrtSession.Result result = null; + try { + inputTensor = OnnxTensor.createTensor(environment, FloatBuffer.wrap(imageData), shape); + + HashMap stringOnnxTensorHashMap = new HashMap<>(); + stringOnnxTensorHashMap.put(session.getInputInfo().keySet().iterator().next(), inputTensor); + + // 执行推理 + result = session.run(stringOnnxTensorHashMap); + + log.info("YOLO26 输出数量: {}", result.size()); + + // 获取第一个输出（检测输出） + float[][][] detOutput = (float[][][]) result.get(0).getValue(); + + // 调试：打印输出维度 + int batch = detOutput.length; + int dim1 = detOutput[0].length; + int dim2 = detOutput[0][0].length; + log.info("YOLO26 输出维度: [{}, {}, {}]", batch, dim1, dim2); + + // 判断输出格式 + // [1, 300, 6]: NMS 后的输出，格式为 [x1, y1, x2, y2, conf, class_id] + // [1, 84, 8400]: YOLOv8 原始输出格式 + + boolean isNMSOutput = (dim2 == 6); // 第三个维度是 6，说明是 NMS 后的输出 + + List detections = new ArrayList<>(); + + if (isNMSOutput) { + // [1, 300, 6] -> NMS 后的输出 + // 格式: [x1, y1, x2, y2, conf, class_id] + log.info("检测到 NMS 后的输出格式 [1, {}, 6]，直接解析", dim1); + + // 获取原始图像尺寸，用于坐标转换 + int originalWidth = 0; + int originalHeight = 0; + try { + File file = new File(imagePath); + BufferedImage originalImage = ImageIO.read(file); + if (originalImage != null) { + originalWidth = originalImage.getWidth(); + originalHeight = originalImage.getHeight(); + log.info("原始图像尺寸: {} x {}", originalWidth, originalHeight); + } + } catch (IOException e) { + log.warn("无法读取原始图像尺寸: {}", e.getMessage()); + } + + int imageSize = config.getImageSize(); + log.info("模型输入尺寸: {} x {}", imageSize, imageSize); + + double scale = (originalWidth > 0 && originalHeight > 0) + ? Math.min((double) imageSize / originalWidth, (double) imageSize / originalHeight) + : 1.0; + int scaledWidth = (int) (originalWidth * scale); + int scaledHeight = (int) (originalHeight * scale); + int offsetX = (imageSize - scaledWidth) / 2; + int offsetY = (imageSize - scaledHeight) / 2; + + log.info("缩放参数: scale={}, scaledWidth={}, scaledHeight={}, offsetX={}, offsetY={}", + scale, scaledWidth, scaledHeight, offsetX, offsetY); + + for (int i = 0; i < dim1; i++) { + float x1 = detOutput[0][i][0]; + float y1 = detOutput[0][i][1]; + float x2 = detOutput[0][i][2]; + float y2 = detOutput[0][i][3]; + float conf = detOutput[0][i][4]; + int classId = (int) detOutput[0][i][5]; + + // 如果置信度为 0 或低于阈值，跳过（NMS 后的输出，空的框 conf 为 0） + if (conf < config.getConfThreshold()) { + continue; + } + + // 将检测框坐标从缩放后图像转换回原始图像 + // 先减去偏移，再除以缩放比例 + x1 = (float) ((x1 - offsetX) / scale); + y1 = (float) ((y1 - offsetY) / scale); + x2 = (float) ((x2 - offsetX) / scale); + y2 = (float) ((y2 - offsetY) / scale); + + // xyxy -> xywh（YOLO 的 xywh 是左上角坐标 + 宽高） + float x = x1; + float y = y1; + float w = x2 - x1; + float h = y2 - y1; + + BoundingBox box = new BoundingBox(); + box.setX(x); + box.setY(y); + box.setW(w); + box.setH(h); + box.setConfidence(conf); + box.setIndex(classId); + if (classId < config.getNames().length) { + box.setName(config.getNames()[classId]); + } + detections.add(box); + } + } else { + // YOLOv8 原始输出格式 [1, 84, 8400] + // 需要 transpose(1, 0) 变成 [8400, 84] + boolean isYOLOv8Format = (dim1 > dim2); + + // 获取原始图像尺寸，用于坐标转换 + int originalWidth = 0; + int originalHeight = 0; + try { + File file = new File(imagePath); + BufferedImage originalImage = ImageIO.read(file); + if (originalImage != null) { + originalWidth = originalImage.getWidth(); + originalHeight = originalImage.getHeight(); + log.info("原始图像尺寸: {} x {}", originalWidth, originalHeight); + } + } catch (IOException e) { + log.warn("无法读取原始图像尺寸: {}", e.getMessage()); + } + + int imageSize = config.getImageSize(); + double scale = (originalWidth > 0 && originalHeight > 0) + ? Math.min((double) imageSize / originalWidth, (double) imageSize / originalHeight) + : 1.0; + int scaledWidth = (int) (originalWidth * scale); + int scaledHeight = (int) (originalHeight * scale); + int offsetX = (imageSize - scaledWidth) / 2; + int offsetY = (imageSize - scaledHeight) / 2; + + int numPredictions, numFeatures; + + if (isYOLOv8Format) { + // [1, 84, 8400] -> 需要转置 + numPredictions = dim2; // 8400 + numFeatures = dim1; // 84 + log.info("检测到 YOLOv8 格式 [1, {}, {}]，转置后 [8400, {}]", dim1, dim2, numFeatures); + } else { + // [1, 8400, 84] + numPredictions = dim1; // 8400 + numFeatures = dim2; // 84 + log.info("检测到标准格式 [1, {}, {}]", dim1, dim2); + } + + log.info("预测框数量: {}, 特征数: {}", numPredictions, numFeatures); + + // 解析每个预测框 + for (int i = 0; i < numPredictions; i++) { + float x, y, w, h, conf; + + if (isYOLOv8Format) { + // detOutput[0] 是 [84, 8400] + // 格式: [x, y, w, h, conf, class_scores...] + + x = detOutput[0][0][i]; + y = detOutput[0][1][i]; + w = detOutput[0][2][i]; + h = detOutput[0][3][i]; + conf = detOutput[0][4][i]; + + if (conf < config.getConfThreshold()) { + continue; + } + + // 将检测框坐标从缩放后图像转换回原始图像 + // YOLOv8 的 xywh 是中心点坐标，需要转换为左上角坐标 + // 然后减去偏移，再除以缩放比例 + x = (float) ((x - w / 2 - offsetX) / scale); + y = (float) ((y - h / 2 - offsetY) / scale); + w = (float) (w / scale); + h = (float) (h / scale); + + float[] clsScores = new float[numFeatures - 5]; + for (int j = 0; j < clsScores.length; j++) { + clsScores[j] = detOutput[0][5 + j][i]; + } + + int classId = argmax(clsScores, 0.0f); + + if (classId != -1) { + BoundingBox box = new BoundingBox(); + box.setX(x); + box.setY(y); + box.setW(w); + box.setH(h); + box.setConfidence(conf); + box.setIndex(classId); + if (classId < config.getNames().length) { + box.setName(config.getNames()[classId]); + } + detections.add(box); + } + } else { + // detOutput[0] 是 [8400, 84] + x = detOutput[0][i][0]; + y = detOutput[0][i][1]; + w = detOutput[0][i][2]; + h = detOutput[0][i][3]; + conf = detOutput[0][i][4]; + + if (conf < config.getConfThreshold()) { + continue; + } + + // 将检测框坐标从缩放后图像转换回原始图像 + // YOLOv8 的 xywh 是中心点坐标，需要转换为左上角坐标 + // 然后减去偏移，再除以缩放比例 + x = (float) ((x - w / 2 - offsetX) / scale); + y = (float) ((y - h / 2 - offsetY) / scale); + w = (float) (w / scale); + h = (float) (h / scale); + + float[] clsScores = new float[numFeatures - 5]; + for (int j = 0; j < clsScores.length; j++) { + clsScores[j] = detOutput[0][i][5 + j]; + } + + int classId = argmax(clsScores, 0.0f); + + if (classId != -1) { + BoundingBox box = new BoundingBox(); + box.setX(x); + box.setY(y); + box.setW(w); + box.setH(h); + box.setConfidence(conf); + box.setIndex(classId); + if (classId < config.getNames().length) { + box.setName(config.getNames()[classId]); + } + detections.add(box); + } + } + } + } + + // NMS 过滤 + List filteredDetections = nonMaxSuppression(detections, 0.5f); + + log.info("YOLO26 检测结果: {} 个", filteredDetections.size()); + return filteredDetections; + } finally { + if (inputTensor != null) { + inputTensor.close(); + } + if (result != null) { + result.close(); + } + } + } + + /** + * YOLO26 专用分类方法 + * 处理 YOLO26 的两种输出格式： + * 1. [1, 300, 6]: NMS 后的输出，格式为 [x1, y1, x2, y2, conf, class_id] + * 2. [1, 84, 8400]: YOLOv8 原始输出，84 = 4(xywh) + 1(conf) + 79(classes) + * + * @param imagePath 输入图片路径 + * @param type 模型类型 + * @return 分类结果 + * @throws OrtException + */ + public ClassifyEntity classify26(String imagePath, String type) throws OrtException { + + OrtSession session = getSession(type); + AppConfig.YoloModelConfig config = getConfig(type); + // 处理图像 + float[] imageData = new float[0]; + try { + imageData = processImageFromURL(imagePath, config.getImageSize()); + } catch (IOException ex) { + log.error("处理图像时出错: {}", ex.getMessage(), ex); + ClassifyEntity errorResult = new ClassifyEntity(); + errorResult.setName("Unknown"); + return errorResult; + } + + // 构建输入张量 + long[] shape = new long[]{1, 3, config.getImageSize(), config.getImageSize()}; + OnnxTensor inputTensor = null; + OrtSession.Result result = null; + try { + inputTensor = OnnxTensor.createTensor(environment, FloatBuffer.wrap(imageData), shape); + + HashMap stringOnnxTensorHashMap = new HashMap<>(); + stringOnnxTensorHashMap.put(session.getInputInfo().keySet().iterator().next(), inputTensor); + + // 执行推理 + result = session.run(stringOnnxTensorHashMap); + + log.info("YOLO26 输出数量: {}", result.size()); + + // YOLO26 第一个输出是检测输出 + float[][][] detOutput = (float[][][]) result.get(0).getValue(); + + // 判断输出格式 + int dim1 = detOutput[0].length; + int dim2 = detOutput[0][0].length; + boolean isNMSOutput = (dim2 == 6); // NMS 后的输出 + + float bestConfidence = -Float.MAX_VALUE; + int bestClassId = -1; + + if (isNMSOutput) { + // [1, 300, 6] -> NMS 后的输出，格式为 [x1, y1, x2, y2, conf, class_id] + log.info("YOLO26 分类检测到 NMS 后的输出格式 [1, {}, 6]", dim1); + + for (int i = 0; i < dim1; i++) { + float conf = detOutput[0][i][4]; + int classId = (int) detOutput[0][i][5]; + + if (conf < config.getConfThreshold()) { + continue; + } + + if (conf > bestConfidence) { + bestConfidence = conf; + bestClassId = classId; + } + } + } else { + // YOLOv8 原始输出格式 + boolean isYOLOv8Format = (dim1 > dim2); + int numPredictions = isYOLOv8Format ? dim2 : dim1; + int numFeatures = isYOLOv8Format ? dim1 : dim2; + + log.info("YOLO26 分类预测框数量: {}, 特征数: {}, YOLOv8格式: {}", + numPredictions, numFeatures, isYOLOv8Format); + + // 找到置信度最高的类别 + for (int i = 0; i < numPredictions; i++) { + float conf; + int classId; + float[] clsScores; + + if (isYOLOv8Format) { + // detOutput[0] 是 [84, 8400] + conf = detOutput[0][4][i]; + + if (conf < config.getConfThreshold()) { + continue; + } + + // 获取类别分数 + clsScores = new float[numFeatures - 5]; + for (int j = 0; j < clsScores.length; j++) { + clsScores[j] = detOutput[0][5 + j][i]; + } + + classId = argmax(clsScores, 0.0f); + float classScore = clsScores[classId]; + + if (classScore > bestConfidence) { + bestConfidence = classScore; + bestClassId = classId; + } + } else { + // detOutput[0] 是 [8400, 84] + conf = detOutput[0][i][4]; + + if (conf < config.getConfThreshold()) { + continue; + } + + clsScores = new float[numFeatures - 5]; + for (int j = 0; j < clsScores.length; j++) { + clsScores[j] = detOutput[0][i][5 + j]; + } + + classId = argmax(clsScores, 0.0f); + float classScore = clsScores[classId]; + + if (classScore > bestConfidence) { + bestConfidence = classScore; + bestClassId = classId; + } + } + } + } + + ClassifyEntity classifyEntity = new ClassifyEntity(); + classifyEntity.setIndex(bestClassId); + + if (bestClassId != -1 && bestClassId < config.getNames().length) { + String className = config.getNames()[bestClassId]; + classifyEntity.setName(className); + classifyEntity.setConfidence(bestConfidence); + log.info("YOLO26 识别模版：{}，置信度：{}", className, bestConfidence); + } else { + log.info("YOLO26 未识别到模版"); + classifyEntity.setName("Unknown"); + } + + return classifyEntity; + } finally { + if (inputTensor != null) { + inputTensor.close(); + } + if (result != null) { + result.close(); + } + } + } + /** * 销毁时清理资源 */ diff --git a/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/camera/lx/config/BoxCountResponse.java b/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/camera/lx/config/BoxCountResponse.java index 7809dd1..0a99335 100644 --- a/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/camera/lx/config/BoxCountResponse.java +++ b/src/main/java/com/example/lxcameraapi/service/IndustrialCamera/camera/lx/config/BoxCountResponse.java @@ -5,6 +5,7 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; import java.util.Map; /** @@ -84,6 +85,11 @@ public class BoxCountResponse { private String imagePath; private String result; + /** + * 二维码解码结果列表 + */ + private List qrCodeResults; + /** * 从 BoxCountResult 创建响应对象 */ diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 0929a3c..6f7a701 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -57,9 +57,18 @@ picUrl: "http://127.0.0.1:9012/pic/" yoloModelConfig: # 模型地址 - modelPath: "D:/PycharmProjects/yolo/runs/detect/train22/weights/best.onnx" + modelPath: "D:/git/测试/lxCameraApi/yolo/qrcode.onnx" # 图片大小 imgSize: 640 # 置信度 confThreshold: 0.5 names: ['0143','0153','0173','0177','0191','0253','0256','0266','0268','0286','0302','0304','0305','0307','0320','0326','0336','0339','0343','0352','0458','0461','0462','0473','0477','0486','0490','0492','0930','1101','1102','1104','1262','1269','1302','1308','1359','1366','1622','1625','1919','1976','20','2165','2188','2210','2224','2445','2476','2611','2730','2731','2910','2914','2943','3027','3028','3029','3212','3226','3344','3501','3509','3538','3725','3741','3751','3754','3763','3766','3808'] + modelMap: + qrCode: # 模型地址 + modelPath: "D:/git/测试/lxCameraApi/yolo/best.onnx" + # 图片大小 + imgSize: 2048 + # 置信度 + confThreshold: 0.95 + names: ['code'] +