package com.example.lxcameraapi.service; import ai.onnxruntime.OrtException; import com.example.lxcameraapi.conf.AppConfig; import com.example.lxcameraapi.service.IndustrialCamera.algorithm.ONNXServiceNew; import com.example.lxcameraapi.service.IndustrialCamera.camera.lx.LxCameraService; import com.example.lxcameraapi.service.IndustrialCamera.camera.lx.config.BoxCountRequest; import com.example.lxcameraapi.service.IndustrialCamera.camera.lx.config.BoxCountResult; import com.example.lxcameraapi.service.IndustrialCamera.camera.lx.config.CaptureResult; import com.example.lxcameraapi.service.IndustrialCamera.opencv.OpencvService; import com.example.lxcameraapi.service.IndustrialCamera.yolo.BoundingBox; import com.example.lxcameraapi.service.IndustrialCamera.QrCode.WeChatDeCode; import org.opencv.core.Mat; import com.google.zxing.BinaryBitmap; import com.google.zxing.Result; import com.google.zxing.client.j2se.BufferedImageLuminanceSource; import com.google.zxing.common.HybridBinarizer; import com.google.zxing.qrcode.QRCodeReader; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import jakarta.annotation.Resource; import java.awt.*; import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import javax.imageio.ImageIO; @Slf4j @Service public class AsyncProcessingService { @Resource private ONNXServiceNew onnxServiceNew; @Resource private OpencvService opencvService; @Resource private LxCameraService lxCameraService; @Resource private HttpNotifyService httpNotifyService; @Resource private AppConfig appConfig; /** * 异步处理品规识别 */ @Async public void processCategoryAsync(BoxCountRequest request, String path, String url, String targetIp, Integer targetPort, String targetPath) { try { log.info("异步品规识别任务开始"); List detectResults = onnxServiceNew.detect26(path, null); opencvService.drawBoundingBoxesOnImage(detectResults, path, path + "detect.jpg"); log.info("检测结果: {}", detectResults); String result = "Unknown"; if (!detectResults.isEmpty()) { // 统计每个 index 出现的次数 Map indexCountMap = detectResults.stream() .collect(Collectors.groupingBy( BoundingBox::getIndex, Collectors.counting() )); // 找到出现次数最多的 index Map.Entry maxIndexEntry = indexCountMap.entrySet().stream() .max(Map.Entry.comparingByValue()) .orElse(null); if (maxIndexEntry != null) { // 根据最多的 index 找到对应的 name result = detectResults.stream() .filter(box -> box.getIndex().equals(maxIndexEntry.getKey())) .findFirst() .map(BoundingBox::getName) .orElse("Unknown"); log.info("检测结果统计 - 最常见的类别: {}, 出现次数: {}", result, maxIndexEntry.getValue()); } } // 发送HTTP通知到 singleInventoryReturn if (targetIp != null && targetPort != null && targetPath != null) { httpNotifyService.sendBoxCountResult(targetIp, targetPort, targetPath, request.getTaskId(), request.getScanType(), result, url, 0); } else { log.warn("异步任务完成,但未配置目标服务器信息"); } } catch (OrtException e) { log.error("异步品规识别异常", e); } catch (Exception e) { log.error("异步品规识别异常", e); } } /** * 异步处理箱子计数 */ @Async public void processBoxCountAsync(BoxCountRequest request, CaptureResult captureResult, String targetIp, Integer targetPort, String targetPath) { try { log.info("异步箱子计数任务开始"); request.setPcdFilePath(captureResult.getFilePath("pointCloud")); BoxCountResult result = lxCameraService.countBoxes(request); log.info("箱子数量计算结果: {}", result.getSummary()); String imagePath = captureResult.getFilePath("rgb").replace(appConfig.getPicPath(), appConfig.getPicUrl()); String resultStr = String.valueOf(result.getTotalBoxCount()); // 发送HTTP通知到 singleInventoryReturn if (targetIp != null && targetPort != null && targetPath != null) { httpNotifyService.sendBoxCountResult(targetIp, targetPort, targetPath, request.getTaskId(), request.getScanType(), resultStr, imagePath, result.getTotalBoxCount()); } else { log.warn("异步任务完成,但未配置目标服务器信息"); } } catch (Exception e) { log.error("异步箱子计数异常", e); } } /** * 异步处理二维码识别 */ @Async public void processQrCodeAsync(Integer cameraId, String taskId, String path, String url, String targetIp, Integer targetPort, String targetPath, String scanType) { try { log.info("异步识别任务开始,路径: {}", path); List detectResults = onnxServiceNew.detect26(path, "qrCode"); log.info("异步检测结果: {}", detectResults); // 读取原始图片 BufferedImage originalImage = ImageIO.read(new File(path)); if (originalImage == null) { log.error("异步任务:无法读取图片: {}", path); return; } // 存储解码结果 List qrCodeResults = new ArrayList<>(); List detectCVResults = new ArrayList<>(); for (int i = 0; i < detectResults.size(); i++) { BoundingBox box = detectResults.get(i); try { BufferedImage croppedImage = cropBoundingBox(originalImage, box, 30); // 最小尺寸放大 int minSize = 320; if (croppedImage.getWidth() < minSize || croppedImage.getHeight() < minSize) { int newWidth = Math.max(croppedImage.getWidth(), minSize); int newHeight = Math.max(croppedImage.getHeight(), minSize); croppedImage = resizeImage(croppedImage, newWidth, newHeight); } // 解码,最多重试2次(每次放大1.5倍) String qrCodeContent = decodeQrCodeWithRetry(croppedImage, box.getConfidence(),2); if (qrCodeContent != null) { qrCodeResults.add(qrCodeContent); detectCVResults.add(box); box.setName(qrCodeContent); }else { ImageIO.write(croppedImage, "jpg", new File(path + "_" + i + ".jpg")); } } catch (Exception e) { log.error("异步解码二维码异常,box: {}", box, e); } } opencvService.drawBoundingBoxesOnImage(detectCVResults, path, path+ ".jpg"); log.info("二维码识别结果: {}", qrCodeResults); // 构建结果 String result = "Unknown"; if (qrCodeResults.size() > 0 && qrCodeResults.get(0).length() >= 8) { result = qrCodeResults.get(0).substring(0, 8); } else if (qrCodeResults.size() > 0) { result = qrCodeResults.get(0); } String imagePath = url + ".jpg"; // 发送HTTP通知到 singleInventoryQrCodeReturn if (targetIp != null && targetPort != null && targetPath != null) { httpNotifyService.sendQrCodeResult(targetIp, targetPort, targetPath, taskId, result, imagePath, qrCodeResults, scanType); } else { log.warn("异步任务完成,但未配置目标服务器信息"); } } catch (Exception e) { log.error("异步任务执行异常", e); } } /** * 异步处理多图片二维码识别 * 所有图片识别完成后统一发送结果 */ @Async public void processMultiQrCodeAsync(Integer cameraId, String taskId, List imagePaths, List imageUrls, String targetIp, Integer targetPort, String targetPath, String scanType) { try { log.info("异步多图片识别任务开始,图片数量: {}", imagePaths.size()); // 收集所有图片的二维码结果 List allQrCodeResults = new ArrayList<>(); List allImagePaths = new ArrayList<>(); for (int i = 0; i < imagePaths.size(); i++) { String path = imagePaths.get(i); String url = imageUrls.get(i); try { log.info("开始识别第{}/{}张图片: {}", i + 1, imagePaths.size(), path); List detectResults = onnxServiceNew.detect26(path, "qrCode"); log.info("第{}张图片检测结果: {}", i + 1, detectResults); // 读取原始图片 BufferedImage originalImage = ImageIO.read(new File(path)); if (originalImage == null) { log.warn("无法读取图片: {}", path); continue; } List qrCodeResults = new ArrayList<>(); List validBoxes = new ArrayList<>(); for (int j = 0; j < detectResults.size(); j++) { BoundingBox box = detectResults.get(j); try { BufferedImage croppedImage = cropBoundingBox(originalImage, box, 30); // 保存裁剪的原始图片用于调试 String debugPath = path + "_crop_" + i + "_" + j + ".jpg"; ImageIO.write(croppedImage, "jpg", new File(debugPath)); log.info("DEBUG: 裁剪图片保存到: {}, 尺寸: {}x{}, 置信度: {}", debugPath, croppedImage.getWidth(), croppedImage.getHeight(), box.getConfidence()); // 最小尺寸放大 int minSize = 320; if (croppedImage.getWidth() < minSize || croppedImage.getHeight() < minSize) { int newWidth = Math.max(croppedImage.getWidth(), minSize); int newHeight = Math.max(croppedImage.getHeight(), minSize); croppedImage = resizeImage(croppedImage, newWidth, newHeight); log.info("DEBUG: 图片已放大到: {}x{}", newWidth, newHeight); } // 解码 String qrCodeContent = decodeQrCodeWithRetry(croppedImage, box.getConfidence(), 2); if (qrCodeContent != null) { qrCodeResults.add(qrCodeContent); validBoxes.add(box); } } catch (Exception e) { log.error("第{}张图片解码异常,box: {}", i + 1, box, e); } } // 绘制检测框 opencvService.drawBoundingBoxesOnImage(validBoxes, path, path + ".jpg"); // 收集结果 allQrCodeResults.addAll(qrCodeResults); allImagePaths.add(url + ".jpg"); } catch (Exception e) { log.error("第{}张图片处理异常: {}", i + 1, path, e); } } log.info("多图片识别完成,总二维码数量: {}", allQrCodeResults.size()); // 构建结果 String result = "Unknown"; if (!allQrCodeResults.isEmpty() && allQrCodeResults.get(0).length() >= 8) { result = allQrCodeResults.get(0).substring(0, 8); } else if (!allQrCodeResults.isEmpty()) { result = allQrCodeResults.get(0); } // 发送HTTP通知 if (targetIp != null && targetPort != null && targetPath != null) { httpNotifyService.sendQrCodeResult(targetIp, targetPort, targetPath, taskId, result, String.join(",", allImagePaths), allQrCodeResults, scanType); } else { log.warn("多图片异步任务完成,但未配置目标服务器信息"); } } catch (Exception e) { log.error("多图片异步任务执行异常", e); } } public static void main(String[] args) { String path = "D:\\data\\media\\2026-05-15\\1\\2026-05-15-00-07-20-909.png_3.jpg"; BufferedImage croppedImage = null; AsyncProcessingService asyncProcessingService = new AsyncProcessingService(); String qrCodeContent = null; try { croppedImage = ImageIO.read(new File(path)); // 如果图片小于320px,放大到320px // 最小尺寸放大 int minSize = 320; if (croppedImage.getWidth() < minSize || croppedImage.getHeight() < minSize) { int newWidth = Math.max(croppedImage.getWidth(), minSize); int newHeight = Math.max(croppedImage.getHeight(), minSize); croppedImage = asyncProcessingService.resizeImage(croppedImage, newWidth, newHeight); } // 解码,最多重试2次(每次放大1.5倍) qrCodeContent = asyncProcessingService.decodeQrCodeWithRetry(croppedImage, 0.9f,2); } catch (IOException e) { throw new RuntimeException(e); } // ImageIO.write(croppedImage, "jpg", new File(path + "_" + i + ".jpg")); // 先尝试 ZXing 解码 System.out.println("解码结果: " + qrCodeContent); } private BufferedImage cropBoundingBox(BufferedImage originalImage, BoundingBox box, int expandPixels) { int imageWidth = originalImage.getWidth(); int imageHeight = originalImage.getHeight(); int x = (int) box.getX(); int y = (int) box.getY(); int width = (int) box.getW(); int height = (int) box.getH(); x = x - expandPixels; y = y - expandPixels; width = width + (expandPixels * 2); height = height + (expandPixels * 2); x = Math.max(0, Math.min(x, imageWidth - 1)); y = Math.max(0, Math.min(y, imageHeight - 1)); width = Math.min(width, imageWidth - x); height = Math.min(height, imageHeight - y); width = Math.max(1, width); height = Math.max(1, height); return originalImage.getSubimage(x, y, width, height); } /** * 带重试的二维码解码 * @param image 图片 * @param confidence 检测置信度 * @param maxRetries 最大重试次数(每次放大1.5倍) * @return 解码结果,null表示失败 */ private String decodeQrCodeWithRetry(BufferedImage image, double confidence, int maxRetries) { String qrCodeContent = null; BufferedImage currentImage = image; for (int retry = 0; retry <= maxRetries; retry++) { if (retry > 0) { // 放大图片1.5倍 int newWidth = (int) (currentImage.getWidth() * 1.5); int newHeight = (int) (currentImage.getHeight() * 1.5); currentImage = resizeImage(currentImage, newWidth, newHeight); log.debug("二维码放大重试 {},尺寸: {}x{}", retry, newWidth, newHeight); } // 尝试不同预处理方式 qrCodeContent = decodeWithPreprocessing(currentImage, confidence, retry); if (qrCodeContent != null) { return qrCodeContent; } } log.warn("异步解码失败,置信度: {}", confidence); return null; } /** * 使用不同预处理方式解码 */ private String decodeWithPreprocessing(BufferedImage image, double confidence, int retryIndex) { // 1. 尝试原始图片 String result = tryDecode(image, confidence, retryIndex, "原始"); if (result != null) return result; // 2. 尝试灰度化 try { log.info("DEBUG: 灰度化开始, image类型={}, 尺寸={}x{}", image.getType(), image.getWidth(), image.getHeight()); Mat mat = opencvService.bufferedImageToMat(image); log.info("DEBUG: bufferedImageToMat完成, mat类型={}, channels={}, depth={}", mat.type(), mat.channels(), mat.depth()); Mat gray = opencvService.toGrayscale(mat); log.info("DEBUG: toGrayscale完成, gray channels={}, depth={}", gray.channels(), gray.depth()); BufferedImage grayImage = opencvService.matToBufferedImage(gray); log.info("DEBUG: matToBufferedImage完成, grayImage类型={}", grayImage.getType()); mat.release(); gray.release(); result = tryDecode(grayImage, confidence, retryIndex, "灰度"); if (result != null) return result; } catch (Exception e) { log.error("灰度预处理失败: type={}, msg={}", e.getClass().getName(), e.getMessage(), e); } // 3. 尝试CLAHE增强 try { log.info("DEBUG: CLAHE开始"); Mat mat = opencvService.bufferedImageToMat(image); Mat clahe = opencvService.clahe(mat); log.info("DEBUG: clahe完成, clahe channels={}, depth={}", clahe.channels(), clahe.depth()); BufferedImage claheImage = opencvService.matToBufferedImage(clahe); mat.release(); clahe.release(); result = tryDecode(claheImage, confidence, retryIndex, "CLAHE"); if (result != null) return result; } catch (Exception e) { log.error("CLAHE预处理失败: type={}, msg={}", e.getClass().getName(), e.getMessage(), e); } // 4. 尝试二值化 try { log.info("DEBUG: 二值化开始"); Mat mat = opencvService.bufferedImageToMat(image); Mat binary = opencvService.adaptiveThreshold(mat); log.info("DEBUG: adaptiveThreshold完成, binary channels={}, depth={}", binary.channels(), binary.depth()); BufferedImage binaryImage = opencvService.matToBufferedImage(binary); mat.release(); binary.release(); result = tryDecode(binaryImage, confidence, retryIndex, "二值化"); if (result != null) return result; } catch (Exception e) { log.error("二值化预处理失败: type={}, msg={}", e.getClass().getName(), e.getMessage(), e); } // 5. 尝试组合预处理(去噪+灰度+CLAHE+二值化) try { log.info("DEBUG: 组合预处理开始"); Mat mat = opencvService.bufferedImageToMat(image); Mat processed = opencvService.preprocessForQrCode(mat); log.info("DEBUG: preprocessForQrCode完成, processed channels={}, depth={}", processed.channels(), processed.depth()); BufferedImage processedImage = opencvService.matToBufferedImage(processed); mat.release(); processed.release(); result = tryDecode(processedImage, confidence, retryIndex, "组合预处理"); if (result != null) return result; } catch (Exception e) { log.error("组合预处理失败: type={}, msg={}", e.getClass().getName(), e.getMessage(), e); } return null; } /** * 尝试解码 */ private String tryDecode(BufferedImage image, double confidence, int retryIndex, String preprocessType) { String qrCodeContent; // 先尝试 ZXing 解码 try { BufferedImageLuminanceSource source = new BufferedImageLuminanceSource(image); HybridBinarizer binarizer = new HybridBinarizer(source); BinaryBitmap bitmap = new BinaryBitmap(binarizer); QRCodeReader reader = new QRCodeReader(); Result result = reader.decode(bitmap); qrCodeContent = result.getText(); if (isValidQrCode(qrCodeContent)) { log.info("ZXing解码成功({}放大{}次): {}, 置信度: {}", preprocessType, retryIndex, qrCodeContent, confidence); return qrCodeContent; } else { log.warn("ZXing解码内容无效: {}, 继续尝试", qrCodeContent); } } catch (Exception zxingEx) { log.debug("ZXing解码失败({}): {}", preprocessType, zxingEx.getMessage()); } // ZXing 失败,尝试 WeChat 解码 try { qrCodeContent = WeChatDeCode.deCode(image); if (isValidQrCode(qrCodeContent)) { log.info("WeChat解码成功({}放大{}次): {}, 置信度: {}", preprocessType, retryIndex, qrCodeContent, confidence); return qrCodeContent; } else { log.warn("WeChat解码内容无效: {}, 继续尝试", qrCodeContent); } } catch (Exception wechatEx) { log.debug("WeChat解码失败({}): {}", preprocessType, wechatEx.getMessage()); } return null; } /** * 检查二维码内容是否有效 * @param content 二维码内容 * @return true表示有效 */ private boolean isValidQrCode(String content) { return content != null && !content.trim().isEmpty() && !content.equals("0"); } /** * 调整图片大小 */ private BufferedImage resizeImage(BufferedImage originalImage, int targetWidth, int targetHeight) { BufferedImage resizedImage = new BufferedImage(targetWidth, targetHeight, BufferedImage.TYPE_3BYTE_BGR); Graphics2D g2d = resizedImage.createGraphics(); g2d.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); g2d.drawImage(originalImage, 0, 0, targetWidth, targetHeight, null); g2d.dispose(); return resizedImage; } }