/*! \file ImageCompareModel.cpp Copyright (C) 2014 Hangzhou Leaper. Created by bang.jin at 2017/02/04. */ #include "ImageCompareModel.h" #include "CVUtils.h" #include "MultiScaleImageCompareModel.h" #include //#define DEBUG_VIEW_PLOT // #define DEBUG_VIEW_PLOT #define IMGCMP_STR_NAME "name" #define IMGCMP_STR_IMAGE_COMPARE_MODEL "imageCompareModel" #define IMGCMP_STR_ALIGN_BASE_IMAGE "alignBaseImage" #define IMGCMP_STR_COMPARE_BASE_IMAGE "compareBaseImage" #define IMGCMP_STR_WEIGHT_MAT "weightMat" #define IMGCMP_STR_MATCH_VAL_SCALE "matchValScale" #define IMGCMP_STR_TARGET_MEAN_VAL "targetMeanVal" #define IMGCMP_STR_TARGET_STDDEV_VAL "targetStddevVal" #define IMGCMP_STR_REPEAT_NUM "repeatNum" #define IMGCMP_STR_TRUE_SAMPLE_DIS_MEAN "trueSampleDisMean" #define IMGCMP_STR_TRUE_SAMPLE_DIS_STDDEV "trueSampleDisStddev" #define IMGCMP_STR_TRUE_SAMPLE_DIS_MIN "trueSampleDisMin" #define IMGCMP_STR_TRUE_SAMPLE_DIS_MAX "trueSampleDisMax" #define IMGCMP_STR_DIS_THRE "disThre" #define IMGCMP_STR_FALSE_SAMPLE_MIN_DIS "falseSampleMinDis" #define IMGCMP_STR_TRAIN_DIS "trainDis" #define IMGCMP_STR_SMALLEST_MATCH_DIS "falseMinDis" #define IMGCMP_STR_AVER_DIAMETER "averageDiameter" #define IMGCMP_CACHE_MAX_SIZE 100 using namespace std; // void meanvarnorm(Mat& src, Mat &dst, double avg, double var, Mat mask = Mat()) // { // Scalar m, v; // cv::meanStdDev(src, m, v, mask); // // (I - m)*var/v + avg = I*var/v - m*var/v + avg // double s = var / v.val[0]; // double t = avg - m.val[0] * var / v.val[0]; // dst = src * s + t; // } float angleNorm(float angle) { angle = angle - (int)(angle / 360.0) * 360; if (angle < 0) { angle += 360.0; } return angle; } float angleDis(float angle0, float angle1) { angle0 = angleNorm(angle0); angle1 = angleNorm(angle1); float dis = angle0 - angle1; float ldis = dis; if (dis < 0) { ldis = dis + 360; } else if (dis > 0) { ldis = dis - 360; } if (abs(ldis) < abs(dis)) { dis = ldis; } return dis; } cv::Point2f ImageCompareModel::refineCircleCen(const Mat& img, Point2f cen) { int r = 2; vector vec; double minDiff = DBL_MAX; Point2f bestCenter; vector bestDftVec; for (float y = cen.y - r; y <= cen.y + r; y += 0.5) { for (float x = cen.x - r; x <= cen.x + r; x += 0.5) { std::cout << "newCenter: " << x << "-" << y << ", "; Point2f newCenter(x, y); vector vec; selfRotationSimilarityFeature(img, vec, newCenter); vector dftVec; dft(vec, dftVec); // CvPlot::plot("refineCenter", &(vec[0]), vec.size()); float diffVal = sum(vec.begin(), vec.end()); // CvPlot::clear("dft"); // CvPlot::plot("dft", &(dftVec[0]), 25); // // float diffVal = dftVec[9]; vec.push_back(diffVal); if (minDiff > diffVal) { minDiff = diffVal; bestCenter = newCenter; bestDftVec = dftVec; } // CvPlot::clear("bestDft"); // CvPlot::plot("bestDft", &(bestDftVec[0]), 25); std::cout << "diff: " << diffVal << ", " " minDiff: " << minDiff << ", " " bestCenter: " << bestCenter.x << "-" << bestCenter.y << std::endl; } } return bestCenter; } cv::Mat ImageCompareModel::genWeightImage(const Mat& img, Point2f center, float innerR /*= -1*/, float outterR /*= -1*/) { if (innerR == -1) { // default is 30 innerR = img.rows*0.175; } if (outterR == -1) { // default is max radius - 10 outterR = img.rows*0.475; } Mat weightImg; weightImg.create(img.size(), CV_32FC1); weightImg.setTo(0); uchar* pData = weightImg.data; for (int y = 0; y < weightImg.rows; y++) { float* pFData = (float*)weightImg.row(y).data; for (int x = 0; x < weightImg.cols; x++) { float& val = pFData[x]; float dx = center.x - x; float dy = center.y - y; float r = sqrt(dx*dx + dy*dy); if (r > outterR || r < innerR) { val = 0; } else { val = min(outterR - r, r - innerR); } } } return weightImg; } void ImageCompareModel::selfRotationSimilarityFeature( const Mat& img, vector& vec, Point2f center /*= Point2f(-1, -1)*/) { float angleStep = 1.0; if (center.x < 0 || center.y < 0) { center = Point2f(img.cols / 2.0, img.rows / 2.0); } double bestVal = DBL_MAX; Mat bestRImg; Mat fImg; img.convertTo(fImg, CV_32FC1); Mat weightImg = genWeightImage(img, center); fImg = fImg.mul(weightImg); double sumVal = sum(weightImg).val[0]; Mat baseImg; { Mat t = getRotationMatrix2D(center, 1.0, 1.0); Mat rImg; warpAffine(fImg, rImg, t, fImg.size(), CV_INTER_CUBIC); t = getRotationMatrix2D(center, -1.0, 1.0); warpAffine(rImg, baseImg, t, rImg.size(), CV_INTER_CUBIC); } #ifdef DEBUG_VIEW_INTERNAL_MAT Mat viewBaseImg = baseImg / 255.0/25.0; Mat viewfImg = fImg / 255.0/25.0; #endif for (float a = 0; a <= 360.0; a += angleStep) { Mat t = getRotationMatrix2D(center, a, 1.0); Mat rImg; warpAffine(fImg, rImg, t, fImg.size(), CV_INTER_CUBIC); Mat diffImg = baseImg - rImg; //diffImg = diffImg.mul(weightImg); double diffVal = norm(diffImg)/sumVal; vec.push_back(diffVal); } } cv::Mat ImageCompareModel::genMask(const Mat& img, Point2f center, float innerR /*= -1*/, float outterR /*= -1*/, int type /*= CV_32FC1*/) { Mat mask(img.size(), CV_8UC1); mask.setTo(0); if (innerR == -1) { // default is 30 innerR = img.rows*0.178; } if (outterR == -1) { // default is max radius - 10 outterR = img.rows*0.425; } circle(mask, center, outterR, Scalar(255), -1); circle(mask, center, innerR, Scalar(0), -1); if (type != CV_8UC1) { converToType(mask, type); mask /= 255; } return mask; } void ImageCompareModel::genMask() { m32fMaskImg = genMask(mAlignBaseImg, Point2f(mAlignBaseImg.cols / 2.0, mAlignBaseImg.rows / 2.0)); m8uMaskImg = genMask(mAlignBaseImg, Point2f(mAlignBaseImg.cols / 2.0, mAlignBaseImg.rows / 2.0), -1.0, -1.0, CV_8UC1); } void ImageCompareModel::printInfo() { std::cout << IMGCMP_STR_IMAGE_COMPARE_MODEL << ": "; std::cout << std::endl; printProperty(1, IMGCMP_STR_NAME, mName); printIndent(1); std::cout << IMGCMP_STR_ALIGN_BASE_IMAGE << ": "; std::cout << std::endl; printMatInfo(mAlignBaseImg, 2); std::cout << std::endl; printIndent(1); std::cout << IMGCMP_STR_WEIGHT_MAT << ": "; std::cout << std::endl; printMatInfo(mWeightMat, 2); std::cout << std::endl; printProperty(1, IMGCMP_STR_MATCH_VAL_SCALE, mMatchValScale); printProperty(1, IMGCMP_STR_TARGET_MEAN_VAL, mTargetMeanVal); printProperty(1, IMGCMP_STR_TARGET_STDDEV_VAL, mTargetStddevVal); printProperty(1, IMGCMP_STR_REPEAT_NUM, mRepeatNum); printProperty(1, IMGCMP_STR_DIS_THRE, mDisThre); printProperty(1, IMGCMP_STR_TRUE_SAMPLE_DIS_STDDEV, mTrueSampleDisStddev); printProperty(1, IMGCMP_STR_TRUE_SAMPLE_DIS_MEAN, mTrueSampleDisMean); printProperty(1, IMGCMP_STR_TRUE_SAMPLE_DIS_MAX, mTrueSampleDisMax); printProperty(1, IMGCMP_STR_TRUE_SAMPLE_DIS_MIN, mTrueSampleDisMin); printProperty(1, "filePath", mFilePath); printProperty(1, "isEnableCache", mIsEnableCache); } void ImageCompareModel::saveImages(string dirPath) { imwrite(dirPath + "/base.png", mAlignBaseImg); imwrite(dirPath + "/weight.png", mWeightMat); } bool ImageCompareModel::save2file(string filePath) { FileStorage fs(filePath, FileStorage::WRITE); if (!fs.isOpened()) { return false; } fs << IMGCMP_STR_NAME << mName << IMGCMP_STR_ALIGN_BASE_IMAGE << mAlignBaseImg << IMGCMP_STR_COMPARE_BASE_IMAGE << mCompareBaseImg << IMGCMP_STR_WEIGHT_MAT << mWeightMat << IMGCMP_STR_MATCH_VAL_SCALE << mMatchValScale << IMGCMP_STR_TARGET_MEAN_VAL << mTargetMeanVal << IMGCMP_STR_TARGET_STDDEV_VAL << mTargetStddevVal << IMGCMP_STR_REPEAT_NUM << mRepeatNum << IMGCMP_STR_TRUE_SAMPLE_DIS_MEAN << mTrueSampleDisMean << IMGCMP_STR_TRUE_SAMPLE_DIS_STDDEV << mTrueSampleDisStddev << IMGCMP_STR_TRUE_SAMPLE_DIS_MIN << mTrueSampleDisMin << IMGCMP_STR_TRUE_SAMPLE_DIS_MAX << mTrueSampleDisMax << IMGCMP_STR_FALSE_SAMPLE_MIN_DIS << getFalseSampleMinDis() << IMGCMP_STR_DIS_THRE << mDisThre << IMGCMP_STR_TRAIN_DIS <> mAlignBaseImg; fs[IMGCMP_STR_COMPARE_BASE_IMAGE] >> mCompareBaseImg; fs[IMGCMP_STR_WEIGHT_MAT] >> mWeightMat; mMatchValScale = (double)fs[IMGCMP_STR_MATCH_VAL_SCALE]; FileNode fn = fs[IMGCMP_STR_TARGET_MEAN_VAL]; if (!fn.empty()) { mTargetMeanVal = (int)fn; } fn = fs[IMGCMP_STR_TARGET_STDDEV_VAL]; if (!fn.empty()) { mTargetStddevVal = (int)fn; } fn = fs[IMGCMP_STR_REPEAT_NUM]; if (!fn.empty()) { mRepeatNum = (int)fn; } else { //mRepeatNum = computeRepeatNum(); } fn = fs[IMGCMP_STR_NAME]; if (!fn.empty()) { mName = (string)fn; } fn = fs[IMGCMP_STR_TRUE_SAMPLE_DIS_MEAN]; if (!fn.empty()) { mTrueSampleDisMean = (double)fn; } fn = fs[IMGCMP_STR_TRUE_SAMPLE_DIS_STDDEV]; if (!fn.empty()) { mTrueSampleDisStddev = (double)fn; } fn = fs[IMGCMP_STR_TRUE_SAMPLE_DIS_MIN]; if (!fn.empty()) { mTrueSampleDisMin = (double)fn; } fn = fs[IMGCMP_STR_TRUE_SAMPLE_DIS_MAX]; if (!fn.empty()) { mTrueSampleDisMax = (double)fn; } fn = fs[IMGCMP_STR_FALSE_SAMPLE_MIN_DIS]; if (!fn.empty()) { setFalseSampleMinDis((double)fn); } fn = fs[IMGCMP_STR_DIS_THRE]; if (!fn.empty()) { mDisThre = (double)fn; } fn = fs[IMGCMP_STR_AVER_DIAMETER]; if (!fn.empty()) { meanDiameter = (int)fn; } setFilePath(filePath); genMask(); return true; } void ImageCompareModel::preProcessImage(Mat& img) const { preProcessImage(img, m8uMaskImg, mTargetMeanVal, mTargetStddevVal, mHighlightsThreshold); } void ImageCompareModel::preProcessImage(Mat& img, const Mat& mask, double tarMean, double tarStddev, int highlightsThreshold) const { if (img.channels() > 1) { img = getFirstChannel(img); } if (img.type() != CV_32FC1) { converToType(img, CV_32FC1); } Mat gaussImg; GaussianBlur(img, gaussImg, Size(3, 3), 5.0); img = gaussImg; Mat dilatedMask; dilate(mask, dilatedMask, Mat::ones(Size(3, 3), CV_32FC1)); Mat hightlightsMask = img < highlightsThreshold; Mat imgMask = hightlightsMask & dilatedMask; Scalar meanScalar, stddevScalar; meanStdDev(img, meanScalar, stddevScalar, imgMask); img = (img - meanScalar.val[0]) * tarStddev / stddevScalar.val[0] + tarMean; converToType(imgMask, CV_32FC1); imgMask /= 255.0; Mat imgNorm = cocentricNorm(img, Point2f(img.cols / 2.0, img.rows / 2.0), imgMask, 125); #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vImgNorm = imgNorm / 255.0; #endif img = imgNorm; } void ImageCompareModel::preProcessImage(Mat& img, const Mat& mask, const Mat& weightMat, double dstMean, double dstStddev, int highlightsThreshold) const { if (img.channels() > 1) { img = getFirstChannel(img); } if (img.type() != CV_32FC1) { converToType(img, CV_32FC1); } Mat gaussImg; GaussianBlur(img, gaussImg, Size(3, 3), 5.0); img = gaussImg; Mat hightlightsMask = img < highlightsThreshold; Mat imgMask = hightlightsMask & mask & (weightMat > 0); Scalar meanScalar, stddevScalar; meanStdDev(img, meanScalar, stddevScalar, imgMask); img = (img - meanScalar.val[0]) * dstMean / stddevScalar.val[0] + dstMean; converToType(hightlightsMask, CV_32FC1); Mat imgNorm = cocentricNorm(img, Point2f(img.cols / 2.0, img.rows / 2.0), weightMat.mul(hightlightsMask), 125); #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vImgNorm = imgNorm / 255.0; #endif img = imgNorm; } void ImageCompareModel::preProcessImage0(Mat& img) const { } double ImageCompareModel::compare(Mat img0, Mat img1, Mat* pRImg0, Mat* pRImg1) { if (img0.size() != img1.size() && img0.size() != mAlignBaseImg.size()) { return -1; } double bestMatchVal = DBL_MAX; int bestAngle = -1; Mat bestImg1; double bestDD = 0; double w = 0; Point2f center((float)img1.cols / 2.0, (float)img1.rows / 2.0); Mat rImg0 = rotateMatch(img0).mBestRImg; Mat rImg1 = rotateMatch(img1).mBestRImg; Mat dMat = rImg0 - rImg1; converToType(dMat, CV_32FC1); dMat = dMat.mul(mWeightMat); if (pRImg0) { *pRImg0 = rImg0; } if (pRImg1) { *pRImg1 = rImg1; } return genMatchValue(dMat); } double ImageCompareModel::compare(Mat img, Mat* pRImg /*= NULL*/, int levelNum /*= 1*/, bool isFilterSize /*= true*/, int flag, double md_diameter, double md_height) { if (mIsEnableCache) { auto cacheIter = mDisCache.find(img.data); if (cacheIter != mDisCache.end()) { return cacheIter->second; } } Mat rawImg = img.clone(); int nRadiusDiff = 15; if (mAlignBaseImg.size() != img.size()) { if (isFilterSize && abs(mAlignBaseImg.size().width- img.size().width) > nRadiusDiff) { return DBL_MAX; } resize(img, img, mAlignBaseImg.size()); } Mat matchImg = img.clone(); preProcessImage(matchImg); if (!mpMultiScaleModel) { initMultiScaleModel(); } RotateMatchResult rmr = rotateMatch(matchImg, levelNum); Mat rImg = rmr.mBestRImg; if (rImg.empty()) { return DBL_MAX; } #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vRImg = rImg / 255.0; #endif double bestAngle = rmr.mBestAngle; Mat cmpImg = rotateImage(img, Point2f(img.cols / 2.0, img.rows / 2.0), bestAngle); // remove highlights Mat hightlightsMask = cmpImg < mHighlightsThreshold; converToType(hightlightsMask, CV_32FC1); hightlightsMask /= 255.0; Mat unifiedMask = m32fMaskImg.mul(hightlightsMask).mul(mWeightMat); preProcessImage(cmpImg, m8uMaskImg, mWeightMat, mTargetMeanVal, mTargetStddevVal, mHighlightsThreshold); cmpImg.setTo(0, cmpImg < 0); converToType(cmpImg, CV_32FC1); normSectors_tarImg(cmpImg, unifiedMask, imgCen(cmpImg), 1, mCompareBaseImg); #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vRotatedImg0 = cmpImg / 255.0; #endif #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vRotatedImg1 = cmpImg / 255.0; #endif cmpImg.setTo(0, cmpImg < 10); if (pRImg) { *pRImg = cmpImg; } double ret = genMatchValue(cmpImg, mCompareBaseImg, unifiedMask, flag, rawImg.rows, md_diameter, md_height); /*if (flag == 1) { double matchedVal = ret; ret = penltyCoeff(matchedVal, meanDiameter, rawImg.rows); printf("the matched value %f", ret); }*/ if (mIsEnableCache) { mDisCache[img.data] = ret; if (mDisCache.size() > IMGCMP_CACHE_MAX_SIZE) { mDisCache.clear(); } } return ret; } void ImageCompareModel::train(const vector& imgVec) { #ifdef _DEBUG cout << getName() << endl; #endif _DEBUG if (0 == imgVec.size()) { return; } mAlignBaseImg = imgVec.front(); genMask(); preProcessImage(mAlignBaseImg); Mat sumMat = Mat::zeros(mAlignBaseImg.size(), CV_32FC1); Mat minMat(mAlignBaseImg.size(), mAlignBaseImg.type()); minMat.setTo(FLT_MAX); vector rImgVec; vector rmrVec; vector resizedImgVec; vector diametersVec; for (int i = 0; i < imgVec.size(); ++i) { diametersVec.push_back(imgVec[i].rows); Mat img = imgVec[i].clone(); if (img.size() != mAlignBaseImg.size()) { Mat resizedImg; resize(img, resizedImg, mAlignBaseImg.size()); img = resizedImg; } resizedImgVec.push_back(img); preProcessImage(img); RotateMatchResult rmr = rotateMatch(img); rmrVec.push_back(rmr); Mat rImg = rmr.mBestRImg; rImgVec.push_back(rImg); minMat = min(minMat, rImg); sumMat += rImg; #ifdef DEBUG_VIEW_INTERNAL_MAT Mat viewRImg = rImg / 255.0; Mat viewMinMat = minMat / 255.0; #endif } meanDiameter = mean(diametersVec)[0]; Mat avgMat = sumMat / imgVec.size(); mAlignBaseImg = avgMat; #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vAvgMat = avgMat / 255.0; #endif mWeightMat = minMat; mWeightMat /= 255.0; /*luffy_base::luffy_imageProc::*/meanvarnorm(mWeightMat, mWeightMat, mTargetMeanVal, 150, m8uMaskImg); mWeightMat.setTo(0, mWeightMat < 10); { if(mRepeatNum <= 0) { mRepeatNum = computeRepeatNum(); } selfRotateMin(mWeightMat, mWeightMat, mRepeatNum); } #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vWeightMat = mWeightMat / 255.0; Mat vBaseMat = mAlignBaseImg / 255.0; Mat vMinMat = minMat / 255.0; #endif sumMat.setTo(0); for (int i = 0; i < rmrVec.size(); ++i) { double bestAngle = rmrVec[i].mBestAngle; Mat img = resizedImgVec[i]; Mat rotatedImg = rotateImage(img, Point2f(img.cols/2.0, img.rows/2.0), bestAngle); preProcessImage(rotatedImg, m8uMaskImg, mWeightMat, mTargetMeanVal, mTargetStddevVal, mHighlightsThreshold); sumMat += rotatedImg; } mCompareBaseImg = sumMat / resizedImgVec.size(); #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vCompareBaseImg = mCompareBaseImg / 255.0; vWeightMat = mWeightMat / 255.0; #endif trueSampleWeightRecon(resizedImgVec, rmrVec); mWeightMat = falseReConMat.clone(); } void ImageCompareModel::trueSampleWeightRecon(const vector& resizedVec, vector rmrVec) { falseReConMat = mWeightMat.clone(); for (int i = 0; i < resizedVec.size(); i++) { const Mat &img = resizedVec[i]; double bestAngle = rmrVec[i].mBestAngle; Mat processedImg = img.clone(); preProcessImage(processedImg); Mat rotatedImg = rotateImage(processedImg, Point2f(img.cols / 2.0, img.rows / 2.0), bestAngle); Mat hightlightsMask = rotatedImg < mHighlightsThreshold; converToType(hightlightsMask, CV_32FC1); hightlightsMask /= 255.0; Mat unifiedMask = m32fMaskImg.mul(hightlightsMask).mul(mWeightMat); preProcessImage(rotatedImg, m8uMaskImg, mWeightMat, mTargetMeanVal, mTargetStddevVal, mHighlightsThreshold); rotatedImg.setTo(0, rotatedImg < 0); converToType(rotatedImg, CV_32FC1); normSectors_tarImg(rotatedImg, unifiedMask, imgCen(rotatedImg), 1, mCompareBaseImg); Mat vRotatedImg0 = rotatedImg / 255.0; #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vRotatedImg1 = rotatedImg / 255.0; #endif rotatedImg.setTo(0, rotatedImg < 10); trueWeightRecon(rotatedImg, mCompareBaseImg); } } void ImageCompareModel::trueWeightRecon(const Mat& rImg, const Mat& baseImg) { Mat dMat = abs(rImg - baseImg); //Mat norImg = Mat::ones(dMat.size(), dMat.type()) * 255; //dMat = max(dMat, 254); Mat mData; dMat.copyTo(mData, dMat > 1000); mData = minMaxNorm(mData, 0, 255); //mData.copyTo(mData, mData > 0.9); double minVal, maxVal; minMaxIdx(mData, &minVal, &maxVal); weightMapping(mData, maxVal); } void ImageCompareModel::weightMapping(const Mat& mData, double maxVal) { for (int i = 0; i < mData.rows; i++) { float* pData = (float*)mData.row(i).data; float*pWeight = (float*)falseReConMat.row(i).data; for (int j = 0; j < mData.cols; j++) { double pixVal = pData[j]; if (pixVal != 0) { double affineCoeff = descendFunction(pixVal, maxVal); pWeight[j] *= affineCoeff; } } } } double ImageCompareModel::descendFunction(double pixVal, double maxVal) { return (-1.0 / maxVal)*pixVal + 1.0; //return -1 * pow(pixVal, 2) + 1; } void ImageCompareModel::calculateAllParams(const vector& imgVec) { mDisThre = DBL_MAX; Mat disMat(1, imgVec.size(), CV_64FC1); for (int i = 0; i < imgVec.size(); ++i) { const Mat& img = imgVec[i]; double dis = compare(img, NULL, 1, false, 0); disMat.at(i) = dis; //disMat.copyTo(mDisMat); } if (disMat.cols == 1) { mTrueSampleDisStddev = 0; mTrueSampleDisMean = disMat.at(0); } else { Scalar m, s; meanStdDev(disMat, m, s); mTrueSampleDisStddev = s.val[0]; mTrueSampleDisMean = m.val[0]; } disMat.copyTo(mDisMat); minMaxIdx(disMat, &mTrueSampleDisMin, &mTrueSampleDisMax); } void ImageCompareModel::computeDisThre(const vector& imgVec, double* pMinDis, double md_diameter, double md_height) { if (imgVec.empty()) { return; } mDisThre = DBL_MAX; vector disVec; for_each(imgVec.begin(), imgVec.end(), [&](Mat img) { double dis = compare(img, NULL, 3, false, 1, md_diameter, md_height); disVec.push_back(dis); }); double minDis = *(min_element(disVec.begin(), disVec.end())); setFalseSampleMinDis(minDis); if (pMinDis) { *pMinDis = minDis; } if (minDis > mTrueSampleDisMax) { mDisThre = mTrueSampleDisMax*0.2 + minDis*0.8; //mDisThre = mTrueSampleDisMax*0.15 + minDis*0.85 + mTrueSampleDisStddev; //mDisThre = minDis; if (mDisThre > mTrueSampleDisMax*1.8) { mDisThre = mTrueSampleDisMax*1.8; } } else { mDisThre = DBL_MAX; std::cout << "max distance in training > min distance in testing" << std::endl; } } void ImageCompareModel::genUniformSepIdx(int num, int startIdx, int endIdx, vector& cenVec) { float step = (endIdx - startIdx) / (num); for (int i = 0; i < num; i++) { cenVec.push_back(floorf(step*i + startIdx)); } } void ImageCompareModel::genAngleRanges(vector cenVec, vector& rangeVec, int localRange) { rangeVec.clear(); for_each(cenVec.begin(), cenVec.end(), [&](int cen) { Range r; r.start = cen - localRange; r.end = cen + localRange + 1; rangeVec.push_back(r); }); } void ImageCompareModel::genCandidateAngleRanges(vector disVec, float angleStep, vector& rangeVec) { auto minDisIter = min_element(disVec.begin(), disVec.end()); int startIdx = minDisIter - disVec.begin(); rotate(disVec.begin(), minDisIter, disVec.end()); Mat disVecMat(1, disVec.size(), CV_32FC1, &(disVec[0])); Mat normDisVecMat; /*luffy_base::luffy_imageProc::*/meanvarnorm(disVecMat, normDisVecMat, 0, 1.0); set canNums; genCandidateRepeatNums(canNums); vector cenVec; int repeatNum = recognizeLowerExtremes(normDisVecMat, canNums, 5, &cenVec); #ifdef _TEST if (repeatNum <= 0) { std::cout << "repeatNum < 0" << std::endl; //waitKey(); } #endif if (repeatNum < 0) { rangeVec.clear(); return; cenVec.clear(); cenVec.push_back(disVec.size() / 2); genAngleRanges(cenVec, rangeVec, disVec.size() / 2); } else { if (cenVec.size() >= 2 && angleDis(cenVec.front(), cenVec.back()) < 2) { cenVec.pop_back(); } for_each(cenVec.begin(), cenVec.end(), [&](int& cen) { cen += startIdx; cen *= angleStep; cen = angleNorm(cen); }); for (auto i = cenVec.begin(); i != cenVec.end(); ++i) { if (*i < startIdx*angleStep) { rotate(cenVec.begin(), i, cenVec.end()); break; } } #ifdef _TEST for (int i = 0; i < cenVec.size() - 1; ++i) { float curVal = cenVec[i]; float nxtVal = cenVec[i + 1]; if (nxtVal <= curVal) { std::cout << "assert nxtVal <= curVal failed" << std::endl; waitKey(); } } #endif genAngleRanges(cenVec, rangeVec, 10); } } void ImageCompareModel::genCandidateAngleRanges(const Mat& disMat, float angleStep, vector& rangeVec, float rangeScale /*= 1*/) { for (int i = 0; i < disMat.rows; ++i) { Mat disVec = disMat.row(i); double minVal, maxVal; Point minPt, maxPt; minMaxLoc(disVec, &minVal, &maxVal, &minPt, &maxPt); Range &r = rangeVec[i]; float cenAngle = r.start + angleStep*minPt.x; float angleRangeWidth = r.end - r.start; r.start = floorf(cenAngle - angleRangeWidth * 0.5 * rangeScale); r.end = floorf(cenAngle + angleRangeWidth * 0.5 * rangeScale); } } void ImageCompareModel::selfRotateMin(const Mat& img, Mat& weightMat, int repeatNum) { Point2f cen(img.cols / 2.0, img.rows / 2.0); float angleStep = 360.0 / repeatNum; Mat dSumMat = Mat::ones(img.size(), img.type())*255; for (int i = 1; i < repeatNum; ++i) { Mat rotatedImg = rotateImage(img, cen, i*angleStep); Mat dilatedImg; dilate(rotatedImg, dilatedImg, Mat::ones(3, 3, CV_32FC1)); dSumMat = min(dSumMat, dilatedImg); } weightMat = dSumMat; } double ImageCompareModel::genMatchValue(const Mat& dMat) const { double v = norm(dMat, NORM_L2); double ret = scaleMatchValue(v); return ret; } double ImageCompareModel::genMatchValue(const Mat& rImg, const Mat& baseImg, const Mat& maskImg, int flag, int diameter, double md_diameter, double md_height) const { Mat dMat = rImg - baseImg; dMat = abs(dMat.mul(maskImg)); #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vRImg = rImg / 255.0; Mat vDMat0 = abs(dMat) / 255.0; Mat vBase = baseImg / 255.0; Mat vDMat = abs(dMat) / 255.0 / 255.0; Mat vMask = maskImg / 255.0; #endif Mat minmaxScaleMat = minMaxNorm(dMat, 0, 255, m8uMaskImg); converToType(minmaxScaleMat, CV_8UC1); Mat mMask = lowerMajorityMask(minmaxScaleMat, m8uMaskImg, 0.97); converToType(mMask, CV_32FC1); mMask /= 255.0; dMat = dMat.mul(mMask); #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vDMat1 = abs(dMat) / 255.0 / 255.0; #endif double s = sum(mMask.mul(maskImg)).val[0]; double ret = genMatchValue(dMat) / s; if (flag == 1) { double matchedVal = ret; ret = penltyCoeff(matchedVal, meanDiameter, diameter, md_diameter, md_height); printf("the matched value %f", ret); } if (ret > mDisThre) { return DBL_MAX; } else { return ret; } } double ImageCompareModel::scaleMatchValue(double val) const { return val*mMatchValScale; } ImageCompareModel* ImageCompareModel::scale(float s) { Mat baseImage; resize(mAlignBaseImg, baseImage, Size(), s, s, INTER_CUBIC); Mat weightMat; resize(mWeightMat, weightMat, Size(), s, s, INTER_CUBIC); ImageCompareModel* pRet = new ImageCompareModel; *pRet = *this; pRet->setBaseImg(baseImage); pRet->setWeightMat(weightMat); pRet->genMask(); // Mat mask8uImg; // resize(m8uMaskImg, mask8uImg, Size(), s, s, INTER_NEAREST); // Mat mask32fImg; // resize(m32fMaskImg, mask32fImg, Size(), s, s, INTER_NEAREST); // // pRet->set8uMaskImg(mask8uImg); // pRet->set32fMaskImg(mask32fImg); return pRet; } ImageCompareModel::RotateMatchResult ImageCompareModel::rotateMatch(const Mat& img, int levelNum /*= 1*/, float angleStep /*= 1.0*/, float startAngle /*= 0*/, float endAngle /*= 360*/) const { #ifdef _TEST static int idx = 0; std::cout << idx << " "; #endif RotateData* pData = new RotateData; vector angleRangeVec; if (levelNum > 1) { MultiScaleImage msi; msi.setLevelNum(3); msi.setBaseLevel(img); msi.genMultiScale(); ImageCompareData imgCmpData(&msi); imgCmpData.setStartAngle(startAngle); imgCmpData.setEndAngle(endAngle); mpMultiScaleModel->proc(&imgCmpData); *pData = *imgCmpData.getRotateData(); } else { rotateMatchData(img, pData, angleStep, startAngle, endAngle); } #ifdef _TEST if(1) { //if (idx == 15) { RotateData* pTestData = new RotateData; rotateMatchData(img, pTestData, angleStep, startAngle, endAngle); stringstream ss; ss << "multi_scale_comp_plot"; CvPlot::clear(ss.str()); CvPlot::plot(ss.str(), &(pTestData->mDisValVec[0]), pTestData->mDisValVec.size()); float baseAngle = pTestData->bestAngle(); float curAngle = pData->bestAngle(); std::cout << baseAngle << " " << curAngle << " " << baseAngle - curAngle << std::endl; delete pTestData; if (abs(angleDis(baseAngle, curAngle)) > 1) { waitKey(); } } idx++; } #endif RotateMatchResult rmr; if (pData->mDisValVec.empty()) { delete pData; return rmr; } else { size_t bestIndex = min_element(pData->mDisValVec.begin(), pData->mDisValVec.end()) - pData->mDisValVec.begin(); Mat ret = pData->mRImgVec[bestIndex]; rmr.mBestRImg = ret; rmr.mBestAngle = pData->bestAngle(); delete pData; return rmr; } } void ImageCompareModel::rotateMatchData(const Mat& _img, RotateData* pData, float angleStep /*= 1.0*/, float startAngle /*= 0*/, float endAngle /*= 360*/) const { Mat img = _img.clone(); Point2f center(img.cols / 2.0, img.rows / 2.0); int nNum = (endAngle - startAngle) / angleStep; RotateData& data = *pData; data.init(_img, center, angleStep, nNum); data.mStartAngle = startAngle; data.mEndAngle = endAngle; parallel_for_(Range(0, nNum), ImageCompareModelInvoker(this, pData)); } void ImageCompareModel::rotateMatchData(const Mat& img, RotateData* pData, float angleStep /*= 1.0*/, const vector& angleRangeVec) const { float minDis = FLT_MAX; RotateData bestData; for_each(angleRangeVec.begin(), angleRangeVec.end(), [&](const Range& r) { float startAngle = r.start; float endAngle = r.end; rotateMatchData(img, pData, angleStep, startAngle, endAngle); float localMinDis = *min_element(pData->mDisValVec.begin(), pData->mDisValVec.end()); if (localMinDis < minDis) { minDis = localMinDis; bestData = *pData; } }); *pData = bestData; } void ImageCompareModel::rotateMatchData(const Mat& img, RotateData* pData, const vector& angleRangeVec, float angleStep, Mat& disMat) const { if (angleRangeVec.empty()) { return; } disMat.create(angleRangeVec.size(), angleRangeVec.front().size(), CV_32FC1); pData->mRImgVec.resize(angleRangeVec.size()); pData->mDisValVec.resize(angleRangeVec.size(), FLT_MAX); double minDis = DBL_MAX; for (int i = 0; i < angleRangeVec.size(); ++i) { Range r = angleRangeVec[i]; Mat disVec = disMat.row(i); RotateData rotateData; float startAngle = r.start; float endAngle = r.end; rotateMatchData(img, &rotateData, angleStep, startAngle, endAngle); std::copy(rotateData.mDisValVec.begin(), rotateData.mDisValVec.end(), (float*)disVec.data); double localMinDis, localMaxDis; Point minPt, maxPt; minMaxLoc(disVec, &localMinDis, &localMaxDis, &minPt, &maxPt); if (localMinDis < minDis) { minDis = localMinDis; *pData = rotateData; } } } void ImageCompareModel::genCandidateRepeatNums(set& canNums) { canNums.clear(); for (int i = 4; i < 25; ++i) { canNums.insert(i); } } bool isValidExtremas(const Mat& vec, const vector& lessExtremaIdxVec, const vector& moreExtremaIdxVec) { float* pVec = (float*)vec.data; auto lessIdxIter = lessExtremaIdxVec.begin(); auto moreIdxIter = moreExtremaIdxVec.begin(); while (lessIdxIter != lessExtremaIdxVec.end()) { auto leftLessIdxIter = lessIdxIter; auto leftMoreIdxIter = moreIdxIter; auto rightLessIdxIter = lessIdxIter; rightLessIdxIter++; if (rightLessIdxIter == lessExtremaIdxVec.end()) { break; } auto rightMoreIdxIter = leftMoreIdxIter; while (*rightLessIdxIter > *rightMoreIdxIter) { rightMoreIdxIter++; } float leftLessExtrema = pVec[*leftLessIdxIter]; float rightLessExtrema = pVec[*rightLessIdxIter]; float localMaxVal = *max_element(pVec + *leftLessIdxIter, pVec + *rightLessIdxIter + 1); float valRange = localMaxVal - min(leftLessExtrema, rightLessExtrema); float valTor = valRange*0.05; float slope = (rightLessExtrema - leftLessExtrema) / (*rightLessIdxIter - *leftLessIdxIter); leftMoreIdxIter++; while (leftMoreIdxIter != rightMoreIdxIter) { float interVal = leftLessExtrema + slope*(*leftMoreIdxIter - *leftLessIdxIter); float curExtremaVal = pVec[*leftMoreIdxIter]; if (curExtremaVal < interVal + valTor) { return false; } leftMoreIdxIter++; } lessIdxIter++; moreIdxIter = rightMoreIdxIter; } return true; } int ImageCompareModel::recognizeLowerExtremes(const Mat& vec, set canNums, int extremeRefineRange /*= 5*/, vector *pExtremaIdxVec /*= 0*/) { unsigned int size = vec.cols; set validNs; float* pVec = (float*)vec.data; double vecMinVal, vecMaxVal; minMaxIdx(vec, &vecMinVal, &vecMaxVal); #ifdef DEBUG_VIEW_PLOT CvPlot::clear("recognizeLowerExtreams"); CvPlot::plot("recognizeLowerExtreams", (float*)vec.data, vec.cols); #endif map > repeatNumLocalExtremaIdxMap; map repeatNumLocalExtremaValMap; for (auto iter = canNums.rbegin(); iter != canNums.rend(); ++iter) { unsigned int n = *iter; Mat extremaValVec; vector extremaIdxVec; uniformLocalMinExtremas(vec, extremaValVec, extremaIdxVec, n, extremeRefineRange); float* pExtremaVecVec = (float*)extremaValVec.data; float curVal, nxtVal; bool isValid = true; Mat preLocalVec; for (unsigned int i = 0; i < extremaIdxVec.size() - 1; ++i) { unsigned int curIdx = extremaIdxVec[i]; unsigned int nxtIdx = extremaIdxVec[i + 1]; curVal = pVec[curIdx]; nxtVal = pVec[nxtIdx]; // no smaller value between two neighbor extremes if (anyInRange(pVec + curIdx + 1, pVec + nxtIdx, -FLT_MAX, min(curVal, nxtVal))) { isValid = false; break; } if (!preLocalVec.empty()) { Mat curLocalVec = vec.colRange(curIdx, nxtIdx + 1); if (curLocalVec.size() != preLocalVec.size()) { resize(curLocalVec, curLocalVec, preLocalVec.size()); } float sim = compareHist(curLocalVec, preLocalVec, CV_COMP_CORREL); if (sim < 0) { isValid = false; break; } } preLocalVec = vec.colRange(curIdx, nxtIdx + 1); } if (!isValid) { continue; } set nToBeErased; for_each(validNs.begin(), validNs.end(), [&](int preN) { if (preN % n != 0 || !isValid) { return; } vector preExtremaIdxVec = repeatNumLocalExtremaIdxMap[preN]; if (isValidExtremas(vec, extremaIdxVec, preExtremaIdxVec)) { nToBeErased.insert(preN); } else { isValid = false; } }); if (!isValid) { continue; } if (nToBeErased.size() == validNs.size()) { validNs.clear(); } else { for_each(nToBeErased.begin(), nToBeErased.end(), [&](int toBeErasedN) { validNs.erase(toBeErasedN); }); } validNs.insert(n); repeatNumLocalExtremaIdxMap[n] = extremaIdxVec; repeatNumLocalExtremaValMap[n] = extremaValVec; if (pExtremaIdxVec) { *pExtremaIdxVec = extremaIdxVec; } } if (validNs.size() == 0) { return -1; } if (validNs.size() > 1) { return -2; } return *(validNs.begin()); } int ImageCompareModel::recognizeRepeatedLocalExtremas(const Mat& vec, const set& canNums, int refineRange /*= 5*/, vector *pIdxVec /*= 0*/) { assert(vec.type() == CV_32FC1); float* pVecData = (float*)vec.data; for_each(canNums.begin(), canNums.end(), [&](int n) { Mat localMinExtremaValVec; vector localMinExtremaIdxVec; uniformLocalMinExtremas(vec, localMinExtremaValVec, localMinExtremaIdxVec, n, refineRange); float* pExtremaVecVec = (float*)localMinExtremaValVec.data; // get locally top 10% values }); return -1; } int ImageCompareModel::computeRepeatNum(const Mat& _img) { Mat img; cv::GaussianBlur(_img, img, Size(3, 3), 5.0); vector vec; selfRotationSimilarityFeature(img, vec, Point2f(img.cols / 2.0, img.rows / 2.0)); #ifdef DEBUG_VIEW_INTERNAL_MAT Mat viewImg = img / 255.0 /255.0; Mat view_Img = _img / 255.0 /255.0; #endif vector normVec(vec); Mat vecMat(1, vec.size(), CV_32FC1, &(vec[0])); Mat normVecMat(1, normVec.size(), CV_32FC1, &(normVec[0])); /*luffy_base::luffy_imageProc::*/meanvarnorm(vecMat, normVecMat, 0, 1.0); #ifdef DEBUG_VIEW_PLOT Mat dftVec; dft(normVec, dftVec); dftVec = abs(dftVec); Mat dVecMat = normVecMat.clone(); genSobelImage(dVecMat); Mat dDftVec; dft(dVecMat, dDftVec); Mat ddVecMat = dVecMat.clone(); genSobelImage(ddVecMat); Mat ddDftVec; dft(ddVecMat, ddDftVec); CvPlot::plot("normVec", (float*)&(normVec[0]), normVec.size()); CvPlot::plot("dVec", (float*)dVecMat.data, dVecMat.cols); CvPlot::plot("ddVec", (float*)ddVecMat.data, ddVecMat.cols); CvPlot::plot("ddft", (float*)dDftVec.data, 50); CvPlot::plot("dft", (float*)dftVec.data, 50); CvPlot::plot("dddft", (float*)ddDftVec.data, 50); waitKey(); #endif set canNums; genCandidateRepeatNums(canNums); return recognizeLowerExtremes(normVecMat, canNums); } int ImageCompareModel::computeRepeatNum() { if (mWeightMat.empty() || mAlignBaseImg.empty()) { return 0; } Mat img = mAlignBaseImg.mul(mWeightMat); #ifdef DEBUG_VIEW_INTERNAL_MAT Mat viewImg = img / 255.0 / 255.0; #endif return computeRepeatNum(img); } void ImageCompareModel::initMultiScaleModel() { if (mpMultiScaleModel) { delete mpMultiScaleModel; } mpMultiScaleModel = new MultiScaleImageCompareModel; mpMultiScaleModel->setLevelNum(3); mpMultiScaleModel->setBaseLevel(this); mpMultiScaleModel->genMultiScale(); } void ImageCompareModelInvoker::operator()(const cv::Range& range) const { int i0 = range.start; int i1 = range.end; //assert(abs(i1 - i0) == 1); m_pModel->parallelDetect(i0, m_pData); } void ImageCompareModel::parallelDetect(int index, void *p) const { RotateData *pData = (RotateData *)p; Mat t = getRotationMatrix2D(pData->mCenter, pData->angle(index), 1.0); Mat rImg; warpAffine(pData->mImgSrc, rImg, t, pData->mImgSrc.size()); if (rImg.size() != mAlignBaseImg.size()) { resize(rImg, rImg, mAlignBaseImg.size()); } Mat imgRes = rImg - mAlignBaseImg; #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vRImg, vBaseImg; vRImg = rImg / 255.0; vBaseImg = mAlignBaseImg / 255.0; Mat vImgRes = imgRes / 255.0; Mat vWeightMat = mWeightMat / 255.0; #endif imgRes = imgRes.mul(m32fMaskImg); if (!mWeightMat.empty()) { imgRes.mul(mWeightMat); } double val = norm(imgRes); #ifdef DEBUG_VIEW_INTERNAL_MAT vImgRes = imgRes / 255.0; #endif pData->mDisValVec[index] = val; pData->mRImgVec[index] = rImg; } double ImageCompareModel::filterTrainImage(const Mat&img) { if (img.empty()) { return DBL_MAX; } mDisThre = DBL_MAX; double dis = compare(img, NULL, 3, false); /*if (dis < mTrueSampleDisMax) { dis = mTrueSampleDisMax; } double absDiff = abs(dis - mTrueSampleDisMax);*/ return dis; } void ImageCompareModel::weightOptimization(const vector& falseSamples) { reConMat = mWeightMat.clone(); for (int i = 0; i < falseSamples.size(); i++) { Mat falseSample = falseSamples[i].clone(); preWeightReconstruction(falseSample, reConMat); } mWeightMat = reConMat; } void ImageCompareModel::preWeightReconstruction(Mat img, Mat &reConImg) { if (mAlignBaseImg.size() != img.size()) { resize(img, img, mAlignBaseImg.size()); } Mat matchImg = img.clone(); preProcessImage(matchImg); if (!mpMultiScaleModel) { initMultiScaleModel(); } RotateMatchResult rmr = rotateMatch(matchImg, 1); Mat rImg = rmr.mBestRImg; #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vRImg = rImg / 255.0; #endif double bestAngle = rmr.mBestAngle; Mat cmpImg = rotateImage(img, Point2f(img.cols / 2.0, img.rows / 2.0), bestAngle); // remove highlights Mat hightlightsMask = cmpImg < mHighlightsThreshold; converToType(hightlightsMask, CV_32FC1); hightlightsMask /= 255.0; Mat unifiedMask = m32fMaskImg.mul(hightlightsMask).mul(mWeightMat); preProcessImage(cmpImg, m8uMaskImg, mWeightMat, mTargetMeanVal, mTargetStddevVal, mHighlightsThreshold); cmpImg.setTo(0, cmpImg < 0); converToType(cmpImg, CV_32FC1); normSectors_tarImg(cmpImg, unifiedMask, imgCen(cmpImg), 1, mCompareBaseImg); #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vRotatedImg0 = cmpImg / 255.0; #endif #ifdef DEBUG_VIEW_INTERNAL_MAT Mat vRotatedImg1 = cmpImg / 255.0; #endif cmpImg.setTo(0, cmpImg < 10); weightReconstruction(cmpImg, mCompareBaseImg, unifiedMask, reConImg); } void ImageCompareModel::weightReconstruction(const Mat& rImg, const Mat& baseImg, const Mat& maskImg, Mat &reConImg) { Mat dMat = abs(rImg - baseImg); Mat norImg = Mat::ones(dMat.size(), dMat.type()) * 255; dMat = min(dMat, norImg); Mat w; dMat.copyTo(w, dMat < 0.5); w = minMaxNorm(w, 0, 1.0); for (int i = 0; i < reConImg.rows; i++) { float* pData = (float*)reConImg.row(i).data; float* pWeight = (float*)w.row(i).data; for (int j = 0; j < reConImg.cols; j++) { if (pWeight[j]!=0) { //pData[j] *= pWeight[j]; pWeight[j] = -1 * pow(pWeight[j] - 1, 2) + 1; pData[j] *= pWeight[j]; } } } } double ImageCompareModel::penltyCoeff(double matchedVal, int diameterMean, int targetDiameter, double modelDiamter, double modelHeight) const { int diff = abs(diameterMean - targetDiameter); double modelDiameterDiff = abs(modelDiamter - realWidth); double modelHeightDiff = abs(modelHeight - realHeight); if (diff >=10) { return matchedVal*(diff / 100.0 + 1) * (modelHeightDiff / 100.0 + 1) * (modelDiameterDiff / 100.0 + 1); } else { return matchedVal; } }