/*! * \file BlobInstance.h * \date 2019/08/30 * * \author Lin, Chi * Contact: lin.chi@hzleaper.com * * * \note */ #ifndef __BlobInstance_h_ #define __BlobInstance_h_ #include "CVUtils.h" class BlobDetector; /*! \brief Store geometric and optical properties for one single blob * * Most properties are lazy calculated, which means it's calculated on the first time its get function is called */ struct BlobInstance { BlobInstance(int _boundSize = 1) : boundSize(_boundSize) {} BlobInstance(const vector& _contour, int _boundSize = 1) : contour(_contour), boundSize(_boundSize) {} BlobInstance(vector&& _contour, int _boundSize = 1) : boundSize(_boundSize) { contour = std::move(_contour); } /*! Set hole contours, calculate properties such as area more precisely considering holes */ inline void setHoleContour(const vector >& holes) { holeContour = holes; } /** @overload */ inline void setHoleContour(vector >&& holes) { holeContour = std::move(holes); } /*! Set source image for optical properties calculation, and also generate local mask */ inline void setSourceImg(const Mat& img); /*! Set source soft mask for soft-thresholding */ inline void setSourceSoftMask(const Mat& softMask); /*! Update confidence by factor */ inline void updateConfidence(float factor) { confidence *= factor; } /*! Set transform matrix, if this blob is created inside ROI */ inline void setTransMat(const Mat& mat) { transMat = mat; // transform matrix updated, clean up cache contour_transed.clear(); holeContour_transed.clear(); rr_transed.size = Size(0, 0); } /*! Get source image */ inline const Mat& getSourceImg(); /*! Get source image in gray scale */ inline const Mat& getGrayImg(); /*! Get mask of contour and holes */ inline const Mat& getMask(); /*! Get bounded mask of contour and holes */ inline const Mat& getMaskBounded(); /*! Get bounded source image in gray scale */ inline const Mat& getSourceImgBounded(); /*! Get blob contour, transform matrix is applied if there's one */ inline const vector& getContour(); /*! Get blob contour in integer, transform matrix is applied if there's one */ inline vector getContour2i(); /*! Get blob's hole contours, transform matrix is applied if there's one */ inline const vector >& getHoleContour(); /*! Get confidence property */ inline float getConfidence() { return confidence; } /*! Get area, holes are excluded */ inline float getArea(); /*! Get perimeter of blob's contour */ inline float getPerimeter(); /*! Get width of blob's rotated bounding rect */ inline float getWidth(); /*! Get height of blob's rotated bounding rect */ inline float getHeight(); /*! Get angle of blob's rotated bounding rect */ inline float getAngle(); enum AngleMode { /*! Default, point to longer axis*/ Default = 0, /*! Always return 0 */ Ignore, /*! Always return -90 ~ 90 */ AlwaysUp }; /** @overload */ inline float getAngle(AngleMode mode); /** @overload */ inline float getAngle(int mode) { return getAngle(static_cast(mode)); } /*! Get circularity, closer to 1, much more likely it is a circle */ inline float getCircularity(); /*! Get convexity, closer to 1, much more likely it is a convex hull */ inline float getConvexity(); /*! Get inertia, closer to 1, much more likely it has the same length of minor and major axes, like a square */ inline float getInertia(); /*! Get center of gravity of blob */ inline Point2f getCenter(); /*! Get rotated bounding rect */ inline RotatedRect getBoundingRR(); /*! Get valley point of blob, which is the center of biggest flat field */ inline Point2f getValley(); /*! Get average luminance of blob */ inline float getLuminanceAverage(); /*! Get standard deviation luminance of blob */ inline float getLuminanceDeviation(); /*! Get variance luminance of blob */ inline float getLuminanceVariance(); /*! Get minimum luminance of blob */ inline float getLuminanceMin(); /*! Get maximum luminance of blob */ inline float getLuminanceMax(); /*! Get average of top n% brightest pixels in blob */ inline float getLuminanceTopNAvg(float topN); /*! Get average of top n% darkest pixels in blob */ inline float getLuminanceBottomNAvg(float bottomN); /*! Get majority luminance of blob */ inline float getLuminanceMajor(); /*! Get average contrast of blob */ inline float getContrastAverage(); /*! Get standard deviation contrast of blob */ inline float getContrastDeviation(); /*! Get variance contrast of blob */ inline float getContrastVariance(); /*! Get minimum contrast of blob */ inline float getContrastMin(); /*! Get maximum contrast of blob */ inline float getContrastMax(); /*! Get average of top n% pixels with biggest contrast in blob */ inline float getContrastTopNAvg(float topN); /*! Get average of top n% pixels with smallest contrast in blob */ inline float getContrastBottomNAvg(float bottomN); /*! Get majority contrast of blob */ inline float getContrastMajor(); /*! Get average color of blob, for greyscale image, it's the same as luminance */ inline Scalar getColorAverage(); /*! Get standard deviation color of blob, for greyscale image, it's the same as luminance */ inline Scalar getColorDeviation(); /*! Get variance color of blob, for greyscale image, it's the same as luminance */ inline Scalar getColorVariance(); /*! Get sharpness of blob, for color image, it's computed in greyscale */ inline float getSharpness(); protected: inline void calcMoms(); inline float getContourArea(); inline const Rect& getBoundingRect(); inline bool hasProps(int val); inline void setProps(int val, bool on = true); inline void applyTrans(const vector& pnts, vector& transed); inline Point2f applyTrans(const Point2f& p); inline bool hasImage() { return !sourceImg.empty(); } inline const Mat& getContrastImg(); inline std::vector& getGrayPixels(); inline std::vector& getContrastPixels(); protected: vector contour; vector > holeContour; // no hole contour if detector told us to float confidence = 1.; int boundSize; // bounded size Mat sourceImg; // source image for advanced properties calculation Mat sourceImgBounded; // n pixel bounded, shared memory with sourceImg Mat sourceSoftMask; // source soft mask Mat sourceSoftMaskBounded; // n pixel bounded, shared memory with sourceSoftMask Mat sourceMask; // source mask for advanced properties calculation Mat sourceMaskBounded; // n pixel bounded, shared memory with sourceMask Mat sourceImgGray; // optional Mat sourceImgContrast; // optional std::vector grayPixels; // optional std::vector contrastPixels; // optional Mat transMat; // matrix for perform inverted transform in roi vector contour_transed; vector > holeContour_transed; RotatedRect rr_transed; enum BlobProp { Reserved = 0, Area = 0x00000001, // pixel-count-area AreaContour = 0x00000002, // green-formula-area of outer contour Perimeter = 0x00000004, RR = 0x00000008, // rotated bounding rect, width, height, angle Circularity = 0x00000010, Convexity = 0x00000020, Mom = 0x00000040, // inertia, center BR = 0x00000080, // bounding rect Valley = 0x00000100, LuminanceMinMax = 0x00000200, LuminanceAvgDev = 0x00000400, ContrastMinMax = 0x00000800, ContrastAvgDev = 0x00001000, Color = 0x00004000, Shapness = 0x00008000, LuminanceMajor = 0x00010000, ContrastMajor = 0x00020000, }; int props = BlobProp::Reserved; // at most, 32 kinds of properties float area; float perimeter; RotatedRect rr; float circularity; float convexity; float inertia; Point2f center; float cArea; Rect r; Point2f valley; float lumMin; float lumMax; float lumAvg; float lumDev; float lumMajor; float contrastMin; float contrastMax; float contrastAvg; float contrastDev; float contrastMajor; float sharp; Scalar colorAvg; Scalar colorDev; }; float BlobInstance::getArea() { if (!hasProps(BlobProp::Area)) { area = sum(getMask())[0] / 255; setProps(BlobProp::Area); } return area; } float BlobInstance::getPerimeter() { if (!hasProps(BlobProp::Perimeter)) { perimeter = arcLength(contour, true); setProps(BlobProp::Perimeter); } return perimeter; } float BlobInstance::getWidth() { return getBoundingRR().size.width; } float BlobInstance::getHeight() { return getBoundingRR().size.height; } float BlobInstance::getAngle() { return getBoundingRR().angle; } float BlobInstance::getAngle(AngleMode mode) { if (mode == AngleMode::Ignore) return 0; float a = getAngle(); if (mode == AngleMode::AlwaysUp) { if (a < -45) return a; // -90 ~ -45 else if (a > 45) return a - 180; // 45 ~ 90 else return a - 90; // -45 ~ 45 } return a; // default, point to longer axis } float BlobInstance::getCircularity() { if (!hasProps(BlobProp::Circularity)) { circularity = 4 * CV_PI * getContourArea() / (getPerimeter() * getPerimeter()); setProps(BlobProp::Circularity); } return circularity; } float BlobInstance::getConvexity() { if (!hasProps(BlobProp::Convexity)) { convexity = getContourArea() / getConvexHullArea(contour); setProps(BlobProp::Convexity); } return convexity; } float BlobInstance::getInertia() { calcMoms(); return inertia; } Point2f BlobInstance::getCenter() { calcMoms(); return applyTrans(center); } RotatedRect BlobInstance::getBoundingRR() { if (!hasProps(BlobProp::RR)) { rr = minAreaRect2(contour); setProps(BlobProp::RR); } if (rr_transed.size.empty()) { rr_transed = rr; if (!transMat.empty()) { transRotateRect23(rr_transed, (double*)transMat.data); } } return rr_transed; } const Rect& BlobInstance::getBoundingRect() { if (!hasProps(BlobProp::BR)) { r = boundingRect(contour); setProps(BlobProp::BR); } return r; } bool BlobInstance::hasProps(int val) { return (props & val) != 0; } void BlobInstance::setProps(int val, bool on) { on ? (props |= val) : (props &= (~val)); } void BlobInstance::applyTrans(const vector& pnts, vector& transed) { bool doTrans = !transMat.empty(); double* p_mat = (double*)transMat.data; int pcount = pnts.size(); transed.clear(); transed.reserve(pcount); for (int i = 0; i < pcount; ++i) { Point2f p = pnts[i]; if (doTrans) { transPoint23(p.x, p.y, p_mat); } transed.push_back(std::move(p)); } } Point2f BlobInstance::applyTrans(const Point2f& p) { if (transMat.empty()) return p; Point2f p2 = p; transPoint23(p2.x, p2.y, (double*)(transMat.data)); return p2; } const Mat& BlobInstance::getSourceImg() { return sourceImg; } const Mat& BlobInstance::getGrayImg() { if (sourceImgGray.empty()) { ensureGrayImg(sourceImg, sourceImgGray); } return sourceImgGray; } const Mat& BlobInstance::getContrastImg() { if (sourceImgContrast.empty()) { gradiant(getGrayImg(), sourceImgContrast, 3, 3); } return sourceImgContrast; } const Mat& BlobInstance::getMask() { if (sourceMask.empty()) { // generate local mask sourceMaskBounded = Mat::zeros(sourceImg.rows + boundSize * 2, sourceImg.cols + boundSize * 2, CV_8U); sourceMask = Mat(sourceMaskBounded, Rect(boundSize, boundSize, sourceImg.cols, sourceImg.rows)); createMaskByContour(sourceMask, contour, holeContour, getBoundingRect().tl(), true); if (!sourceSoftMask.empty()) { sourceMask &= sourceSoftMask; } } return sourceMask; } const Mat& BlobInstance::getMaskBounded() { if (sourceMaskBounded.empty()) { getMask(); // delegate to generate mask } return sourceMaskBounded; } const Mat& BlobInstance::getSourceImgBounded() { return sourceImgBounded; } std::vector& BlobInstance::getGrayPixels() { if (grayPixels.empty()) { grayPixels = cvtMat2Vec(getGrayImg(), getMask()); } return grayPixels; } std::vector& BlobInstance::getContrastPixels() { if (contrastPixels.empty()) { contrastPixels = cvtMat2Vec(getContrastImg(), getMask()); } return contrastPixels; } Point2f BlobInstance::getValley() { if (!hasProps(BlobProp::Valley)) { Point tl(getBoundingRect().x - boundSize, getBoundingRect().y - boundSize); valley = detectValley(getMaskBounded()); valley.x += (getBoundingRect().x - boundSize); // valley position is detected on bounded mask. valley.y += (getBoundingRect().y - boundSize); setProps(BlobProp::Valley); } return applyTrans(valley); } float BlobInstance::getLuminanceAverage() { if (!hasProps(BlobProp::LuminanceAvgDev)) { if (!hasImage()) return 0; getColorAverage(); // delegate to average color calculation if (sourceImg.channels() == 1) { lumAvg = colorAvg[0]; lumDev = colorDev[0]; } else { lumAvg = (colorAvg[0] + colorAvg[1] + colorAvg[2]) / 3; lumDev = (colorDev[0] + colorDev[1] + colorDev[2]) / 3; } setProps(BlobProp::LuminanceAvgDev); } return lumAvg; } float BlobInstance::getLuminanceDeviation() { if (!hasProps(BlobProp::LuminanceAvgDev)) { getLuminanceAverage(); // delegate to average luminance calculation } return lumDev; } float BlobInstance::getLuminanceVariance() { float v = getLuminanceDeviation(); return v * v; } float BlobInstance::getLuminanceMin() { if (!hasProps(BlobProp::LuminanceMinMax)) { if (!hasImage()) return 0; double minVal, maxVal; minMaxLoc(getGrayImg(), &minVal, &maxVal, nullptr, nullptr, getMask()); lumMin = minVal; lumMax = maxVal; setProps(BlobProp::LuminanceMinMax); } return lumMin; } float BlobInstance::getLuminanceMax() { if (!hasProps(BlobProp::LuminanceMinMax)) { getLuminanceMin(); // delegate to minimum luminance calculation } return lumMax; } float BlobInstance::getLuminanceTopNAvg(float topN) { vector& grayPixels = getGrayPixels(); int topNCount = grayPixels.size() * topN / 100; partial_sort(grayPixels.begin(), grayPixels.begin() + topNCount, grayPixels.end(), [](const uchar& v1, const uchar& v2) { return v1 > v2; }); Mat topNMat(1, topNCount, CV_8U, grayPixels.data()); return mean(topNMat)[0]; } float BlobInstance::getLuminanceBottomNAvg(float bottomN) { vector& grayPixels = getGrayPixels(); int bottomNCount = grayPixels.size() * bottomN / 100; partial_sort(grayPixels.begin(), grayPixels.begin() + bottomNCount, grayPixels.end(), [](const uchar& v1, const uchar& v2) { return v1 < v2; }); Mat bottomNMat(1, bottomN, CV_8U, grayPixels.data()); return mean(bottomNMat)[0]; } float BlobInstance::getLuminanceMajor() { if (!hasProps(BlobProp::LuminanceMajor)) { if (!hasImage()) return 0; lumMajor = getMajor(getGrayImg(), getMask()); setProps(BlobProp::LuminanceMajor); } return lumMajor; } float BlobInstance::getContrastAverage() { if (!hasProps(BlobProp::ContrastAvgDev)) { if (!hasImage()) return 0; Scalar meanVal, devVal; meanStdDev(getContrastImg(), meanVal, devVal, getMask()); contrastAvg = meanVal[0]; contrastDev = devVal[0]; setProps(BlobProp::ContrastAvgDev); } return contrastAvg; } float BlobInstance::getContrastDeviation() { if (!hasProps(BlobProp::ContrastAvgDev)) { getContrastAverage(); // delegate to average contrast calculation } return contrastDev; } float BlobInstance::getContrastVariance() { float v = getContrastDeviation(); return v * v; } float BlobInstance::getContrastMin() { if (!hasProps(BlobProp::ContrastMinMax)) { if (!hasImage()) return 0; double minVal, maxVal; minMaxLoc(getContrastImg(), &minVal, &maxVal, nullptr, nullptr, getMask()); contrastMin = minVal; contrastMax = maxVal; setProps(BlobProp::ContrastMinMax); } return contrastMin; } float BlobInstance::getContrastMax() { if (!hasProps(BlobProp::ContrastMinMax)) { getContrastMin(); // delegate to minimum contrast calculation } return contrastMax; } float BlobInstance::getContrastTopNAvg(float topN) { vector& contrastPixels = getContrastPixels(); int topNCount = contrastPixels.size() * topN / 100; partial_sort(contrastPixels.begin(), contrastPixels.begin() + topNCount, contrastPixels.end(), [](const uchar& v1, const uchar& v2) { return v1 > v2; }); Mat topNMat(1, topNCount, CV_8U, contrastPixels.data()); return mean(topNMat)[0]; } float BlobInstance::getContrastBottomNAvg(float bottomN) { vector& contrastPixels = getContrastPixels(); int bottomNCount = contrastPixels.size() * bottomN / 100; partial_sort(contrastPixels.begin(), contrastPixels.begin() + bottomNCount, contrastPixels.end(), [](const uchar& v1, const uchar& v2) { return v1 < v2; }); Mat bottomNMat(1, bottomN, CV_8U, contrastPixels.data()); return mean(bottomNMat)[0]; } float BlobInstance::getContrastMajor() { if (!hasProps(BlobProp::ContrastMajor)) { if (!hasImage()) return 0; contrastMajor = getMajor(getContrastImg(), getMask()); setProps(BlobProp::ContrastMajor); } return contrastMajor; } float BlobInstance::getSharpness() { if (!hasProps(BlobProp::Shapness)) { if (!hasImage()) return 0; sharp = sharpness(getGrayImg(), &getMask(), nullptr); setProps(BlobProp::Shapness); } return sharp; } Scalar BlobInstance::getColorAverage() { if (!hasProps(BlobProp::Color)) { if (!hasImage()) return Scalar(); meanStdDev(sourceImg, colorAvg, colorDev, getMask()); setProps(BlobProp::Color); } return colorAvg; } Scalar BlobInstance::getColorDeviation() { if (!hasProps(BlobProp::Color)) { getColorAverage(); // delegate to average color calculation } return colorDev; } Scalar BlobInstance::getColorVariance() { Scalar v = getColorDeviation(); return Scalar(v[0] * v[0], v[1] * v[1], v[2] * v[2]); } void BlobInstance::calcMoms() { if (!hasProps(BlobProp::Mom)) { Moments moms = moments(contour); inertia = GeomUtils::getInertia(moms); if (!sourceSoftMask.empty()) { Mat m = getMask(); center = findMaskCenter(m); Rect r = getBoundingRect(); center.x += r.x; center.y += r.y; } else if (moms.m00 == 0) center = getBoundingRR().center; else center = Point2f(moms.m10 / moms.m00, moms.m01 / moms.m00); setProps(BlobProp::Mom); } } float BlobInstance::getContourArea() { if (!hasProps(BlobProp::AreaContour)) { cArea = contourArea(contour); setProps(BlobProp::AreaContour); } return cArea; } void BlobInstance::setSourceImg(const Mat& img) { const Rect& r = getBoundingRect(); Rect rBounded = scaleRect(r, boundSize); if (rBounded.x >= 0 && rBounded.y >= 0 && rBounded.x + rBounded.width < img.cols && rBounded.y + rBounded.height < img.rows) { sourceImgBounded = Mat(img, rBounded); sourceImg = Mat(img, r); } else { sourceImgBounded = Mat::zeros(rBounded.size(), CV_8U); copyMakeBorder(Mat(img, r), sourceImgBounded, boundSize, boundSize, boundSize, boundSize, BORDER_REPLICATE); sourceImg = Mat(sourceImgBounded, Rect(boundSize, boundSize, r.width, r.height)); } } void BlobInstance::setSourceSoftMask(const Mat& softMask) { sourceSoftMask = Mat(softMask, getBoundingRect()); const Rect& r = getBoundingRect(); Rect rBounded = scaleRect(r, boundSize); if (rBounded.x >= 0 && rBounded.y >= 0 && rBounded.x + rBounded.width < softMask.cols && rBounded.y + rBounded.height < softMask.rows) { sourceSoftMaskBounded = Mat(softMask, rBounded); sourceSoftMask = Mat(softMask, r); } else { sourceSoftMaskBounded = Mat::zeros(rBounded.size(), CV_8U); sourceSoftMask = Mat(sourceSoftMaskBounded, Rect(boundSize, boundSize, r.width, r.height)); Mat(softMask, r).copyTo(sourceSoftMask); } } const vector& BlobInstance::getContour() { if (!contour_transed.empty()) { return contour_transed; } applyTrans(contour, contour_transed); return contour_transed; } vector BlobInstance::getContour2i() { if (transMat.empty()) { return contour; } else { const vector& ret = getContour(); return saturate_cast_points(ret); } } const vector >& BlobInstance::getHoleContour() { if (!holeContour_transed.empty()) { return holeContour_transed; } int holeCount = holeContour.size(); for (int i = 0; i < holeCount; ++i) { vector hole2; applyTrans(holeContour[i], hole2); holeContour_transed.push_back(std::move(hole2)); } return holeContour_transed; } #endif // BlobInstance_h_