Visual Servoing Platform version 3.5.0
vpFernClassifier.cpp
1/****************************************************************************
2 *
3 * ViSP, open source Visual Servoing Platform software.
4 * Copyright (C) 2005 - 2019 by Inria. All rights reserved.
5 *
6 * This software is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 * See the file LICENSE.txt at the root directory of this source
11 * distribution for additional information about the GNU GPL.
12 *
13 * For using ViSP with software that can not be combined with the GNU
14 * GPL, please contact Inria about acquiring a ViSP Professional
15 * Edition License.
16 *
17 * See http://visp.inria.fr for more information.
18 *
19 * This software was developed at:
20 * Inria Rennes - Bretagne Atlantique
21 * Campus Universitaire de Beaulieu
22 * 35042 Rennes Cedex
23 * France
24 *
25 * If you have questions regarding the use of this file, please contact
26 * Inria at visp@inria.fr
27 *
28 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
29 * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
30 *
31 * Description:
32 * Class that implements the Fern classifier and the YAPE detector thanks
33 * to the OpenCV library.
34 *
35 * Authors:
36 * Romain Tallonneau
37 *
38 *****************************************************************************/
39
40#include <visp3/core/vpConfig.h>
41
42#if (VISP_HAVE_OPENCV_VERSION >= 0x020000) && \
43 (VISP_HAVE_OPENCV_VERSION < 0x030000) // Require opencv >= 2.0.0 and < 3.0.0
44
45#include <visp3/core/vpColor.h>
46#include <visp3/core/vpDisplay.h>
47#include <visp3/core/vpImageConvert.h>
48#include <visp3/core/vpImageTools.h>
49#include <visp3/vision/vpFernClassifier.h>
50
55vpFernClassifier::vpFernClassifier()
56 : vpBasicKeyPoint(), ldetector(), fernClassifier(),
57 gen(0, 256, 5, true, 0.6, 1.5, -CV_PI / 2, CV_PI / 2, -CV_PI / 2, CV_PI / 2), hasLearn(false), threshold(20),
58 nbView(2000), dist(2), nbClassfier(100), ClassifierSize(11), nbOctave(2), patchSize(32), radius(7), nbPoints(200),
59 blurImage(true), radiusBlur(7), sigmaBlur(1), nbMinPoint(10),
60#if (VISP_HAVE_OPENCV_VERSION >= 0x020408)
61 curImg(),
62#else
63 curImg(NULL),
64#endif
65 objKeypoints(), modelROI_Ref(), modelROI(), modelPoints(), imgKeypoints(), refPt(), curPt()
66{
67}
68
80vpFernClassifier::vpFernClassifier(const std::string &_dataFile, const std::string &_objectName)
81 : vpBasicKeyPoint(), ldetector(), fernClassifier(),
82 gen(0, 256, 5, true, 0.6, 1.5, -CV_PI / 2, CV_PI / 2, -CV_PI / 2, CV_PI / 2), hasLearn(false), threshold(20),
83 nbView(2000), dist(2), nbClassfier(100), ClassifierSize(11), nbOctave(2), patchSize(32), radius(7), nbPoints(200),
84 blurImage(true), radiusBlur(7), sigmaBlur(1), nbMinPoint(10),
85#if (VISP_HAVE_OPENCV_VERSION >= 0x020408)
86 curImg(),
87#else
88 curImg(NULL),
89#endif
90 objKeypoints(), modelROI_Ref(), modelROI(), modelPoints(), imgKeypoints(), refPt(), curPt()
91{
92 this->load(_dataFile, _objectName);
93}
94
99vpFernClassifier::~vpFernClassifier()
100{
101#if (VISP_HAVE_OPENCV_VERSION < 0x020408)
102 if (curImg != NULL) {
103 if (curImg->width % 8 == 0) {
104 curImg->imageData = NULL;
105 cvReleaseImageHeader(&curImg);
106 } else {
107 cvReleaseImage(&curImg);
108 }
109 curImg = NULL;
110 }
111#endif
112}
113
120void vpFernClassifier::init()
121{
122 hasLearn = false;
123 nbClassfier = 100;
124 ClassifierSize = 11;
125 nbPoints = 200;
126#if (VISP_HAVE_OPENCV_VERSION < 0x020408)
127 curImg = NULL;
128#endif
129 blurImage = true;
130 radiusBlur = 7;
131 sigmaBlur = 1;
132 patchSize = 32;
133 radius = 7;
134 threshold = 20;
135 nbOctave = 2;
136 nbView = 2000;
137 dist = 2;
138 nbMinPoint = 10;
139}
140
144void vpFernClassifier::train()
145{
146 // initialise detector
147 cv::LDetector d(radius, threshold, nbOctave, nbView, patchSize, dist);
148
149 // blur
150 cv::Mat obj = (cv::Mat)curImg;
151
152 if (this->getBlurSetting()) {
153 cv::GaussianBlur(obj, obj, cv::Size(getBlurSize(), getBlurSize()), getBlurSigma(), getBlurSigma());
154 }
155
156 // build pyramid
157 std::vector<cv::Mat> objpyr;
158 cv::buildPyramid(obj, objpyr, d.nOctaves - 1);
159
160 // getPoints
161 d.getMostStable2D(obj, objKeypoints, 100, gen);
162
163 ldetector = d;
164
165 // train classifier
166 modelROI = cv::Rect(0, 0, objpyr[0].cols, objpyr[0].rows);
167 ldetector.getMostStable2D(objpyr[0], modelPoints, 100, gen);
168
169 fernClassifier.trainFromSingleView(objpyr[0], modelPoints, patchSize, (int)modelPoints.size(), 100, 11, 10000,
170 cv::FernClassifier::COMPRESSION_NONE, gen);
171
172 /* from OpenCV format to ViSP format */
173 referenceImagePointsList.resize(0);
174 for (unsigned int i = 0; i < modelPoints.size(); i += 1) {
175 vpImagePoint ip(modelPoints[i].pt.y + modelROI_Ref.y, modelPoints[i].pt.x + modelROI_Ref.x);
176 referenceImagePointsList.push_back(ip);
177 }
178
179 // set flag
180 hasLearn = true;
181}
182
196unsigned int vpFernClassifier::buildReference(const vpImage<unsigned char> &_I)
197{
198 this->setImage(_I);
199
200 train();
201
202 _reference_computed = true;
203 return (unsigned int)objKeypoints.size();
204}
205
224unsigned int vpFernClassifier::buildReference(const vpImage<unsigned char> &_I, const vpImagePoint &_iP,
225 const unsigned int _height, const unsigned int _width)
226{
227 if ((_iP.get_i() + _height) >= _I.getHeight() || (_iP.get_j() + _width) >= _I.getWidth()) {
228 vpTRACE("Bad size for the subimage");
229 throw(vpException(vpImageException::notInTheImage, "Bad size for the subimage"));
230 }
231
232 vpImage<unsigned char> subImage;
233 vpImageTools::crop(_I, (unsigned int)_iP.get_i(), (unsigned int)_iP.get_j(), _height, _width, subImage);
234 this->setImage(subImage);
235
236 /* initialise a structure containing the region of interest used in the
237 reference image */
238 modelROI_Ref.x = (int)_iP.get_u();
239 modelROI_Ref.y = (int)_iP.get_v();
240 modelROI_Ref.width = (int)_width;
241 modelROI_Ref.height = (int)_height;
242
243 train();
244
245 return (unsigned int)objKeypoints.size();
246}
247
264unsigned int vpFernClassifier::buildReference(const vpImage<unsigned char> &_I, const vpRect &_rectangle)
265{
266 vpImagePoint iP;
267 iP.set_i(_rectangle.getTop());
268 iP.set_j(_rectangle.getLeft());
269 return (this->buildReference(_I, iP, (unsigned int)_rectangle.getHeight(), (unsigned int)_rectangle.getWidth()));
270}
271
284unsigned int vpFernClassifier::matchPoint(const vpImage<unsigned char> &_I)
285{
286 if (!hasLearn) {
287 vpERROR_TRACE("The object has not been learned. ");
288 throw vpException(vpException::notInitialized, "object is not learned, load database or build the reference ");
289 }
290
291 setImage(_I);
292 // resize image
293 // cv::resize(_image, image, Size(), 1./imgscale, 1./imgscale,
294 // INTER_CUBIC);
295 cv::Mat img = this->curImg;
296
297 if (this->getBlurSetting()) {
298 cv::GaussianBlur(img, img, cv::Size(this->getBlurSize(), this->getBlurSize()), this->getBlurSigma(),
299 this->getBlurSigma());
300 }
301
302 std::vector<cv::Mat> imgPyr;
303 cv::buildPyramid(img, imgPyr, ldetector.nOctaves - 1);
304
305 ldetector(imgPyr, imgKeypoints, 500);
306
307 unsigned int m = (unsigned int)modelPoints.size();
308 unsigned int n = (unsigned int)imgKeypoints.size();
309 std::vector<int> bestMatches(m, -1);
310 std::vector<float> maxLogProb(m, -FLT_MAX);
311 std::vector<float> signature;
312 unsigned int totalMatch = 0;
313
314 /* part of code from OpenCV planarObjectDetector */
315 currentImagePointsList.resize(0);
316 matchedReferencePoints.resize(0);
317
318 for (unsigned int i = 0; i < n; i++) {
319 cv::KeyPoint kpt = imgKeypoints[i];
320 kpt.pt.x /= (float)(1 << kpt.octave);
321 kpt.pt.y /= (float)(1 << kpt.octave);
322 int k = fernClassifier(imgPyr[(unsigned int)kpt.octave], kpt.pt, signature);
323 if (k >= 0 && (bestMatches[(unsigned int)k] < 0 || signature[(unsigned int)k] > maxLogProb[(unsigned int)k])) {
324 maxLogProb[(unsigned int)k] = signature[(unsigned int)k];
325 bestMatches[(unsigned int)k] = (int)i;
326 totalMatch++;
327
328 vpImagePoint ip_cur(imgKeypoints[i].pt.y, imgKeypoints[i].pt.x);
329
330 currentImagePointsList.push_back(ip_cur);
331 matchedReferencePoints.push_back((unsigned int)k);
332 }
333 }
334
335 refPt.resize(0);
336 curPt.resize(0);
337 for (unsigned int i = 0; i < m; i++) {
338 if (bestMatches[i] >= 0) {
339 refPt.push_back(modelPoints[i].pt);
340 curPt.push_back(imgKeypoints[(unsigned int)bestMatches[i]].pt);
341 }
342 }
343
344 return totalMatch;
345}
346
360unsigned int vpFernClassifier::matchPoint(const vpImage<unsigned char> &_I, const vpImagePoint &_iP,
361 const unsigned int _height, const unsigned int _width)
362{
363 if ((_iP.get_i() + _height) >= _I.getHeight() || (_iP.get_j() + _width) >= _I.getWidth()) {
364 vpTRACE("Bad size for the subimage");
365 throw(vpException(vpImageException::notInTheImage, "Bad size for the subimage"));
366 }
367
368 vpImage<unsigned char> subImage;
369
370 vpImageTools::crop(_I, (unsigned int)_iP.get_i(), (unsigned int)_iP.get_j(), _height, _width, subImage);
371
372 return this->matchPoint(subImage);
373}
374
386unsigned int vpFernClassifier::matchPoint(const vpImage<unsigned char> &_I, const vpRect &_rectangle)
387{
388 vpImagePoint iP;
389 iP.set_i(_rectangle.getTop());
390 iP.set_j(_rectangle.getLeft());
391 return (this->matchPoint(_I, iP, (unsigned int)_rectangle.getHeight(), (unsigned int)_rectangle.getWidth()));
392}
393
411void vpFernClassifier::display(const vpImage<unsigned char> &_Iref, const vpImage<unsigned char> &_Icurrent,
412 unsigned int size)
413{
414 for (unsigned int i = 0; i < matchedReferencePoints.size(); i++) {
415 vpDisplay::displayCross(_Iref, referenceImagePointsList[matchedReferencePoints[i]], size, vpColor::red);
416 vpDisplay::displayCross(_Icurrent, currentImagePointsList[i], size, vpColor::green);
417 }
418}
419
431void vpFernClassifier::display(const vpImage<unsigned char> &_Icurrent, unsigned int size, const vpColor &color)
432{
433 for (unsigned int i = 0; i < matchedReferencePoints.size(); i++) {
434 vpDisplay::displayCross(_Icurrent, currentImagePointsList[i], size, color);
435 }
436}
437
438/* IO METHODS */
439
449void vpFernClassifier::load(const std::string &_dataFile, const std::string & /*_objectName*/)
450{
451 std::cout << " > Load data for the planar object detector..." << std::endl;
452
453 /* part of code from OpenCV planarObjectDetector */
454 cv::FileStorage fs(_dataFile, cv::FileStorage::READ);
455 cv::FileNode node = fs.getFirstTopLevelNode();
456
457 cv::FileNodeIterator it = node["model-roi"].begin(), it_end;
458 it >> modelROI.x >> modelROI.y >> modelROI.width >> modelROI.height;
459
460 ldetector.read(node["detector"]);
461 fernClassifier.read(node["fern-classifier"]);
462
463 const cv::FileNode node_ = node["model-points"];
464 cv::read(node_, modelPoints);
465
466 cv::LDetector d(radius, threshold, nbOctave, nbView, patchSize, dist);
467 ldetector = d;
468 hasLearn = true;
469}
470
478void vpFernClassifier::record(const std::string &_objectName, const std::string &_dataFile)
479{
480 /* part of code from OpenCV planarObjectDetector */
481 cv::FileStorage fs(_dataFile, cv::FileStorage::WRITE);
482
483 cv::WriteStructContext ws(fs, _objectName, CV_NODE_MAP);
484
485 {
486 cv::WriteStructContext wsroi(fs, "model-roi", CV_NODE_SEQ + CV_NODE_FLOW);
487 cv::write(fs, modelROI_Ref.x);
488 cv::write(fs, modelROI_Ref.y);
489 cv::write(fs, modelROI_Ref.width);
490 cv::write(fs, modelROI_Ref.height);
491 }
492
493 ldetector.write(fs, "detector");
494 cv::write(fs, "model-points", modelPoints);
495 fernClassifier.write(fs, "fern-classifier");
496}
497
504void vpFernClassifier::setImage(const vpImage<unsigned char> &I)
505{
506#if (VISP_HAVE_OPENCV_VERSION >= 0x020408)
507 vpImageConvert::convert(I, curImg);
508#else
509 if (curImg != NULL) {
510 cvResetImageROI(curImg);
511 if ((curImg->width % 8) == 0) {
512 curImg->imageData = NULL;
513 cvReleaseImageHeader(&curImg);
514 } else {
515 cvReleaseImage(&curImg);
516 }
517 curImg = NULL;
518 }
519 if ((I.getWidth() % 8) == 0) {
520 curImg = cvCreateImageHeader(cvSize((int)I.getWidth(), (int)I.getHeight()), IPL_DEPTH_8U, 1);
521 if (curImg != NULL) {
522 curImg->imageData = (char *)I.bitmap;
523 } else {
524 throw vpException(vpException::memoryAllocationError, "Could not create the image in the OpenCV format.");
525 }
526 } else {
527 vpImageConvert::convert(I, curImg);
528 }
529 if (curImg == NULL) {
530 std::cout << "!> conversion failed" << std::endl;
531 throw vpException(vpException::notInitialized, "conversion failed");
532 }
533#endif
534}
535
536#elif !defined(VISP_BUILD_SHARED_LIBS)
537// Work arround to avoid warning: libvisp_vision.a(vpFernClassifier.cpp.o) has
538// no symbols
539void dummy_vpFernClassifier(){};
540#endif
class that defines what is a Keypoint. This class provides all the basic elements to implement classe...
Class to define RGB colors available for display functionnalities.
Definition: vpColor.h:158
static const vpColor red
Definition: vpColor.h:217
static const vpColor green
Definition: vpColor.h:220
static void displayCross(const vpImage< unsigned char > &I, const vpImagePoint &ip, unsigned int size, const vpColor &color, unsigned int thickness=1)
error that can be emited by ViSP classes.
Definition: vpException.h:72
@ notInitialized
Used to indicate that a parameter is not initialized.
Definition: vpException.h:98
@ memoryAllocationError
Memory allocation error.
Definition: vpException.h:88
static void convert(const vpImage< unsigned char > &src, vpImage< vpRGBa > &dest)
Class that defines a 2D point in an image. This class is useful for image processing and stores only ...
Definition: vpImagePoint.h:88
void set_j(double jj)
Definition: vpImagePoint.h:177
double get_j() const
Definition: vpImagePoint.h:214
void set_i(double ii)
Definition: vpImagePoint.h:166
double get_u() const
Definition: vpImagePoint.h:262
double get_i() const
Definition: vpImagePoint.h:203
double get_v() const
Definition: vpImagePoint.h:273
static void crop(const vpImage< Type > &I, double roi_top, double roi_left, unsigned int roi_height, unsigned int roi_width, vpImage< Type > &crop, unsigned int v_scale=1, unsigned int h_scale=1)
Definition: vpImageTools.h:305
unsigned int getWidth() const
Definition: vpImage.h:246
Type * bitmap
points toward the bitmap
Definition: vpImage.h:143
unsigned int getHeight() const
Definition: vpImage.h:188
Defines a rectangle in the plane.
Definition: vpRect.h:80
double getWidth() const
Definition: vpRect.h:228
double getLeft() const
Definition: vpRect.h:174
double getHeight() const
Definition: vpRect.h:167
double getTop() const
Definition: vpRect.h:193
#define vpTRACE
Definition: vpDebug.h:416
#define vpERROR_TRACE
Definition: vpDebug.h:393