35#include <visp3/core/vpConfig.h>
37#if (VISP_HAVE_OPENCV_VERSION >= 0x030403) && defined(HAVE_OPENCV_DNN) && (VISP_CXX_STANDARD >= VISP_CXX_STANDARD_17)
38#include <visp3/core/vpImageConvert.h>
39#include <visp3/detection/vpDetectorDNNOpenCV.h>
40#include <visp3/core/vpIoTools.h>
50 std::string list =
"[";
90 name =
"ssd-mobilenet";
96 name =
"user-specified";
116 bool hasFoundMatch =
false;
118 for (
int id = 0;
id <
COUNT && !hasFoundMatch;
id++) {
122 hasFoundMatch =
true;
140 return NetConfig::parseClassNamesFile(filename);
144 : m_applySizeFilterAfterNMS(false), m_blob(), m_I_color(), m_img(),
145 m_net(), m_netConfig(), m_outNames(), m_dnnRes(),
159 : m_applySizeFilterAfterNMS(false), m_blob(), m_I_color(), m_img(),
160 m_net(), m_netConfig(config), m_outNames(), m_dnnRes()
169#ifdef VISP_HAVE_NLOHMANN_JSON
177 : m_applySizeFilterAfterNMS(false), m_blob(), m_I_color(), m_img(),
178 m_net(), m_netConfig(), m_outNames(), m_dnnRes()
192 std::ifstream file(jsonPath);
194 std::stringstream ss;
195 ss <<
"Problem opening file " << jsonPath <<
". Make sure it exists and is readable" << std::endl;
200 j = json::parse(file);
202 catch (json::parse_error &e) {
203 std::stringstream msg;
204 msg <<
"Could not parse JSON file : \n";
206 msg << e.what() << std::endl;
207 msg <<
"Byte position of error: " << e.byte;
222 std::ofstream file(jsonPath);
223 const json j = *
this;
348 catch (
const cv::Exception &e) {
349 std::cerr <<
"Caught an exception trying to run inference:" << std::endl <<
"\t"
351 <<
"\nCuda and/or GPU driver might not be correctly installed. Setting preferable backend to CPU and trying again." << std::endl;
352 m_net.setPreferableBackend(cv::dnn::DNN_BACKEND_DEFAULT);
353 m_net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
359 size_t nbClassNames =
m_netConfig.m_classNames.size();
360 for (
size_t i = 0; i <
m_indices.size(); ++i) {
362 cv::Rect box = proposals.
m_boxes[idx];
363 std::optional<std::string> classname_opt;
364 if (nbClassNames > 0) {
367 output.emplace_back(box.x, box.x + box.width, box.y, box.y + box.height
378 return !output.empty();
401 catch (
const cv::Exception &e) {
402 std::cerr <<
"Caught an exception trying to run inference:" << std::endl <<
"\t"
404 <<
"\nCuda and/or GPU driver might not be correctly installed. Setting preferable backend to CPU and trying again." << std::endl;
405 m_net.setPreferableBackend(cv::dnn::DNN_BACKEND_DEFAULT);
406 m_net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
412 size_t nbClassNames =
m_netConfig.m_classNames.size();
413 for (
size_t i = 0; i <
m_indices.size(); ++i) {
415 cv::Rect box = proposals.
m_boxes[idx];
416 std::string classname;
417 if (nbClassNames > 0) {
421 classname = std::to_string(proposals.
m_classIds[idx]);
423 std::optional<std::string> classname_opt = std::optional<std::string>(classname);
424 output[classname].emplace_back(box.x, box.x + box.width, box.y, box.y + box.height
434 return !output.empty();
446 std::map< std::string, std::vector<DetectedFeatures2D>> map_output;
447 bool returnStatus =
detect(I, map_output);
448 for (
auto key_val : map_output) {
449 output.push_back(key_val);
454#if (VISP_HAVE_OPENCV_VERSION == 0x030403)
462 static std::vector<cv::String> names;
464 std::vector<int> outLayers =
m_net.getUnconnectedOutLayers();
465 std::vector<cv::String> layersNames =
m_net.getLayerNames();
466 names.resize(outLayers.size());
467 for (
size_t i = 0; i < outLayers.size(); ++i)
468 names[i] = layersNames[outLayers[i] - 1];
500#if defined(VISP_BUILD_DEPRECATED_FUNCTIONS)
533std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D>
537 double originalNumberOfObj =
static_cast<double>(detected_features.size());
538 double meanFactor = 1. / originalNumberOfObj;
542 meanArea += feature.m_bbox.getArea();
544 meanArea *= meanFactor;
547 std::vector<DetectedFeatures2D> filtered_features;
549 if (feature.m_bbox.getArea() >= minRatioOfAreaOk * meanArea && feature.m_bbox.getArea() < meanArea / minRatioOfAreaOk) {
550 filtered_features.push_back(feature);
554 return filtered_features;
567std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D>
570#ifndef DOXYGEN_SHOULD_SKIP_THIS
575 class MeanAreaComputer
578 std::map<int, std::pair<int, double>> m_map_id_pairOccurrencesAreas;
581 std::map<int, double> m_mapMeans;
588 double computeMeanArea(
const int &class_id)
590 return m_map_id_pairOccurrencesAreas[class_id].second / (double)m_map_id_pairOccurrencesAreas[class_id].first;
599 for (
const auto &classID_pair : m_map_id_pairOccurrencesAreas) {
600 m_mapMeans[classID_pair.first] = computeMeanArea(classID_pair.first);
604 double getMean(
const int &class_id)
606 if (m_map_id_pairOccurrencesAreas.find(class_id) == m_map_id_pairOccurrencesAreas.end()) {
607 throw(
vpException(
vpException::badValue,
"[MeanAreaComputer::getMean] Asking for class_id \"" + std::to_string(class_id) +
"\" that is not present in m_mapMeans. Did you call computeMeans ?"));
609 return m_mapMeans[class_id];
621 if (m_map_id_pairOccurrencesAreas.find(class_id) == m_map_id_pairOccurrencesAreas.end()) {
622 m_map_id_pairOccurrencesAreas[class_id] = std::pair<int, double>(1, area);
625 std::pair<int, double> prev_state = m_map_id_pairOccurrencesAreas[class_id];
626 m_map_id_pairOccurrencesAreas[class_id] = std::pair<int, double>(prev_state.first + 1, prev_state.second + area);
633 MeanAreaComputer meanComputer;
634 std::for_each(detected_features.begin(), detected_features.end(), meanComputer);
635 meanComputer.computeMeans();
638 std::vector<DetectedFeatures2D> filtered_features;
640 double meanArea = meanComputer.getMean(feature.getClassId());
641 if (feature.m_bbox.getArea() >= minRatioOfAreaOk * meanArea
642 && feature.m_bbox.getArea() < meanArea / minRatioOfAreaOk) {
643 filtered_features.push_back(feature);
647 return filtered_features;
659std::map<std::string, std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D>>
662 std::map<std::string, std::vector<vpDetectorDNNOpenCV::DetectedFeatures2D>> output;
663 for (
auto keyval : detected_features) {
684 size_t nbBatches = dnnRes.size();
686 for (
size_t i = 0; i < nbBatches; i++) {
689 int num_proposal = dnnRes[i].size[0];
690 int nout = dnnRes[i].size[1];
691 if (dnnRes[i].dims > 2) {
692 num_proposal = dnnRes[i].size[1];
693 nout = dnnRes[i].size[2];
694 dnnRes[i] = dnnRes[i].reshape(0, num_proposal);
697 int n = 0, row_ind = 0;
698 float *pdata = (
float *)dnnRes[i].data;
701 for (n = 0; n < num_proposal; n++) {
702 float box_score = pdata[4];
703 if (box_score > netConfig.m_confThreshold) {
704 cv::Mat scores = dnnRes[i].row(row_ind).colRange(5, nout);
705 cv::Point classIdPoint;
706 double max_class_score;
708 cv::minMaxLoc(scores, 0, &max_class_score, 0, &classIdPoint);
710 max_class_score *= box_score;
713 if (max_class_score > netConfig.m_confThreshold) {
714 const int class_idx = classIdPoint.x;
715 float cx = pdata[0] *
m_img.cols;
716 float cy = pdata[1] *
m_img.rows;
717 float w = pdata[2] *
m_img.cols;
718 float h = pdata[3] *
m_img.rows;
720 int left = int(cx - 0.5 * w);
721 int top = int(cy - 0.5 * h);
724 proposals.
m_boxes.push_back(cv::Rect(left, top, (
int)(w), (
int)(h)));
749 float ratioh = (float)
m_img.rows / netConfig.m_inputSize.height, ratiow = (
float)
m_img.cols / netConfig.m_inputSize.width;
750 size_t nbBatches = dnnRes.size();
752 for (
size_t i = 0; i < nbBatches; i++) {
754 int num_proposal = dnnRes[i].size[0];
755 int nout = dnnRes[i].size[1];
756 if (dnnRes[i].dims > 2) {
757 num_proposal = dnnRes[i].size[1];
758 nout = dnnRes[i].size[2];
759 dnnRes[i] = dnnRes[i].reshape(0, num_proposal);
762 int n = 0, row_ind = 0;
763 float *pdata = (
float *)dnnRes[i].data;
766 for (n = 0; n < num_proposal; n++) {
767 float box_score = pdata[4];
769 if (box_score > netConfig.m_confThreshold) {
770 cv::Mat scores = dnnRes[i].row(row_ind).colRange(5, nout);
771 cv::Point classIdPoint;
772 double max_class_score;
774 cv::minMaxLoc(scores, 0, &max_class_score, 0, &classIdPoint);
775 max_class_score *= box_score;
778 if (max_class_score > netConfig.m_confThreshold) {
779 const int class_idx = classIdPoint.x;
780 float cx = pdata[0] * ratiow;
781 float cy = pdata[1] * ratioh;
782 float w = pdata[2] * ratiow;
783 float h = pdata[3] * ratioh;
785 int left = int(cx - 0.5 * w);
786 int top = int(cy - 0.5 * h);
789 proposals.
m_boxes.push_back(cv::Rect(left, top, (
int)(w), (
int)(h)));
815 float ratioh = (float)
m_img.rows / netConfig.m_inputSize.height, ratiow = (
float)
m_img.cols / netConfig.m_inputSize.width;
816 size_t nbBatches = dnnRes.size();
818 for (
size_t i = 0; i < nbBatches; i++) {
820 int num_proposal = dnnRes[i].size[1];
821 int nout = dnnRes[i].size[0];
822 if (dnnRes[i].dims > 2) {
823 num_proposal = dnnRes[i].size[2];
824 nout = dnnRes[i].size[1];
825 dnnRes[i] = dnnRes[i].reshape(0, nout);
827 cv::transpose(dnnRes[i], dnnRes[i]);
829 int n = 0, row_ind = 0;
830 float *pdata = (
float *)dnnRes[i].data;
833 for (n = 0; n < num_proposal; n++) {
834 cv::Mat scores = dnnRes[i].row(row_ind).colRange(4, nout);
835 cv::Point classIdPoint;
836 double max_class_score;
838 cv::minMaxLoc(scores, 0, &max_class_score, 0, &classIdPoint);
841 if (max_class_score > netConfig.m_confThreshold) {
842 const int class_idx = classIdPoint.x;
843 float cx = pdata[0] * ratiow;
844 float cy = pdata[1] * ratioh;
845 float w = pdata[2] * ratiow;
846 float h = pdata[3] * ratioh;
848 int left = int(cx - 0.5 * w);
849 int top = int(cy - 0.5 * h);
852 proposals.
m_boxes.push_back(cv::Rect(left, top, (
int)(w), (
int)(h)));
881 size_t nbBatches = dnnRes.size();
882 for (
size_t j = 0; j < nbBatches; j++) {
883 float *data = (
float *)dnnRes[j].data;
884 for (
size_t i = 0; i < dnnRes[j].total(); i += 7) {
885 float confidence = data[i + 2];
886 if (confidence > netConfig.m_confThreshold) {
887 int left = (int)(data[i + 3] *
m_img.cols);
888 int top = (int)(data[i + 4] *
m_img.rows);
889 int right = (int)(data[i + 5] *
m_img.cols);
890 int bottom = (int)(data[i + 6] *
m_img.rows);
891 int classId = (int)(data[i + 1]);
894 proposals.
m_boxes.push_back(cv::Rect(left, top, right - left + 1, bottom - top + 1));
902#if defined(VISP_BUILD_DEPRECATED_FUNCTIONS)
922 int scores_index =
m_outNames[0] ==
"scores" ? 0 : 1;
923 int boxes_index =
m_outNames[0] ==
"boxes" ? 0 : 1;
925 int N = dnnRes[scores_index].size[1], C = dnnRes[scores_index].size[2];
927 float *confidence = (
float *)dnnRes[scores_index].data;
928 float *bbox = (
float *)dnnRes[boxes_index].data;
931 for (
int i = 0; i < N; i++) {
932 uint32_t maxClass = 0;
933 float maxScore = -1000.0f;
935 for (
int j = 1; j < C; j++)
937 const float score = confidence[i * C + j];
939 if (score < netConfig.m_confThreshold)
942 if (score > maxScore) {
948 if (maxScore > netConfig.m_confThreshold) {
949 int left = (int)(bbox[4 * i] *
m_img.cols);
950 int top = (int)(bbox[4 * i + 1] *
m_img.rows);
951 int right = (int)(bbox[4 * i + 2] *
m_img.cols);
952 int bottom = (int)(bbox[4 * i + 3] *
m_img.rows);
953 int width = right - left + 1;
954 int height = bottom - top + 1;
956 int classId = maxClass;
958 proposals.
m_boxes.push_back(cv::Rect(left, top, width, height));
983 CV_Assert(dnnRes.size() == 1);
984 float *data = (
float *)dnnRes[0].data;
985 for (
size_t i = 0; i < dnnRes[0].total(); i += 7) {
986 float confidence = data[i + 2];
987 if (confidence > netConfig.m_confThreshold) {
988 int left = (int)(data[i + 3] *
m_img.cols);
989 int top = (int)(data[i + 4] *
m_img.rows);
990 int right = (int)(data[i + 5] *
m_img.cols);
991 int bottom = (int)(data[i + 6] *
m_img.rows);
992 int classId = (int)(data[i + 1]) - 1;
995 proposals.
m_boxes.push_back(cv::Rect(left, top, right - left + 1, bottom - top + 1));
1041 m_net = cv::dnn::readNet(model, config, framework);
1042#if (VISP_HAVE_OPENCV_VERSION == 0x030403)
1090 if (
m_netConfig.m_filterSizeRatio > std::numeric_limits<double>::epsilon()) {
1142 std::cout <<
"[vpDetectorDNNOpenCV::setParsingMethod] WARNING: scale factor should be 1/255. to normalize pixels value." << std::endl;
1162 m_netConfig.m_parsingMethodType = typeParsingMethod;
1166 std::cout <<
"[vpDetectorDNNOpenCV::setParsingMethod] NB: scale factor changed to 1/255. to normalize pixels value." << std::endl;
1169#if defined(VISP_BUILD_DEPRECATED_FUNCTIONS)
1171 std::cout <<
"[vpDetectorDNNOpenCV::setParsingMethod] WARNING: The chosen type of network is " <<
dnnResultsParsingTypeToString(
m_netConfig.m_parsingMethodType) <<
" VISP_BUILD_DEPRECATED_FUNCTIONS is set to true." << std::endl;
1172 std::cout <<
"\tThe parsing method that worked with the networks quoted in the ViSP documentation was postProcess_ResNet_10 instead of postProcess_SSD_MobileNet." << std::endl;
1173 std::cout <<
"\tIf the SSD-MobileNet network does not seem to work, please try to recompile ViSP setting VISP_BUILD_DEPRECATED_FUNCTIONS as false." << std::endl << std::flush;
1178#elif !defined(VISP_BUILD_SHARED_LIBS)
1181void dummy_vpDetectorDNN() { };
Structure containing the bounding box, expressed in pixels, confidence and class information about an...
vpRect getBoundingBox() const
unsigned int getClassId() const
Structure containing some information required for the configuration of a vpDetectorDNNOpenCV object.
cv::Mat m_blob
Buffer for the blob in input net.
void postProcess_YoloV5_V7(DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
void setScaleFactor(const double &scaleFactor)
void initFromJSON(const std::string &jsonPath)
void readNet(const std::string &model, const std::string &config="", const std::string &framework="")
static std::string getAvailableDnnResultsParsingTypes()
Get the list of the parsing methods / types of DNNs supported by the vpDetectorDNNOpenCV class.
static void postProcess_unimplemented(DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
void setDetectionFilterSizeRatio(const double &sizeRatio)
DNNResultsParsingType
Enumeration listing the types of DNN for which the vpDetectorDNNOpenCV furnishes the methods permitti...
static DNNResultsParsingType dnnResultsParsingTypeFromString(const std::string &name)
void postProcess_SSD_MobileNet(DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
std::vector< cv::String > m_outNames
Names of layers with unconnected outputs.
void setMean(const double &meanR, const double &meanG, const double &meanB)
void setSwapRB(const bool &swapRB)
cv::Mat m_img
Buffer for the input image.
static std::vector< std::string > parseClassNamesFile(const std::string &filename)
Parse the designated file that contains the list of the classes the network can detect....
std::vector< int > m_indices
Indices for NMS.
void setParsingMethod(const DNNResultsParsingType &typeParsingMethod, void(*parsingMethod)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)=postProcess_unimplemented)
NetConfig m_netConfig
Configuration of the DNN.
std::vector< cv::Mat > m_dnnRes
Contains all output blobs for each layer specified in m_outNames.
cv::dnn::Net m_net
DNN network.
bool m_applySizeFilterAfterNMS
If true, filter the detections removing the ones for which the bbox does not respect area(bbox) € [me...
std::vector< cv::String > getOutputsNames()
Get the names of the output layers of the DNN.
void setNetConfig(const NetConfig &config)
void postProcess_YoloV3_V4(DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
virtual bool detect(const vpImage< unsigned char > &I, std::vector< DetectedFeatures2D > &output)
Object detection using OpenCV DNN module.
void postProcess_ResNet_10(DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
void setPreferableBackend(const int &backendId)
void setNMSThreshold(const float &nmsThreshold)
virtual ~vpDetectorDNNOpenCV()
Destroy the vpDetectorDNNOpenCV object.
void postProcess_FasterRCNN(DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
std::vector< DetectedFeatures2D > filterDetectionMultiClassInput(const std::vector< DetectedFeatures2D > &detected_features, const double minRatioOfAreaOk)
Return a new vector, ordered by vpDetectorDNNOpenCV::DetectedFeatures2D::m_cls , where the area of ea...
void setPreferableTarget(const int &targetId)
void setInputSize(const int &width, const int &height)
static std::string dnnResultsParsingTypeToString(const DNNResultsParsingType &type)
void postProcess_YoloV8(DetectionCandidates &proposals, std::vector< cv::Mat > &dnnRes, const NetConfig &netConfig)
void postProcess(DetectionCandidates &proposals)
void(* m_parsingMethod)(DetectionCandidates &, std::vector< cv::Mat > &, const NetConfig &)
Pointer towards the parsing method, used if m_parsingMethodType is equal to m_parsingMethodType::USER...
std::vector< DetectedFeatures2D > filterDetectionSingleClassInput(const std::vector< DetectedFeatures2D > &detected_features, const double minRatioOfAreaOk)
Return a new vector of detected features whose area is greater or equal to the average area x minRati...
void saveConfigurationInJSON(const std::string &jsonPath) const
Save the network configuration in a JSON file.
void setConfidenceThreshold(const float &confThreshold)
vpImage< vpRGBa > m_I_color
Buffer for gray to RGBa image conversion.
error that can be emitted by ViSP classes.
@ badValue
Used to indicate that a value is not in the allowed range.
@ functionNotImplementedError
Function not implemented.
static void convert(const vpImage< unsigned char > &src, vpImage< vpRGBa > &dest)
Definition of the vpImage class member functions.
std::vector< int > m_classIds
std::vector< float > m_confidences
std::vector< cv::Rect > m_boxes