Attach face metadata to pipeline in nvtracker probe second method

CMakeLists.txt

@@ -77,6 +77,7 @@ include_directories(${PROJECT_SOURCE_DIR}/metrics_manager.hpp)
 include_directories(${PROJECT_SOURCE_DIR}/config_manager.hpp)
 include_directories(${PROJECT_SOURCE_DIR}/nv_infer_server_manager.hpp)
 include_directories(${PROJECT_SOURCE_DIR}/nv_tracker_manager.hpp)
+include_directories(${PROJECT_SOURCE_DIR}/face_candid_trace.hpp)
 
 
 
@@ -85,7 +86,7 @@ set(SOURCES src/main.cpp  src/camera_manager.cpp src/pipeline_manager.cpp src/st
             src/nv_video_convert_manager.cpp src/nv_osd_manager.cpp src/queue_manager.cpp 
             src/nv_ds_logger_manager.cpp src/sink_manager.cpp src/message_handling.cpp
             src/rtsp_streaming_manager.cpp src/metrics_manager.cpp src/config_manager.cpp
-            src/nv_infer_server_manager.cpp src/nv_tracker_manager.cpp)
+            src/nv_infer_server_manager.cpp src/nv_tracker_manager.cpp src/face_candid_trace.cpp)
 
 
 # missing initializer for member 'NvDsInferDims::d' [-Werror=missing-field-initializers] NvDsInferDims dims = {0};

src/face_candid_trace.cpp

@@ -0,0 +1,87 @@
+#include "face_candid_trace.hpp"
+
+FaceCandidTrace::FaceCandidTrace() {
+    maxlen = 150;
+    // interval_frame_log=20;
+    face_detection_threshold = 0.5;
+    face_surface_threshold = 100;
+    termostat_threshold = 0.00001;
+}
+
+bool FaceCandidTrace::filter(FaceCandidate* face_candidate) {
+    float w = face_candidate->w;
+    float h = face_candidate->h;
+    float face_surface = w * h;
+    float score_face = face_candidate->face_score;  // result["temp_face_score"]
+    // int frame = face_candidate->frame_number; //result["frame_count"]
+
+    if (score_face < face_detection_threshold ||
+        face_surface < face_surface_threshold) {
+        // if self.logger and self._should_log(frame):
+        //     self.logger.info(f"Ignore instance — surface={face_surface},
+        //     score={score:.2f}, track_id={result['track_id']} @ {frame}",
+        //     extra={"stream_id": stream_id})
+        // elif self._should_log(frame):
+        //     print(f"Ignore instance — surface={face_surface},
+        //     score={score:.2f}, track_id={result['track_id']} @ {frame}")
+        return false;
+    }
+    return true;
+}
+
+float FaceCandidTrace::metric(FaceCandidate* face_candidate) {
+    // x1, y1, x2, y2 = result["face_bbox"]
+    // face_surface = (y2 - y1) * (x2 - x1)
+    float w = face_candidate->w;
+    float h = face_candidate->h;
+    float face_surface = w * h;
+    face_surface = std::max<float>(face_surface, 30000);
+    face_surface = face_surface / 30000;
+    float out = 0.3 * face_candidate->face_score + 0.7 * face_surface;
+    // # self.termostat_threshold = 0.02 * out
+    return out;
+}
+
+float FaceCandidTrace::check_existence(int object_id, int source_id) {
+    for (std::vector<FaceBody>::iterator iter = queue.begin();
+         iter != queue.end(); iter++) {
+        if (((*iter).object_id == object_id) &&
+            ((*iter).source_id == source_id)) {
+            return (*iter).face_score;
+        }
+    }
+    return 0;
+}
+
+bool FaceCandidTrace::add(FaceCandidate* face_candidate) {
+    if (filter(face_candidate)) {
+        float current_metric = metric(face_candidate);
+        float prev = check_existence(face_candidate->object_id,
+                                     face_candidate->source_id);
+        // prev = self.queue.get(track_id, 0)
+        if ((current_metric - termostat_threshold) > prev) {
+            queue.emplace_back(
+                FaceBody{face_candidate->object_id, face_candidate->source_id,
+                         face_candidate->frame_number, current_metric});
+            std::cout << "FaceCandidTrace source_id = "
+                      << face_candidate->source_id
+                      << " frame_num = " << face_candidate->frame_number
+                      << " object_id = " << face_candidate->object_id
+                      << " size queue = " << queue.size() << std::endl;
+            // self.queue.add(track_id, current_metric);
+            return true;
+        }
+        // int frame_count = face_candidate->frame_number;
+        // //result["frame_count"] if self.ogger and
+        // self._should_log(frame_count):
+        //     self.logger.info(f"Ignore (better seen before):
+        //     now={current_metric - self.termostat_threshold:.2f},
+        //     history={prev:.2f}, track_id={result['track_id']} @
+        //     {frame_count}", extra={"stream_id": stream_id})
+        // elif self._should_log(frame_count):
+        //     print(f"Ignore (better seen before): now={current_metric -
+        //     self.termostat_threshold:.2f}, history={prev:.2f},
+        //     track_id={result['track_id']} @ {frame_count}")
+    }
+    return false;
+}

src/face_candid_trace.hpp

@@ -0,0 +1,39 @@
+#include <algorithm>  // Required for std::max
+#include <fstream>
+#include <iostream>
+
+#include "gstnvdsmeta.h"
+#include "nvdsmeta.h"
+#include "nvdsmeta_schema.h"
+
+class FaceCandidTrace {
+   private:
+    int maxlen;
+    // int interval_frame_log;
+    float face_detection_threshold;
+    int face_surface_threshold;
+    float termostat_threshold;
+    struct FaceBody {
+        int object_id = 0;
+        int source_id = 0;
+        int num_frames = 0;
+        float face_score = -1;
+    };
+    std::vector<FaceBody> queue;
+
+   public:
+    struct FaceCandidate {
+        float w = 0;
+        float h = 0;
+        float face_score;
+        gint frame_number;
+        int object_id;
+        int source_id;
+    };
+    FaceCandidTrace();
+    ~FaceCandidTrace();
+    bool filter(FaceCandidate *);
+    float metric(FaceCandidate *face_candidate);
+    bool add(FaceCandidate *);
+    float check_existence(int, int);
+};

src/nv_tracker_manager.cpp

@@ -16,6 +16,8 @@ unsigned int NvTrackerManager::PGIE_NET_HEIGHT = 1;
 unsigned int NvTrackerManager::MUXER_OUTPUT_WIDTH = 1;
 unsigned int NvTrackerManager::MUXER_OUTPUT_HEIGHT = 1;
 std::vector<NvTrackerManager::FaceBody> NvTrackerManager::body_face_list;
+FaceCandidTrace *NvTrackerManager::face_candidate_trace =
+    new FaceCandidTrace();  // nullptr;  // Definition
 
 gint NvTrackerManager::frame_number = 0;
 
@@ -45,6 +47,11 @@ bool NvTrackerManager::create_nv_tracker() {
     return true;
 }
 
+void NvTrackerManager::get_face_score(float *user_meta_data) {
+    face_score =
+        (user_meta_data[8] + user_meta_data[11] + user_meta_data[14]) / 3;
+}
+
 bool NvTrackerManager::check_existence(int object_id, int source_id, float area,
                                        bool *is_area_updated) {
     for (std::vector<FaceBody>::iterator iter = body_face_list.begin();
@@ -75,7 +82,7 @@ void NvTrackerManager::attach_probe_to_element() {
     }
 
     gst_pad_add_probe(src_pad, GST_PAD_PROBE_TYPE_BUFFER,
-                      tracker_src_pad_buffer_probe, NULL, NULL);
+                      tracker_src_pad_buffer_probe, this, NULL);
     gst_object_unref(src_pad);
 }
 
@@ -86,7 +93,11 @@ void NvTrackerManager::attach_probe_to_element() {
 GstPadProbeReturn NvTrackerManager::tracker_src_pad_buffer_probe(
     GstPad *pad, GstPadProbeInfo *info, gpointer u_data) {
     (void)pad;
-    (void)u_data;
+    // (void)u_data;
+
+    // Cast user_data back to NvTrackerManager*
+    NvTrackerManager *manager = static_cast<NvTrackerManager *>(u_data);
+
     GstBuffer *buf = (GstBuffer *)info->data;
     guint num_rects = 0;
     guint person_count = 0;
@@ -182,6 +193,9 @@ GstPadProbeReturn NvTrackerManager::tracker_src_pad_buffer_probe(
                     // face_rect_params.top, .left, .width, .height
                     NvOSD_RectParams *face_rect_params = new NvOSD_RectParams();
 
+                    face_rect_params->left = user_meta_data[index * 57 + 0];
+                    face_rect_params->top = user_meta_data[index * 57 + 1];
+
                     /* Assign bounding box coordinates. */
                     // Right Shoulder - Left Shoulder
                     if (user_meta_data[index * 57 + 24] >
@@ -221,7 +235,29 @@ GstPadProbeReturn NvTrackerManager::tracker_src_pad_buffer_probe(
                     face_obj->rect_params.border_width = 2;
                     face_obj->rect_params.border_color =
                         NvOSD_ColorParams{1.0, 0.0, 0.0, 1.0};  // Red box
-                    // std::quick_exit(0);
+
+                    manager->get_face_score(user_meta_data);
+
+                    // FaceCandidate *face_candidate = new FaceCandidate();
+                    FaceCandidTrace::FaceCandidate *face_candidate =
+                        new FaceCandidTrace::FaceCandidate();
+                    // NvTrackerManager::FaceCandidate* face_candidate = new
+                    // NvTrackerManager::FaceCandidate();
+
+                    // manager->face_candidate
+                    face_candidate->frame_number = frame_meta->frame_num;
+                    face_candidate->h = face_rect_params->height;
+                    face_candidate->w = face_rect_params->width;
+                    face_candidate->face_score = manager->face_score;
+                    face_candidate->object_id = obj_meta->object_id;
+                    face_candidate->source_id = frame_meta->source_id;
+                    bool add_status = face_candidate_trace->add(face_candidate);
+
+                    if (add_status) {
+                        face_obj->class_id = FACE_CLASS_ID;
+                    } else {
+                        face_obj->class_id = 41;
+                    }
 
                     // std::cout << "In Tracker sink "
                     //         << " source_id = " << frame_meta->source_id
@@ -232,35 +268,36 @@ GstPadProbeReturn NvTrackerManager::tracker_src_pad_buffer_probe(
                     //         << " h = " << obj_meta->rect_params.height
                     //         << " score = " << obj_meta->confidence
                     //         << std::endl;
-                    bool is_area_updated = false;
+
-                    FaceBody current_face;
+                    // bool is_area_updated = false;
-                    current_face.largest_area = face_obj->rect_params.height *
+                    // FaceBody current_face;
-                                                face_obj->rect_params.width;
+                    // current_face.largest_area = face_obj->rect_params.height
-                    current_face.object_id = obj_meta->object_id;
+                    // *
-                    current_face.source_id = frame_meta->source_id;
+                    //                             face_obj->rect_params.width;
-                    if (!check_existence(
+                    // current_face.object_id = obj_meta->object_id;
-                            obj_meta->object_id, current_face.source_id,
+                    // current_face.source_id = frame_meta->source_id;
-                            current_face.largest_area, &is_area_updated)) {
+                    // if (!check_existence(
-                        current_face.num_frames = 1;
+                    //         obj_meta->object_id, current_face.source_id,
-                        body_face_list.push_back(current_face);
+                    //         current_face.largest_area, &is_area_updated)) {
-                        std::cout << "source_id = " << current_face.source_id
+                    //     current_face.num_frames = 1;
-                                  << " frame_num = " << frame_meta->frame_num
+                    //     body_face_list.push_back(current_face);
-                                  << " object_id = " << obj_meta->object_id
+                    //     std::cout << "source_id = " << current_face.source_id
-                                  << " size body_face_list = "
+                    //               << " frame_num = " << frame_meta->frame_num
-                                  << body_face_list.size() << std::endl;
+                    //               << " object_id = " << obj_meta->object_id
-                        face_obj->class_id = FACE_CLASS_ID;
+                    //               << " size body_face_list = "
-                    }
+                    //               << body_face_list.size() << std::endl;
-                    if (is_area_updated) {
+                    //     face_obj->class_id = FACE_CLASS_ID;
-                        face_obj->class_id = FACE_CLASS_ID;
+                    // }
-                        std::cout << "source_id = " << current_face.source_id
+                    // if (is_area_updated) {
-                                  << " frame_num = " << frame_meta->frame_num
+                    //     face_obj->class_id = FACE_CLASS_ID;
-                                  << " object_id = " << obj_meta->object_id
+                    //     std::cout << "source_id = " << current_face.source_id
-                                  << " area is updated" << std::endl;
+                    //               << " frame_num = " << frame_meta->frame_num
-                        // std::quick_exit(0);
+                    //               << " object_id = " << obj_meta->object_id
-                    } else {
+                    //               << " area is updated" << std::endl;
-                        face_obj->class_id = 41;
+                    // } else {
-                        // std::cout<<"not is_area_updated "<< std::endl;
+                    //     face_obj->class_id = 41;
-                    }
+                    //     // std::cout<<"not is_area_updated "<< std::endl;
+                    // }
 
                     // NvOSD_RectParams &rect_params = obj_meta->rect_params;
                     // NvOSD_TextParams &text_params = obj_meta->text_params;

src/nv_tracker_manager.hpp

@@ -7,6 +7,7 @@
 #include <vector>
 
 #include "config_manager.hpp"
+#include "face_candid_trace.hpp"
 #include "gstnvdsmeta.h"
 #include "nvdsmeta.h"
 #include "nvdsmeta_schema.h"
@@ -19,14 +20,25 @@ class NvTrackerManager {
         int num_frames = 0;
         float largest_area = -1;
     };
+    struct FaceCandidate {
+        float w = 0;
+        float h = 0;
+        float face_score;
+        gint frame_number;
+        int object_id;
+        int source_id;
+    };
 
     static std::vector<FaceBody> body_face_list;
+    static FaceCandidTrace *face_candidate_trace;
+    float face_score;
 
    public:
     static unsigned int PGIE_NET_WIDTH;
     static unsigned int PGIE_NET_HEIGHT;
     static unsigned int MUXER_OUTPUT_WIDTH;
     static unsigned int MUXER_OUTPUT_HEIGHT;
+
     GstElement *tracker = NULL;
     static gint frame_number;
     std::string ll_config_file;
@@ -39,4 +51,5 @@ class NvTrackerManager {
                                                           GstPadProbeInfo *,
                                                           gpointer);
     static bool check_existence(int, int, float, bool *);
+    void get_face_score(float *);
 };