Attach objects metadata to pgie probe

CMakeLists.txt

@@ -77,6 +77,17 @@ set(SOURCES src/main.cpp  src/camera_manager.cpp src/pipeline_manager.cpp src/st
             src/nv_ds_logger_manager.cpp src/sink_manager.cpp src/message_handling.cpp
             src/rtsp_streaming_manager.cpp src/metrics_manager.cpp src/config_manager.cpp
             src/nv_infer_server_manager.cpp)
+
+
+# missing initializer for member 'NvDsInferDims::d' [-Werror=missing-field-initializers] NvDsInferDims dims = {0};
+set_source_files_properties(
+    src/nv_infer_server_manager.cpp
+    src/main.cpp
+    src/pipeline_manager.cpp
+    PROPERTIES COMPILE_FLAGS "-Wno-missing-field-initializers"
+)
+
+
 # Create the executable
 add_executable(${PROJECT_NAME}  ${SOURCES})
 

data/configuration.json

@@ -17,5 +17,7 @@
         "host": "0.0.0.0"
     },
     "pgie_batch_size": 16,
-    "inferserver_pgie_config_file": "../data/inferserver/primary_detector_config.txt"
+    "inferserver_pgie_config_file": "../data/inferserver/primary_detector_config.txt",
+    "PGIE_NET_WIDTH": 640,
+    "PGIE_NET_HEIGHT": 640
 }

src/nv_infer_server_manager.cpp

@@ -1,10 +1,33 @@
 #include "nv_infer_server_manager.hpp"
 
+#define MAX_DISPLAY_LEN 64
+#define PGIE_CLASS_ID_PERSON 0
+#define PGIE_DETECTED_CLASS_NUM 1
+
+gint NvInferServerManager::frame_number = 0;
+unsigned int NvInferServerManager::PGIE_NET_WIDTH = 1;
+unsigned int NvInferServerManager::PGIE_NET_HEIGHT = 1;
+unsigned int NvInferServerManager::MUXER_OUTPUT_WIDTH = 1;
+unsigned int NvInferServerManager::MUXER_OUTPUT_HEIGHT = 1;
+guint NvInferServerManager::use_device_mem = 0;
+
+const gchar pgie_class_str[PGIE_DETECTED_CLASS_NUM][32] = {"Person"};
+
+/* nvds_lib_major_version and nvds_lib_minor_version is the version number of
+ * deepstream sdk */
+
+unsigned int nvds_lib_major_version = NVDS_VERSION_MAJOR;
+unsigned int nvds_lib_minor_version = NVDS_VERSION_MINOR;
+
 NvInferServerManager::NvInferServerManager() {
     const auto &config = ConfigManager::get_instance().get_config();
     pgie_batch_size = config["pgie_batch_size"];
     inferserver_pgie_config_file =
         config["inferserver_pgie_config_file"].get<std::string>();
+    PGIE_NET_WIDTH = config["PGIE_NET_WIDTH"];
+    PGIE_NET_HEIGHT = config["PGIE_NET_HEIGHT"];
+    MUXER_OUTPUT_WIDTH = config["MUXER_OUTPUT_WIDTH"];
+    MUXER_OUTPUT_HEIGHT = config["MUXER_OUTPUT_HEIGHT"];
 }
 
 bool NvInferServerManager::create_nv_infer_server(int num_sources) {
@@ -26,8 +49,10 @@ bool NvInferServerManager::create_nv_infer_server(int num_sources) {
         g_object_set(G_OBJECT(primary_detector), "batch-size", num_sources,
                      NULL);
     }
-    // std::cout << "✅ Found element: " << GST_ELEMENT_NAME(primary_detector)
+    if (!primary_detector) {
-    // << std::endl;
+        g_printerr("Could not create primary detector. Exiting.\n");
+        return false;
+    }
     return true;
 }
 
@@ -43,7 +68,6 @@ GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe(
     NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
     if (!batch_meta) {
         std::cerr << "No batch metadata found\n";
-        exit(-1);
         return GST_PAD_PROBE_OK;
     }
 
@@ -73,19 +97,265 @@ GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe(
                       << " h=" << obj_meta->rect_params.height << std::endl;
         }
     }
-
     return GST_PAD_PROBE_OK;
 }
 
 // Attach probe to a pad in the pipeline
-void NvInferServerManager::attach_probe_to_element() {
+void NvInferServerManager::attach_probe_to_element(GstElement *nvosd) {
-    GstPad *sink_pad = gst_element_get_static_pad(primary_detector, "sink");
+    GstPad *sink_pad = gst_element_get_static_pad(nvosd, "sink");
     if (!sink_pad) {
         std::cerr << "Unable to get sink pad\n";
         return;
     }
 
     gst_pad_add_probe(sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
-                      osd_sink_pad_buffer_probe, NULL, NULL);
+                      osd_sink_pad_buffer_probe_new, NULL, NULL);
     gst_object_unref(sink_pad);
 }
+
+/* This is the buffer probe function that we have registered on the sink pad
+ * of the OSD element. All the infer elements in the pipeline shall attach
+ * their metadata to the GstBuffer, here we will iterate & process the metadata
+ * forex: class ids to strings, counting of class_id objects etc. */
+GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe_new(
+    GstPad *pad, GstPadProbeInfo *info, gpointer u_data) {
+    (void)pad;
+    (void)u_data;
+    GstBuffer *buf = (GstBuffer *)info->data;
+    guint num_rects = 0;
+    guint person_count = 0;
+    NvDsObjectMeta *obj_meta = NULL;
+    NvDsMetaList *l_frame = NULL;
+    NvDsMetaList *l_obj = NULL;
+    NvDsDisplayMeta *display_meta = NULL;
+
+    NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
+
+    for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
+         l_frame = l_frame->next) {
+        NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data);
+        int offset = 0;
+        for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;
+             l_obj = l_obj->next) {
+            obj_meta = (NvDsObjectMeta *)(l_obj->data);
+            if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) {
+                person_count++;
+                num_rects++;
+                std::cout << "x = " << obj_meta->rect_params.left
+                          << " y = " << obj_meta->rect_params.top
+                          << " w = " << obj_meta->rect_params.width
+                          << " h = " << obj_meta->rect_params.height
+                          << " score = " << obj_meta->confidence << std::endl;
+            }
+        }
+        display_meta = nvds_acquire_display_meta_from_pool(batch_meta);
+        NvOSD_TextParams *txt_params = &display_meta->text_params[0];
+        display_meta->num_labels = 1;
+        txt_params->display_text = (gchar *)g_malloc0(MAX_DISPLAY_LEN);
+        offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN,
+                          "Person = %d ", person_count);
+        (void)offset;
+
+        /* Now set the offsets where the string should appear */
+        txt_params->x_offset = 10;
+        txt_params->y_offset = 12;
+
+        /* Font , font-color and font-size */
+        txt_params->font_params.font_name = (gchar *)"Serif";
+        txt_params->font_params.font_size = 10;
+        txt_params->font_params.font_color.red = 1.0;
+        txt_params->font_params.font_color.green = 1.0;
+        txt_params->font_params.font_color.blue = 1.0;
+        txt_params->font_params.font_color.alpha = 1.0;
+
+        /* Text background color */
+        txt_params->set_bg_clr = 1;
+        txt_params->text_bg_clr.red = 0.0;
+        txt_params->text_bg_clr.green = 0.0;
+        txt_params->text_bg_clr.blue = 0.0;
+        txt_params->text_bg_clr.alpha = 1.0;
+
+        nvds_add_display_meta_to_frame(frame_meta, display_meta);
+    }
+    g_print(
+        "Frame Number = %d "
+        "Person Count = %d\n",
+        frame_number, person_count);
+
+    frame_number++;
+    return GST_PAD_PROBE_OK;
+}
+
+/* This is the buffer probe function that we have registered on the src pad
+ * of the PGIE's next queue element. PGIE element in the pipeline shall attach
+ * its NvDsInferTensorMeta to each frame metadata on GstBuffer, here we will
+ * iterate & parse the tensor data to get detection bounding boxes. The result
+ * would be attached as object-meta(NvDsObjectMeta) into the same frame
+ * metadata.
+ */
+GstPadProbeReturn NvInferServerManager::pgie_pad_buffer_probe(
+    GstPad *pad, GstPadProbeInfo *info, gpointer u_data) {
+    (void)pad;
+    gboolean *use_new_mux = (gboolean *)u_data;
+    guint stream_width = 0, stream_height = 0;
+
+    static NvDsInferNetworkInfo networkInfo{PGIE_NET_WIDTH, PGIE_NET_HEIGHT, 3};
+    NvDsBatchMeta *batch_meta =
+        gst_buffer_get_nvds_batch_meta(GST_BUFFER(info->data));
+
+    /* Iterate each frame metadata in batch */
+    for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != NULL;
+         l_frame = l_frame->next) {
+        NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data;
+        // to solve track not showing up issue
+        nvds_acquire_meta_lock(batch_meta);
+        frame_meta->bInferDone = TRUE;
+        nvds_release_meta_lock(batch_meta);
+        if (*use_new_mux) {
+            stream_width = frame_meta->source_frame_width;
+            stream_height = frame_meta->source_frame_height;
+        } else {
+            stream_width = MUXER_OUTPUT_WIDTH;
+            stream_height = MUXER_OUTPUT_HEIGHT;
+        }
+        (void)stream_height;
+        (void)stream_width;
+
+        /* Iterate user metadata in frames to search PGIE's tensor metadata */
+        for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list;
+             l_user != NULL; l_user = l_user->next) {
+            NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data;
+            if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META)
+                continue;
+
+            /* convert to tensor metadata */
+            NvDsInferTensorMeta *meta =
+                (NvDsInferTensorMeta *)user_meta->user_meta_data;
+            for (unsigned int i = 0; i < meta->num_output_layers; i++) {
+                NvDsInferLayerInfo *info = &meta->output_layers_info[i];
+                info->buffer = meta->out_buf_ptrs_host[i];
+                if (use_device_mem && meta->out_buf_ptrs_dev[i]) {
+                    cudaMemcpy(meta->out_buf_ptrs_host[i],
+                               meta->out_buf_ptrs_dev[i],
+                               info->inferDims.numElements * 4,
+                               cudaMemcpyDeviceToHost);
+                }
+            }
+            /* Parse output tensor and fill detection results into objectList.
+             */
+            std::vector<NvDsInferLayerInfo> outputLayersInfo(
+                meta->output_layers_info,
+                meta->output_layers_info + meta->num_output_layers);
+#if NVDS_VERSION_MAJOR >= 5
+            if (nvds_lib_major_version >= 5) {
+                if (meta->network_info.width != networkInfo.width ||
+                    meta->network_info.height != networkInfo.height ||
+                    meta->network_info.channels != networkInfo.channels) {
+                    g_error("failed to check pgie network info\n");
+                }
+            }
+#endif
+
+            // std::cout << "frame number: " << frame_meta->frame_num
+            //           << " frame id: " << frame_meta->source_id << std::endl;
+
+            float *outputBuffer = (float *)outputLayersInfo[0].buffer;
+            (void)outputBuffer;
+            // NvDsInferDims dims = outputLayersInfo[0].inferDims;
+
+            for (size_t jkl = 0; jkl < outputLayersInfo.size(); ++jkl) {
+                const NvDsInferLayerInfo &layer = outputLayersInfo[jkl];
+
+                unsigned int numDims = layer.inferDims.numDims;
+                unsigned int numElements = layer.inferDims.numElements;
+                (void)numElements;
+                (void)numDims;
+
+                // std::cout << "Layer " << jkl << " (" << layer.layerName <<
+                // "):\n"; std::cout << "  Num Dims: " << numDims << "\n";
+                // std::cout << "  Num Elements: " << numElements << "\n";
+                // std::cout << "  Dims: [";
+                // for (unsigned int mno = 0; mno < numDims; ++mno) {
+                //     std::cout << layer.inferDims.d[mno];
+                //     // layer.inferDims.d[0] = 100;
+                //     // layer.inferDims.d[1] = 57;
+                //     if (mno < numDims - 1)
+                //        std::cout << ", ";
+                // }
+                // std::cout << "]\n";
+            }
+            const NvDsInferLayerInfo &layer =
+                outputLayersInfo[0];  // or loop over all
+
+            uint detected_persons = 0;
+            float *data = static_cast<float *>(layer.buffer);
+            for (unsigned int jkl = 0; jkl < 100; ++jkl) {
+                if (data[jkl * 57 + 4] > 0.05) {
+                    detected_persons++;
+                    std::cout << "x = " << data[jkl * 57 + 0]
+                              << " y = " << data[jkl * 57 + 1]
+                              << " w = " << data[jkl * 57 + 2]
+                              << " h = " << data[jkl * 57 + 3]
+                              << " score = " << data[jkl * 57 + 4] << std::endl;
+                    for (unsigned int mno = 0; mno < 5; ++mno) {
+                        float value = data[jkl * 57 + mno];
+                        std::cout << "data[" << jkl << "][" << mno
+                                  << "] = " << value << std::endl;
+                    }
+                }
+            }
+
+            for (uint index = 0; index < detected_persons; index++) {
+                NvDsObjectMeta *obj_meta =
+                    nvds_acquire_obj_meta_from_pool(batch_meta);
+                obj_meta->unique_component_id = meta->unique_id;
+                obj_meta->confidence = data[index * 57 + 4];
+                // obj_meta->object_id = UNTRACKED_OBJECT_ID;
+                obj_meta->class_id = 0;
+                NvOSD_RectParams &rect_params = obj_meta->rect_params;
+                NvOSD_TextParams &text_params = obj_meta->text_params;
+                /* Assign bounding box coordinates. */
+                rect_params.left =
+                    data[index * 57 + 0] * MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH;
+                rect_params.top = data[index * 57 + 1] * MUXER_OUTPUT_HEIGHT /
+                                  PGIE_NET_HEIGHT;
+                rect_params.width =
+                    (data[index * 57 + 2] - data[index * 57 + 0]) *
+                    MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH;
+                rect_params.height =
+                    (data[index * 57 + 3] - data[index * 57 + 1]) *
+                    MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT;
+                /* Border of width 3. */
+                rect_params.border_width = 3;
+                rect_params.has_bg_color = 0;
+                rect_params.border_color = NvOSD_ColorParams{1, 0, 0, 1};
+                /* display_text requires heap allocated memory. */
+                text_params.display_text = g_strdup(pgie_class_str[0]);
+                /* Display text above the left top corner of the object. */
+                text_params.x_offset = rect_params.left;
+                text_params.y_offset = rect_params.top - 10;
+                /* Set black background for the text. */
+                text_params.set_bg_clr = 1;
+                text_params.text_bg_clr = NvOSD_ColorParams{0, 0, 0, 1};
+                /* Font face, size and color. */
+                text_params.font_params.font_name = (gchar *)"Serif";
+                text_params.font_params.font_size = 11;
+                text_params.font_params.font_color =
+                    NvOSD_ColorParams{1, 1, 1, 1};
+                // adding landmarks to obj_meta as user_meta
+                // NvDsUserMeta* um1 =
+                // nvds_acquire_user_meta_from_pool(batch_meta);
+                // um1->user_meta_data = set_metadata_ptr(&(res[i].landmark[0]),
+                // source_id );  //Add landmarks here um1->base_meta.meta_type =
+                // NVDS_USER_OBJECT_META_LANDMARKS_AND_SOURCE_ID;
+                // um1->base_meta.copy_func = (NvDsMetaCopyFunc)copy_user_meta;
+                // um1->base_meta.release_func =
+                // (NvDsMetaReleaseFunc)release_user_meta;
+                // nvds_add_user_meta_to_obj(obj_meta, um1);
+                nvds_add_obj_meta_to_frame(frame_meta, obj_meta, NULL);
+            }
+        }
+    }
+    use_device_mem = 1 - use_device_mem;
+    return GST_PAD_PROBE_OK;
+}

src/nv_infer_server_manager.hpp

@@ -4,13 +4,23 @@
 #include <iostream>
 
 #include "config_manager.hpp"
+#include "gstnvdsinfer.h"
 #include "gstnvdsmeta.h"
+#include "nvds_version.h"
+#include "nvdsinfer_custom_impl.h"
 
 class NvInferServerManager {
    private:
    public:
     GstElement *primary_detector = NULL;
     int pgie_batch_size;
+
+    static unsigned int PGIE_NET_WIDTH;
+    static unsigned int PGIE_NET_HEIGHT;
+    static unsigned int MUXER_OUTPUT_WIDTH;
+    static unsigned int MUXER_OUTPUT_HEIGHT;
+    static gint frame_number;
+    static guint use_device_mem;
     std::string inferserver_pgie_config_file;
     NvInferServerManager();
     bool create_nv_infer_server(int);
@@ -18,5 +28,10 @@ class NvInferServerManager {
     static GstPadProbeReturn osd_sink_pad_buffer_probe(GstPad *,
                                                        GstPadProbeInfo *,
                                                        gpointer);
-    void attach_probe_to_element();
+    void attach_probe_to_element(GstElement *);
+    static GstPadProbeReturn pgie_pad_buffer_probe(GstPad *, GstPadProbeInfo *,
+                                                   gpointer);
+    static GstPadProbeReturn osd_sink_pad_buffer_probe_new(GstPad *,
+                                                           GstPadProbeInfo *,
+                                                           gpointer);
 };

src/pipeline_manager.cpp

@@ -352,9 +352,9 @@ bool PipelineManager::create_pipeline_elements(int num_sources,
 
     // GstElement *nvinfer = gst_bin_get_by_name(GST_BIN(pipeline),
     // "primary-nvinference-engine");
-    nv_infer_server_manager
+    nv_infer_server_manager->attach_probe_to_element(
-        ->attach_probe_to_element();  // nvinfer Or use "nvtracker" if after
+        nv_osd_manager->nvosd);  // nvinfer Or use "nvtracker" if after
-                                      // tracker
+                                 // tracker
     // gst_object_unref(nvinfer);
 
     message_handling->create_message_handler(pipeline, g_run_forever, loop);
@@ -363,6 +363,19 @@ bool PipelineManager::create_pipeline_elements(int num_sources,
     get_fps_buffer_probe();
     get_fps_probe();
     get_fps_osd();
+
+    /* Add probe to get informed of the meta data generated, we add probe to
+     * the source pad of PGIE's next queue element, since by that time, PGIE's
+     * buffer would have had got tensor metadata. */
+
+    new_mux_str = g_getenv("USE_NEW_NVSTREAMMUX");
+    use_new_mux = !g_strcmp0(new_mux_str, "yes");
+    pgie_src_pad = gst_element_get_static_pad(
+        nv_infer_server_manager->primary_detector, "src");
+    gst_pad_add_probe(pgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER,
+                      nv_infer_server_manager->pgie_pad_buffer_probe,
+                      &use_new_mux, NULL);
+
     auto start = std::chrono::system_clock::now();
     status_playing = playing_pipeline(num_sources, url_camera);
     if (status_playing == false) {

src/pipeline_manager.hpp

@@ -63,6 +63,9 @@ class PipelineManager {
     static guint64 frame_count_osd_sink;
     static guint64 frame_count_fps_probe;
     static guint64 frame_count_buffer_probe;
+    const gchar *new_mux_str;
+    gboolean use_new_mux;
+    GstPad *pgie_src_pad = NULL;
     static std::chrono::time_point<std::chrono::steady_clock>
         last_time_osd_sink;
     static std::chrono::time_point<std::chrono::steady_clock>