From bf0b3d4a79f70bafd01ebdd3f363a92151f63cd6 Mon Sep 17 00:00:00 2001 From: Barzan Hayati Date: Fri, 25 Jul 2025 22:22:55 +0000 Subject: [PATCH] Attach objects metadata to pgie probe --- CMakeLists.txt | 11 ++ data/configuration.json | 4 +- src/nv_infer_server_manager.cpp | 284 +++++++++++++++++++++++++++++++- src/nv_infer_server_manager.hpp | 17 +- src/pipeline_manager.cpp | 19 ++- src/pipeline_manager.hpp | 3 + 6 files changed, 326 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1152bb2..c099355 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,17 @@ set(SOURCES src/main.cpp src/camera_manager.cpp src/pipeline_manager.cpp src/st src/nv_ds_logger_manager.cpp src/sink_manager.cpp src/message_handling.cpp src/rtsp_streaming_manager.cpp src/metrics_manager.cpp src/config_manager.cpp src/nv_infer_server_manager.cpp) + + +# missing initializer for member 'NvDsInferDims::d' [-Werror=missing-field-initializers] NvDsInferDims dims = {0}; +set_source_files_properties( + src/nv_infer_server_manager.cpp + src/main.cpp + src/pipeline_manager.cpp + PROPERTIES COMPILE_FLAGS "-Wno-missing-field-initializers" +) + + # Create the executable add_executable(${PROJECT_NAME} ${SOURCES}) diff --git a/data/configuration.json b/data/configuration.json index 0f4089b..da9ec8c 100644 --- a/data/configuration.json +++ b/data/configuration.json @@ -17,5 +17,7 @@ "host": "0.0.0.0" }, "pgie_batch_size": 16, - "inferserver_pgie_config_file": "../data/inferserver/primary_detector_config.txt" + "inferserver_pgie_config_file": "../data/inferserver/primary_detector_config.txt", + "PGIE_NET_WIDTH": 640, + "PGIE_NET_HEIGHT": 640 } \ No newline at end of file diff --git a/src/nv_infer_server_manager.cpp b/src/nv_infer_server_manager.cpp index 58c2f2c..df4de91 100644 --- a/src/nv_infer_server_manager.cpp +++ b/src/nv_infer_server_manager.cpp @@ -1,10 +1,33 @@ #include "nv_infer_server_manager.hpp" +#define MAX_DISPLAY_LEN 64 +#define PGIE_CLASS_ID_PERSON 0 +#define PGIE_DETECTED_CLASS_NUM 1 + +gint NvInferServerManager::frame_number = 0; +unsigned int NvInferServerManager::PGIE_NET_WIDTH = 1; +unsigned int NvInferServerManager::PGIE_NET_HEIGHT = 1; +unsigned int NvInferServerManager::MUXER_OUTPUT_WIDTH = 1; +unsigned int NvInferServerManager::MUXER_OUTPUT_HEIGHT = 1; +guint NvInferServerManager::use_device_mem = 0; + +const gchar pgie_class_str[PGIE_DETECTED_CLASS_NUM][32] = {"Person"}; + +/* nvds_lib_major_version and nvds_lib_minor_version is the version number of + * deepstream sdk */ + +unsigned int nvds_lib_major_version = NVDS_VERSION_MAJOR; +unsigned int nvds_lib_minor_version = NVDS_VERSION_MINOR; + NvInferServerManager::NvInferServerManager() { const auto &config = ConfigManager::get_instance().get_config(); pgie_batch_size = config["pgie_batch_size"]; inferserver_pgie_config_file = config["inferserver_pgie_config_file"].get(); + PGIE_NET_WIDTH = config["PGIE_NET_WIDTH"]; + PGIE_NET_HEIGHT = config["PGIE_NET_HEIGHT"]; + MUXER_OUTPUT_WIDTH = config["MUXER_OUTPUT_WIDTH"]; + MUXER_OUTPUT_HEIGHT = config["MUXER_OUTPUT_HEIGHT"]; } bool NvInferServerManager::create_nv_infer_server(int num_sources) { @@ -26,8 +49,10 @@ bool NvInferServerManager::create_nv_infer_server(int num_sources) { g_object_set(G_OBJECT(primary_detector), "batch-size", num_sources, NULL); } - // std::cout << "✅ Found element: " << GST_ELEMENT_NAME(primary_detector) - // << std::endl; + if (!primary_detector) { + g_printerr("Could not create primary detector. Exiting.\n"); + return false; + } return true; } @@ -43,7 +68,6 @@ GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe( NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); if (!batch_meta) { std::cerr << "No batch metadata found\n"; - exit(-1); return GST_PAD_PROBE_OK; } @@ -73,19 +97,265 @@ GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe( << " h=" << obj_meta->rect_params.height << std::endl; } } - return GST_PAD_PROBE_OK; } // Attach probe to a pad in the pipeline -void NvInferServerManager::attach_probe_to_element() { - GstPad *sink_pad = gst_element_get_static_pad(primary_detector, "sink"); +void NvInferServerManager::attach_probe_to_element(GstElement *nvosd) { + GstPad *sink_pad = gst_element_get_static_pad(nvosd, "sink"); if (!sink_pad) { std::cerr << "Unable to get sink pad\n"; return; } gst_pad_add_probe(sink_pad, GST_PAD_PROBE_TYPE_BUFFER, - osd_sink_pad_buffer_probe, NULL, NULL); + osd_sink_pad_buffer_probe_new, NULL, NULL); gst_object_unref(sink_pad); +} + +/* This is the buffer probe function that we have registered on the sink pad + * of the OSD element. All the infer elements in the pipeline shall attach + * their metadata to the GstBuffer, here we will iterate & process the metadata + * forex: class ids to strings, counting of class_id objects etc. */ +GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe_new( + GstPad *pad, GstPadProbeInfo *info, gpointer u_data) { + (void)pad; + (void)u_data; + GstBuffer *buf = (GstBuffer *)info->data; + guint num_rects = 0; + guint person_count = 0; + NvDsObjectMeta *obj_meta = NULL; + NvDsMetaList *l_frame = NULL; + NvDsMetaList *l_obj = NULL; + NvDsDisplayMeta *display_meta = NULL; + + NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + + for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); + int offset = 0; + for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; + l_obj = l_obj->next) { + obj_meta = (NvDsObjectMeta *)(l_obj->data); + if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { + person_count++; + num_rects++; + std::cout << "x = " << obj_meta->rect_params.left + << " y = " << obj_meta->rect_params.top + << " w = " << obj_meta->rect_params.width + << " h = " << obj_meta->rect_params.height + << " score = " << obj_meta->confidence << std::endl; + } + } + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = (gchar *)g_malloc0(MAX_DISPLAY_LEN); + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, + "Person = %d ", person_count); + (void)offset; + + /* Now set the offsets where the string should appear */ + txt_params->x_offset = 10; + txt_params->y_offset = 12; + + /* Font , font-color and font-size */ + txt_params->font_params.font_name = (gchar *)"Serif"; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + /* Text background color */ + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); + } + g_print( + "Frame Number = %d " + "Person Count = %d\n", + frame_number, person_count); + + frame_number++; + return GST_PAD_PROBE_OK; +} + +/* This is the buffer probe function that we have registered on the src pad + * of the PGIE's next queue element. PGIE element in the pipeline shall attach + * its NvDsInferTensorMeta to each frame metadata on GstBuffer, here we will + * iterate & parse the tensor data to get detection bounding boxes. The result + * would be attached as object-meta(NvDsObjectMeta) into the same frame + * metadata. + */ +GstPadProbeReturn NvInferServerManager::pgie_pad_buffer_probe( + GstPad *pad, GstPadProbeInfo *info, gpointer u_data) { + (void)pad; + gboolean *use_new_mux = (gboolean *)u_data; + guint stream_width = 0, stream_height = 0; + + static NvDsInferNetworkInfo networkInfo{PGIE_NET_WIDTH, PGIE_NET_HEIGHT, 3}; + NvDsBatchMeta *batch_meta = + gst_buffer_get_nvds_batch_meta(GST_BUFFER(info->data)); + + /* Iterate each frame metadata in batch */ + for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != NULL; + l_frame = l_frame->next) { + NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data; + // to solve track not showing up issue + nvds_acquire_meta_lock(batch_meta); + frame_meta->bInferDone = TRUE; + nvds_release_meta_lock(batch_meta); + if (*use_new_mux) { + stream_width = frame_meta->source_frame_width; + stream_height = frame_meta->source_frame_height; + } else { + stream_width = MUXER_OUTPUT_WIDTH; + stream_height = MUXER_OUTPUT_HEIGHT; + } + (void)stream_height; + (void)stream_width; + + /* Iterate user metadata in frames to search PGIE's tensor metadata */ + for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list; + l_user != NULL; l_user = l_user->next) { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) + continue; + + /* convert to tensor metadata */ + NvDsInferTensorMeta *meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + for (unsigned int i = 0; i < meta->num_output_layers; i++) { + NvDsInferLayerInfo *info = &meta->output_layers_info[i]; + info->buffer = meta->out_buf_ptrs_host[i]; + if (use_device_mem && meta->out_buf_ptrs_dev[i]) { + cudaMemcpy(meta->out_buf_ptrs_host[i], + meta->out_buf_ptrs_dev[i], + info->inferDims.numElements * 4, + cudaMemcpyDeviceToHost); + } + } + /* Parse output tensor and fill detection results into objectList. + */ + std::vector outputLayersInfo( + meta->output_layers_info, + meta->output_layers_info + meta->num_output_layers); +#if NVDS_VERSION_MAJOR >= 5 + if (nvds_lib_major_version >= 5) { + if (meta->network_info.width != networkInfo.width || + meta->network_info.height != networkInfo.height || + meta->network_info.channels != networkInfo.channels) { + g_error("failed to check pgie network info\n"); + } + } +#endif + + // std::cout << "frame number: " << frame_meta->frame_num + // << " frame id: " << frame_meta->source_id << std::endl; + + float *outputBuffer = (float *)outputLayersInfo[0].buffer; + (void)outputBuffer; + // NvDsInferDims dims = outputLayersInfo[0].inferDims; + + for (size_t jkl = 0; jkl < outputLayersInfo.size(); ++jkl) { + const NvDsInferLayerInfo &layer = outputLayersInfo[jkl]; + + unsigned int numDims = layer.inferDims.numDims; + unsigned int numElements = layer.inferDims.numElements; + (void)numElements; + (void)numDims; + + // std::cout << "Layer " << jkl << " (" << layer.layerName << + // "):\n"; std::cout << " Num Dims: " << numDims << "\n"; + // std::cout << " Num Elements: " << numElements << "\n"; + // std::cout << " Dims: ["; + // for (unsigned int mno = 0; mno < numDims; ++mno) { + // std::cout << layer.inferDims.d[mno]; + // // layer.inferDims.d[0] = 100; + // // layer.inferDims.d[1] = 57; + // if (mno < numDims - 1) + // std::cout << ", "; + // } + // std::cout << "]\n"; + } + const NvDsInferLayerInfo &layer = + outputLayersInfo[0]; // or loop over all + + uint detected_persons = 0; + float *data = static_cast(layer.buffer); + for (unsigned int jkl = 0; jkl < 100; ++jkl) { + if (data[jkl * 57 + 4] > 0.05) { + detected_persons++; + std::cout << "x = " << data[jkl * 57 + 0] + << " y = " << data[jkl * 57 + 1] + << " w = " << data[jkl * 57 + 2] + << " h = " << data[jkl * 57 + 3] + << " score = " << data[jkl * 57 + 4] << std::endl; + for (unsigned int mno = 0; mno < 5; ++mno) { + float value = data[jkl * 57 + mno]; + std::cout << "data[" << jkl << "][" << mno + << "] = " << value << std::endl; + } + } + } + + for (uint index = 0; index < detected_persons; index++) { + NvDsObjectMeta *obj_meta = + nvds_acquire_obj_meta_from_pool(batch_meta); + obj_meta->unique_component_id = meta->unique_id; + obj_meta->confidence = data[index * 57 + 4]; + // obj_meta->object_id = UNTRACKED_OBJECT_ID; + obj_meta->class_id = 0; + NvOSD_RectParams &rect_params = obj_meta->rect_params; + NvOSD_TextParams &text_params = obj_meta->text_params; + /* Assign bounding box coordinates. */ + rect_params.left = + data[index * 57 + 0] * MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH; + rect_params.top = data[index * 57 + 1] * MUXER_OUTPUT_HEIGHT / + PGIE_NET_HEIGHT; + rect_params.width = + (data[index * 57 + 2] - data[index * 57 + 0]) * + MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH; + rect_params.height = + (data[index * 57 + 3] - data[index * 57 + 1]) * + MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT; + /* Border of width 3. */ + rect_params.border_width = 3; + rect_params.has_bg_color = 0; + rect_params.border_color = NvOSD_ColorParams{1, 0, 0, 1}; + /* display_text requires heap allocated memory. */ + text_params.display_text = g_strdup(pgie_class_str[0]); + /* Display text above the left top corner of the object. */ + text_params.x_offset = rect_params.left; + text_params.y_offset = rect_params.top - 10; + /* Set black background for the text. */ + text_params.set_bg_clr = 1; + text_params.text_bg_clr = NvOSD_ColorParams{0, 0, 0, 1}; + /* Font face, size and color. */ + text_params.font_params.font_name = (gchar *)"Serif"; + text_params.font_params.font_size = 11; + text_params.font_params.font_color = + NvOSD_ColorParams{1, 1, 1, 1}; + // adding landmarks to obj_meta as user_meta + // NvDsUserMeta* um1 = + // nvds_acquire_user_meta_from_pool(batch_meta); + // um1->user_meta_data = set_metadata_ptr(&(res[i].landmark[0]), + // source_id ); //Add landmarks here um1->base_meta.meta_type = + // NVDS_USER_OBJECT_META_LANDMARKS_AND_SOURCE_ID; + // um1->base_meta.copy_func = (NvDsMetaCopyFunc)copy_user_meta; + // um1->base_meta.release_func = + // (NvDsMetaReleaseFunc)release_user_meta; + // nvds_add_user_meta_to_obj(obj_meta, um1); + nvds_add_obj_meta_to_frame(frame_meta, obj_meta, NULL); + } + } + } + use_device_mem = 1 - use_device_mem; + return GST_PAD_PROBE_OK; } \ No newline at end of file diff --git a/src/nv_infer_server_manager.hpp b/src/nv_infer_server_manager.hpp index ee3bc71..0426302 100644 --- a/src/nv_infer_server_manager.hpp +++ b/src/nv_infer_server_manager.hpp @@ -4,13 +4,23 @@ #include #include "config_manager.hpp" +#include "gstnvdsinfer.h" #include "gstnvdsmeta.h" +#include "nvds_version.h" +#include "nvdsinfer_custom_impl.h" class NvInferServerManager { private: public: GstElement *primary_detector = NULL; int pgie_batch_size; + + static unsigned int PGIE_NET_WIDTH; + static unsigned int PGIE_NET_HEIGHT; + static unsigned int MUXER_OUTPUT_WIDTH; + static unsigned int MUXER_OUTPUT_HEIGHT; + static gint frame_number; + static guint use_device_mem; std::string inferserver_pgie_config_file; NvInferServerManager(); bool create_nv_infer_server(int); @@ -18,5 +28,10 @@ class NvInferServerManager { static GstPadProbeReturn osd_sink_pad_buffer_probe(GstPad *, GstPadProbeInfo *, gpointer); - void attach_probe_to_element(); + void attach_probe_to_element(GstElement *); + static GstPadProbeReturn pgie_pad_buffer_probe(GstPad *, GstPadProbeInfo *, + gpointer); + static GstPadProbeReturn osd_sink_pad_buffer_probe_new(GstPad *, + GstPadProbeInfo *, + gpointer); }; \ No newline at end of file diff --git a/src/pipeline_manager.cpp b/src/pipeline_manager.cpp index 4c981db..f3e8c0f 100644 --- a/src/pipeline_manager.cpp +++ b/src/pipeline_manager.cpp @@ -352,9 +352,9 @@ bool PipelineManager::create_pipeline_elements(int num_sources, // GstElement *nvinfer = gst_bin_get_by_name(GST_BIN(pipeline), // "primary-nvinference-engine"); - nv_infer_server_manager - ->attach_probe_to_element(); // nvinfer Or use "nvtracker" if after - // tracker + nv_infer_server_manager->attach_probe_to_element( + nv_osd_manager->nvosd); // nvinfer Or use "nvtracker" if after + // tracker // gst_object_unref(nvinfer); message_handling->create_message_handler(pipeline, g_run_forever, loop); @@ -363,6 +363,19 @@ bool PipelineManager::create_pipeline_elements(int num_sources, get_fps_buffer_probe(); get_fps_probe(); get_fps_osd(); + + /* Add probe to get informed of the meta data generated, we add probe to + * the source pad of PGIE's next queue element, since by that time, PGIE's + * buffer would have had got tensor metadata. */ + + new_mux_str = g_getenv("USE_NEW_NVSTREAMMUX"); + use_new_mux = !g_strcmp0(new_mux_str, "yes"); + pgie_src_pad = gst_element_get_static_pad( + nv_infer_server_manager->primary_detector, "src"); + gst_pad_add_probe(pgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + nv_infer_server_manager->pgie_pad_buffer_probe, + &use_new_mux, NULL); + auto start = std::chrono::system_clock::now(); status_playing = playing_pipeline(num_sources, url_camera); if (status_playing == false) { diff --git a/src/pipeline_manager.hpp b/src/pipeline_manager.hpp index 8272c72..9bf2ab2 100644 --- a/src/pipeline_manager.hpp +++ b/src/pipeline_manager.hpp @@ -63,6 +63,9 @@ class PipelineManager { static guint64 frame_count_osd_sink; static guint64 frame_count_fps_probe; static guint64 frame_count_buffer_probe; + const gchar *new_mux_str; + gboolean use_new_mux; + GstPad *pgie_src_pad = NULL; static std::chrono::time_point last_time_osd_sink; static std::chrono::time_point