haiati/FaceRecognition
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Attach objects metadata to pgie probe

Browse Source
This commit is contained in:
Barzan Hayati 2025-07-25 22:22:55 +00:00
parent 2c5f678201
commit bf0b3d4a79
6 changed files with 326 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CMakeLists.txt
View File

@ -77,6 +77,17 @@ set(SOURCES src/main.cpp src/camera_manager.cpp src/pipeline_manager.cpp src/st
src/nv_ds_logger_manager.cpp src/sink_manager.cpp src/message_handling.cpp
src/rtsp_streaming_manager.cpp src/metrics_manager.cpp src/config_manager.cpp
src/nv_infer_server_manager.cpp)
# missing initializer for member 'NvDsInferDims::d' [-Werror=missing-field-initializers] NvDsInferDims dims = {0};
set_source_files_properties(
src/nv_infer_server_manager.cpp
src/main.cpp
src/pipeline_manager.cpp
PROPERTIES COMPILE_FLAGS "-Wno-missing-field-initializers"
)
# Create the executable
add_executable(${PROJECT_NAME} ${SOURCES})

4
data/configuration.json
View File

@ -17,5 +17,7 @@
"host": "0.0.0.0"
},
"pgie_batch_size": 16,
"inferserver_pgie_config_file": "../data/inferserver/primary_detector_config.txt"
"inferserver_pgie_config_file": "../data/inferserver/primary_detector_config.txt",
"PGIE_NET_WIDTH": 640,
"PGIE_NET_HEIGHT": 640
}

284
src/nv_infer_server_manager.cpp
View File

@ -1,10 +1,33 @@
#include "nv_infer_server_manager.hpp"
#define MAX_DISPLAY_LEN 64
#define PGIE_CLASS_ID_PERSON 0
#define PGIE_DETECTED_CLASS_NUM 1
gint NvInferServerManager::frame_number = 0;
unsigned int NvInferServerManager::PGIE_NET_WIDTH = 1;
unsigned int NvInferServerManager::PGIE_NET_HEIGHT = 1;
unsigned int NvInferServerManager::MUXER_OUTPUT_WIDTH = 1;
unsigned int NvInferServerManager::MUXER_OUTPUT_HEIGHT = 1;
guint NvInferServerManager::use_device_mem = 0;
const gchar pgie_class_str[PGIE_DETECTED_CLASS_NUM][32] = {"Person"};
/* nvds_lib_major_version and nvds_lib_minor_version is the version number of
* deepstream sdk */
unsigned int nvds_lib_major_version = NVDS_VERSION_MAJOR;
unsigned int nvds_lib_minor_version = NVDS_VERSION_MINOR;
NvInferServerManager::NvInferServerManager() {
const auto &config = ConfigManager::get_instance().get_config();
pgie_batch_size = config["pgie_batch_size"];
inferserver_pgie_config_file =
config["inferserver_pgie_config_file"].get<std::string>();
PGIE_NET_WIDTH = config["PGIE_NET_WIDTH"];
PGIE_NET_HEIGHT = config["PGIE_NET_HEIGHT"];
MUXER_OUTPUT_WIDTH = config["MUXER_OUTPUT_WIDTH"];
MUXER_OUTPUT_HEIGHT = config["MUXER_OUTPUT_HEIGHT"];
}
bool NvInferServerManager::create_nv_infer_server(int num_sources) {
@ -26,8 +49,10 @@ bool NvInferServerManager::create_nv_infer_server(int num_sources) {
g_object_set(G_OBJECT(primary_detector), "batch-size", num_sources,
NULL);
}
// std::cout << "✅ Found element: " << GST_ELEMENT_NAME(primary_detector)
// << std::endl;
if (!primary_detector) {
g_printerr("Could not create primary detector. Exiting.\n");
return false;
}
return true;
}
@ -43,7 +68,6 @@ GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe(
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
if (!batch_meta) {
std::cerr << "No batch metadata found\n";
exit(-1);
return GST_PAD_PROBE_OK;
}
@ -73,19 +97,265 @@ GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe(
<< " h=" << obj_meta->rect_params.height << std::endl;
}
}
return GST_PAD_PROBE_OK;
}
// Attach probe to a pad in the pipeline
void NvInferServerManager::attach_probe_to_element() {
GstPad *sink_pad = gst_element_get_static_pad(primary_detector, "sink");
void NvInferServerManager::attach_probe_to_element(GstElement *nvosd) {
GstPad *sink_pad = gst_element_get_static_pad(nvosd, "sink");
if (!sink_pad) {
std::cerr << "Unable to get sink pad\n";
return;
}
gst_pad_add_probe(sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
osd_sink_pad_buffer_probe, NULL, NULL);
osd_sink_pad_buffer_probe_new, NULL, NULL);
gst_object_unref(sink_pad);
}
/* This is the buffer probe function that we have registered on the sink pad
* of the OSD element. All the infer elements in the pipeline shall attach
* their metadata to the GstBuffer, here we will iterate & process the metadata
* forex: class ids to strings, counting of class_id objects etc. */
GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe_new(
GstPad *pad, GstPadProbeInfo *info, gpointer u_data) {
(void)pad;
(void)u_data;
GstBuffer *buf = (GstBuffer *)info->data;
guint num_rects = 0;
guint person_count = 0;
NvDsObjectMeta *obj_meta = NULL;
NvDsMetaList *l_frame = NULL;
NvDsMetaList *l_obj = NULL;
NvDsDisplayMeta *display_meta = NULL;
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
for (l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data);
int offset = 0;
for (l_obj = frame_meta->obj_meta_list; l_obj != NULL;
l_obj = l_obj->next) {
obj_meta = (NvDsObjectMeta *)(l_obj->data);
if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) {
person_count++;
num_rects++;
std::cout << "x = " << obj_meta->rect_params.left
<< " y = " << obj_meta->rect_params.top
<< " w = " << obj_meta->rect_params.width
<< " h = " << obj_meta->rect_params.height
<< " score = " << obj_meta->confidence << std::endl;
}
}
display_meta = nvds_acquire_display_meta_from_pool(batch_meta);
NvOSD_TextParams *txt_params = &display_meta->text_params[0];
display_meta->num_labels = 1;
txt_params->display_text = (gchar *)g_malloc0(MAX_DISPLAY_LEN);
offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN,
"Person = %d ", person_count);
(void)offset;
/* Now set the offsets where the string should appear */
txt_params->x_offset = 10;
txt_params->y_offset = 12;
/* Font , font-color and font-size */
txt_params->font_params.font_name = (gchar *)"Serif";
txt_params->font_params.font_size = 10;
txt_params->font_params.font_color.red = 1.0;
txt_params->font_params.font_color.green = 1.0;
txt_params->font_params.font_color.blue = 1.0;
txt_params->font_params.font_color.alpha = 1.0;
/* Text background color */
txt_params->set_bg_clr = 1;
txt_params->text_bg_clr.red = 0.0;
txt_params->text_bg_clr.green = 0.0;
txt_params->text_bg_clr.blue = 0.0;
txt_params->text_bg_clr.alpha = 1.0;
nvds_add_display_meta_to_frame(frame_meta, display_meta);
}
g_print(
"Frame Number = %d "
"Person Count = %d\n",
frame_number, person_count);
frame_number++;
return GST_PAD_PROBE_OK;
}
/* This is the buffer probe function that we have registered on the src pad
* of the PGIE's next queue element. PGIE element in the pipeline shall attach
* its NvDsInferTensorMeta to each frame metadata on GstBuffer, here we will
* iterate & parse the tensor data to get detection bounding boxes. The result
* would be attached as object-meta(NvDsObjectMeta) into the same frame
* metadata.
*/
GstPadProbeReturn NvInferServerManager::pgie_pad_buffer_probe(
GstPad *pad, GstPadProbeInfo *info, gpointer u_data) {
(void)pad;
gboolean *use_new_mux = (gboolean *)u_data;
guint stream_width = 0, stream_height = 0;
static NvDsInferNetworkInfo networkInfo{PGIE_NET_WIDTH, PGIE_NET_HEIGHT, 3};
NvDsBatchMeta *batch_meta =
gst_buffer_get_nvds_batch_meta(GST_BUFFER(info->data));
/* Iterate each frame metadata in batch */
for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != NULL;
l_frame = l_frame->next) {
NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data;
// to solve track not showing up issue
nvds_acquire_meta_lock(batch_meta);
frame_meta->bInferDone = TRUE;
nvds_release_meta_lock(batch_meta);
if (*use_new_mux) {
stream_width = frame_meta->source_frame_width;
stream_height = frame_meta->source_frame_height;
} else {
stream_width = MUXER_OUTPUT_WIDTH;
stream_height = MUXER_OUTPUT_HEIGHT;
}
(void)stream_height;
(void)stream_width;
/* Iterate user metadata in frames to search PGIE's tensor metadata */
for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list;
l_user != NULL; l_user = l_user->next) {
NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data;
if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META)
continue;
/* convert to tensor metadata */
NvDsInferTensorMeta *meta =
(NvDsInferTensorMeta *)user_meta->user_meta_data;
for (unsigned int i = 0; i < meta->num_output_layers; i++) {
NvDsInferLayerInfo *info = &meta->output_layers_info[i];
info->buffer = meta->out_buf_ptrs_host[i];
if (use_device_mem && meta->out_buf_ptrs_dev[i]) {
cudaMemcpy(meta->out_buf_ptrs_host[i],
meta->out_buf_ptrs_dev[i],
info->inferDims.numElements * 4,
cudaMemcpyDeviceToHost);
}
}
/* Parse output tensor and fill detection results into objectList.
*/
std::vector<NvDsInferLayerInfo> outputLayersInfo(
meta->output_layers_info,
meta->output_layers_info + meta->num_output_layers);
#if NVDS_VERSION_MAJOR >= 5
if (nvds_lib_major_version >= 5) {
if (meta->network_info.width != networkInfo.width ||
meta->network_info.height != networkInfo.height ||
meta->network_info.channels != networkInfo.channels) {
g_error("failed to check pgie network info\n");
}
}
#endif
// std::cout << "frame number: " << frame_meta->frame_num
// << " frame id: " << frame_meta->source_id << std::endl;
float *outputBuffer = (float *)outputLayersInfo[0].buffer;
(void)outputBuffer;
// NvDsInferDims dims = outputLayersInfo[0].inferDims;
for (size_t jkl = 0; jkl < outputLayersInfo.size(); ++jkl) {
const NvDsInferLayerInfo &layer = outputLayersInfo[jkl];
unsigned int numDims = layer.inferDims.numDims;
unsigned int numElements = layer.inferDims.numElements;
(void)numElements;
(void)numDims;
// std::cout << "Layer " << jkl << " (" << layer.layerName <<
// "):\n"; std::cout << " Num Dims: " << numDims << "\n";
// std::cout << " Num Elements: " << numElements << "\n";
// std::cout << " Dims: [";
// for (unsigned int mno = 0; mno < numDims; ++mno) {
// std::cout << layer.inferDims.d[mno];
// // layer.inferDims.d[0] = 100;
// // layer.inferDims.d[1] = 57;
// if (mno < numDims - 1)
// std::cout << ", ";
// }
// std::cout << "]\n";
}
const NvDsInferLayerInfo &layer =
outputLayersInfo[0]; // or loop over all
uint detected_persons = 0;
float *data = static_cast<float *>(layer.buffer);
for (unsigned int jkl = 0; jkl < 100; ++jkl) {
if (data[jkl * 57 + 4] > 0.05) {
detected_persons++;
std::cout << "x = " << data[jkl * 57 + 0]
<< " y = " << data[jkl * 57 + 1]
<< " w = " << data[jkl * 57 + 2]
<< " h = " << data[jkl * 57 + 3]
<< " score = " << data[jkl * 57 + 4] << std::endl;
for (unsigned int mno = 0; mno < 5; ++mno) {
float value = data[jkl * 57 + mno];
std::cout << "data[" << jkl << "][" << mno
<< "] = " << value << std::endl;
}
}
}
for (uint index = 0; index < detected_persons; index++) {
NvDsObjectMeta *obj_meta =
nvds_acquire_obj_meta_from_pool(batch_meta);
obj_meta->unique_component_id = meta->unique_id;
obj_meta->confidence = data[index * 57 + 4];
// obj_meta->object_id = UNTRACKED_OBJECT_ID;
obj_meta->class_id = 0;
NvOSD_RectParams &rect_params = obj_meta->rect_params;
NvOSD_TextParams &text_params = obj_meta->text_params;
/* Assign bounding box coordinates. */
rect_params.left =
data[index * 57 + 0] * MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH;
rect_params.top = data[index * 57 + 1] * MUXER_OUTPUT_HEIGHT /
PGIE_NET_HEIGHT;
rect_params.width =
(data[index * 57 + 2] - data[index * 57 + 0]) *
MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH;
rect_params.height =
(data[index * 57 + 3] - data[index * 57 + 1]) *
MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT;
/* Border of width 3. */
rect_params.border_width = 3;
rect_params.has_bg_color = 0;
rect_params.border_color = NvOSD_ColorParams{1, 0, 0, 1};
/* display_text requires heap allocated memory. */
text_params.display_text = g_strdup(pgie_class_str[0]);
/* Display text above the left top corner of the object. */
text_params.x_offset = rect_params.left;
text_params.y_offset = rect_params.top - 10;
/* Set black background for the text. */
text_params.set_bg_clr = 1;
text_params.text_bg_clr = NvOSD_ColorParams{0, 0, 0, 1};
/* Font face, size and color. */
text_params.font_params.font_name = (gchar *)"Serif";
text_params.font_params.font_size = 11;
text_params.font_params.font_color =
NvOSD_ColorParams{1, 1, 1, 1};
// adding landmarks to obj_meta as user_meta
// NvDsUserMeta* um1 =
// nvds_acquire_user_meta_from_pool(batch_meta);
// um1->user_meta_data = set_metadata_ptr(&(res[i].landmark[0]),
// source_id ); //Add landmarks here um1->base_meta.meta_type =
// NVDS_USER_OBJECT_META_LANDMARKS_AND_SOURCE_ID;
// um1->base_meta.copy_func = (NvDsMetaCopyFunc)copy_user_meta;
// um1->base_meta.release_func =
// (NvDsMetaReleaseFunc)release_user_meta;
// nvds_add_user_meta_to_obj(obj_meta, um1);
nvds_add_obj_meta_to_frame(frame_meta, obj_meta, NULL);
}
}
}
use_device_mem = 1 - use_device_mem;
return GST_PAD_PROBE_OK;
}

17
src/nv_infer_server_manager.hpp
View File

@ -4,13 +4,23 @@
#include <iostream>
#include "config_manager.hpp"
#include "gstnvdsinfer.h"
#include "gstnvdsmeta.h"
#include "nvds_version.h"
#include "nvdsinfer_custom_impl.h"
class NvInferServerManager {
private:
public:
GstElement *primary_detector = NULL;
int pgie_batch_size;
static unsigned int PGIE_NET_WIDTH;
static unsigned int PGIE_NET_HEIGHT;
static unsigned int MUXER_OUTPUT_WIDTH;
static unsigned int MUXER_OUTPUT_HEIGHT;
static gint frame_number;
static guint use_device_mem;
std::string inferserver_pgie_config_file;
NvInferServerManager();
bool create_nv_infer_server(int);
@ -18,5 +28,10 @@ class NvInferServerManager {
static GstPadProbeReturn osd_sink_pad_buffer_probe(GstPad *,
GstPadProbeInfo *,
gpointer);
void attach_probe_to_element();
void attach_probe_to_element(GstElement *);
static GstPadProbeReturn pgie_pad_buffer_probe(GstPad *, GstPadProbeInfo *,
gpointer);
static GstPadProbeReturn osd_sink_pad_buffer_probe_new(GstPad *,
GstPadProbeInfo *,
gpointer);
};

19
src/pipeline_manager.cpp
View File

@ -352,9 +352,9 @@ bool PipelineManager::create_pipeline_elements(int num_sources,
// GstElement *nvinfer = gst_bin_get_by_name(GST_BIN(pipeline),
// "primary-nvinference-engine");
nv_infer_server_manager
->attach_probe_to_element(); // nvinfer Or use "nvtracker" if after
// tracker
nv_infer_server_manager->attach_probe_to_element(
nv_osd_manager->nvosd); // nvinfer Or use "nvtracker" if after
// tracker
// gst_object_unref(nvinfer);
message_handling->create_message_handler(pipeline, g_run_forever, loop);
@ -363,6 +363,19 @@ bool PipelineManager::create_pipeline_elements(int num_sources,
get_fps_buffer_probe();
get_fps_probe();
get_fps_osd();
/* Add probe to get informed of the meta data generated, we add probe to
* the source pad of PGIE's next queue element, since by that time, PGIE's
* buffer would have had got tensor metadata. */
new_mux_str = g_getenv("USE_NEW_NVSTREAMMUX");
use_new_mux = !g_strcmp0(new_mux_str, "yes");
pgie_src_pad = gst_element_get_static_pad(
nv_infer_server_manager->primary_detector, "src");
gst_pad_add_probe(pgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER,
nv_infer_server_manager->pgie_pad_buffer_probe,
&use_new_mux, NULL);
auto start = std::chrono::system_clock::now();
status_playing = playing_pipeline(num_sources, url_camera);
if (status_playing == false) {

3
src/pipeline_manager.hpp
View File

@ -63,6 +63,9 @@ class PipelineManager {
static guint64 frame_count_osd_sink;
static guint64 frame_count_fps_probe;
static guint64 frame_count_buffer_probe;
const gchar *new_mux_str;
gboolean use_new_mux;
GstPad *pgie_src_pad = NULL;
static std::chrono::time_point<std::chrono::steady_clock>
last_time_osd_sink;
static std::chrono::time_point<std::chrono::steady_clock>