diff --git a/.vscode/settings.json b/.vscode/settings.json index d0275ab..b591c07 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -68,6 +68,13 @@ "thread": "cpp", "cinttypes": "cpp", "typeinfo": "cpp", - "valarray": "cpp" + "valarray": "cpp", + "list": "cpp", + "__bit_reference": "cpp", + "bitset": "cpp", + "charconv": "cpp", + "complex": "cpp", + "typeindex": "cpp", + "variant": "cpp" } } \ No newline at end of file diff --git a/src/custom_gstnvdsinfer.hpp b/src/custom_gstnvdsinfer.hpp new file mode 100644 index 0000000..7bfca40 --- /dev/null +++ b/src/custom_gstnvdsinfer.hpp @@ -0,0 +1,9 @@ +#ifndef CUSTOM_GSTNVDSINFER_HPP +#define CUSTOM_GSTNVDSINFER_HPP + +#include "gstnvdsinfer.h" +#include "nvbufsurface.h" +#include "nvdsinfer_custom_impl.h" +#include "nvds_obj_encode.h" + +#endif \ No newline at end of file diff --git a/src/face_nv_infer_server_manager.cpp b/src/face_nv_infer_server_manager.cpp index c517dc9..1b3c3d0 100644 --- a/src/face_nv_infer_server_manager.cpp +++ b/src/face_nv_infer_server_manager.cpp @@ -8,20 +8,28 @@ #define MAX_DISPLAY_LEN 64 #define PGIE_CLASS_ID_PERSON 0 #define PGIE_DETECTED_CLASS_NUM 1 +#define SGIE_DETECTED_CLASS_NUM 1 +#define IMPRECISE_FACE_COMPONENT_ID 2 +#define FINAL_FACE_COMPONENT_ID 3 gint FaceNvInferServerManager::frame_number = 0; unsigned int FaceNvInferServerManager::FACE_NET_WIDTH = 1; unsigned int FaceNvInferServerManager::FACE_NET_HEIGHT = 1; unsigned int FaceNvInferServerManager::MUXER_OUTPUT_WIDTH = 1; unsigned int FaceNvInferServerManager::MUXER_OUTPUT_HEIGHT = 1; -guint FaceNvInferServerManager::use_device_mem = 0; +guint FaceNvInferServerManager::use_device_mem = 1; float FaceNvInferServerManager::threshold_face_detection = 0; unsigned int FaceNvInferServerManager::nvds_lib_major_version = NVDS_VERSION_MAJOR; unsigned int FaceNvInferServerManager::nvds_lib_minor_version = NVDS_VERSION_MINOR; -const gchar pgie_class_str[PGIE_DETECTED_CLASS_NUM][32] = {"Person"}; +const gchar pgie_class_str[PGIE_DETECTED_CLASS_NUM][32] = {"Person_SGIE"}; +const gchar sgie_class_str[SGIE_DETECTED_CLASS_NUM][32] = {"PreciseFace_SGIE"}; +int high_confidence_faces = 0; + +std::vector + FaceNvInferServerManager::face_body_list; /* nvds_lib_major_version and nvds_lib_minor_version is the version number of * deepstream sdk */ @@ -31,8 +39,10 @@ FaceNvInferServerManager::FaceNvInferServerManager() { face_batch_size = config["pgie_batch_size"]; inferserver_face_config_file = config["inferserver_face_config_file"].get(); - FACE_NET_WIDTH = config["PGIE_NET_WIDTH"]; - FACE_NET_HEIGHT = config["PGIE_NET_HEIGHT"]; + FACE_NET_WIDTH = config["FACE_NET_WIDTH"]; + FACE_NET_HEIGHT = config["FACE_NET_HEIGHT"]; + // FACE_NET_WIDTH = config["PGIE_NET_WIDTH"]; + // FACE_NET_HEIGHT = config["PGIE_NET_HEIGHT"]; MUXER_OUTPUT_WIDTH = config["MUXER_OUTPUT_WIDTH"]; MUXER_OUTPUT_HEIGHT = config["MUXER_OUTPUT_HEIGHT"]; threshold_face_detection = config["threshold_body_detection"]; @@ -108,6 +118,98 @@ bool FaceNvInferServerManager::create_face_nv_infer_server(int num_sources) { // return GST_PAD_PROBE_OK; // } +// Probe function to inspect NvDsObjectMeta + +// GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe( +// GstPad *pad, GstPadProbeInfo *info, gpointer user_data) { +// (void)pad; +// (void)user_data; +// GstBuffer *buf = (GstBuffer *)info->data; + +// // Retrieve batch metadata from buffer +// NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); +// if (!batch_meta) { +// std::cerr << "No batch metadata found\n"; +// return GST_PAD_PROBE_OK; +// } + +// // probe sees the frame metadata (NvDsFrameMeta) — +// // but no object metadata (NvDsObjectMeta) was attached to that frame. + +// for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != +// NULL; +// l_frame = l_frame->next) { +// NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); +// // std::cout << "Frame number: " << frame_meta->frame_num << +// std::endl; +// // if (frame_meta->obj_meta_list == NULL) { +// // std::cout << " ⚠️ No object metadata for this frame.\n"; +// // } + +// for (NvDsMetaList *l_obj = frame_meta->obj_meta_list; l_obj != NULL; +// l_obj = l_obj->next) { +// NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)(l_obj->data); + +// std::cout << " Object ID: " << obj_meta->object_id << std::endl; +// std::cout << " Class ID: " << obj_meta->class_id << std::endl; +// std::cout << " Label: " +// << (obj_meta->obj_label ? obj_meta->obj_label : "N/A") +// << std::endl; +// std::cout << " BBox: x=" << obj_meta->rect_params.left +// << " y=" << obj_meta->rect_params.top +// << " w=" << obj_meta->rect_params.width +// << " h=" << obj_meta->rect_params.height << std::endl; +// } +// } +// return GST_PAD_PROBE_OK; +// } + +// Probe function to inspect NvDsObjectMeta + +// GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe( +// GstPad *pad, GstPadProbeInfo *info, gpointer user_data) { +// (void)pad; +// (void)user_data; +// GstBuffer *buf = (GstBuffer *)info->data; + +// // Retrieve batch metadata from buffer +// NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); +// if (!batch_meta) { +// std::cerr << "No batch metadata found\n"; +// return GST_PAD_PROBE_OK; +// } + +// // probe sees the frame metadata (NvDsFrameMeta) — +// // but no object metadata (NvDsObjectMeta) was attached to that frame. + +// for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != +// NULL; +// l_frame = l_frame->next) { +// NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); +// // std::cout << "Frame number: " << frame_meta->frame_num << +// std::endl; +// // if (frame_meta->obj_meta_list == NULL) { +// // std::cout << " ⚠️ No object metadata for this frame.\n"; +// // } + +// for (NvDsMetaList *l_obj = frame_meta->obj_meta_list; l_obj != NULL; +// l_obj = l_obj->next) { +// NvDsObjectMeta *obj_meta = (NvDsObjectMeta *)(l_obj->data); + +// std::cout << " Object ID: " << obj_meta->object_id << std::endl; +// std::cout << " Class ID: " << obj_meta->class_id << std::endl; +// std::cout << " Label: " +// << (obj_meta->obj_label ? obj_meta->obj_label : "N/A") +// << std::endl; +// std::cout << " BBox: x=" << obj_meta->rect_params.left +// << " y=" << obj_meta->rect_params.top +// << " w=" << obj_meta->rect_params.width +// << " h=" << obj_meta->rect_params.height << std::endl; +// } +// } +// return GST_PAD_PROBE_OK; +// } + // // Attach probe to a pad in the pipeline // void NvInferServerManager::attach_probe_to_element(GstElement *nvosd) { // GstPad *sink_pad = gst_element_get_static_pad(nvosd, "src"); @@ -199,6 +301,162 @@ bool FaceNvInferServerManager::create_face_nv_infer_server(int num_sources) { // return GST_PAD_PROBE_OK; // } +// /* This is the buffer probe function that we have registered on the sink pad +// * of the OSD element. All the infer elements in the pipeline shall attach +// * their metadata to the GstBuffer, here we will iterate & process the +// metadata +// * forex: class ids to strings, counting of class_id objects etc. */ +// GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe_new( +// GstPad *pad, GstPadProbeInfo *info, gpointer u_data) { +// (void)pad; +// (void)u_data; +// GstBuffer *buf = (GstBuffer *)info->data; +// guint num_rects = 0; +// guint person_count = 0; +// NvDsObjectMeta *obj_meta = NULL; +// NvDsMetaList *l_frame = NULL; +// NvDsMetaList *l_obj = NULL; +// NvDsDisplayMeta *display_meta = NULL; + +// NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + +// for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; +// l_frame = l_frame->next) { +// NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); +// int offset = 0; +// for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; +// l_obj = l_obj->next) { +// obj_meta = (NvDsObjectMeta *)(l_obj->data); +// if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { +// person_count++; +// num_rects++; +// std::cout << "In OSD sink " +// << "x = " << obj_meta->rect_params.left +// << " y = " << obj_meta->rect_params.top +// << " w = " << obj_meta->rect_params.width +// << " h = " << obj_meta->rect_params.height +// << " score = " << obj_meta->confidence +// << " Object ID: " << obj_meta->object_id +// << std::endl; +// } +// } +// display_meta = nvds_acquire_display_meta_from_pool(batch_meta); +// NvOSD_TextParams *txt_params = &display_meta->text_params[0]; +// display_meta->num_labels = 1; +// txt_params->display_text = (gchar *)g_malloc0(MAX_DISPLAY_LEN); +// offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, +// "Person = %d ", person_count); +// (void)offset; + +// /* Now set the offsets where the string should appear */ +// txt_params->x_offset = 10; +// txt_params->y_offset = 12; + +// /* Font , font-color and font-size */ +// txt_params->font_params.font_name = (gchar *)"Serif"; +// txt_params->font_params.font_size = 10; +// txt_params->font_params.font_color.red = 1.0; +// txt_params->font_params.font_color.green = 1.0; +// txt_params->font_params.font_color.blue = 1.0; +// txt_params->font_params.font_color.alpha = 1.0; + +// /* Text background color */ +// txt_params->set_bg_clr = 1; +// txt_params->text_bg_clr.red = 0.0; +// txt_params->text_bg_clr.green = 0.0; +// txt_params->text_bg_clr.blue = 0.0; +// txt_params->text_bg_clr.alpha = 1.0; + +// nvds_add_display_meta_to_frame(frame_meta, display_meta); +// } +// g_print( +// "In OSD sink " +// "Frame Number = %d " +// "Person Count = %d\n", +// frame_number, person_count); + +// frame_number++; +// return GST_PAD_PROBE_OK; +// } + +// /* This is the buffer probe function that we have registered on the sink pad +// * of the OSD element. All the infer elements in the pipeline shall attach +// * their metadata to the GstBuffer, here we will iterate & process the +// metadata +// * forex: class ids to strings, counting of class_id objects etc. */ +// GstPadProbeReturn NvInferServerManager::osd_sink_pad_buffer_probe_new( +// GstPad *pad, GstPadProbeInfo *info, gpointer u_data) { +// (void)pad; +// (void)u_data; +// GstBuffer *buf = (GstBuffer *)info->data; +// guint num_rects = 0; +// guint person_count = 0; +// NvDsObjectMeta *obj_meta = NULL; +// NvDsMetaList *l_frame = NULL; +// NvDsMetaList *l_obj = NULL; +// NvDsDisplayMeta *display_meta = NULL; + +// NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); + +// for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; +// l_frame = l_frame->next) { +// NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)(l_frame->data); +// int offset = 0; +// for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; +// l_obj = l_obj->next) { +// obj_meta = (NvDsObjectMeta *)(l_obj->data); +// if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { +// person_count++; +// num_rects++; +// std::cout << "In OSD sink " +// << "x = " << obj_meta->rect_params.left +// << " y = " << obj_meta->rect_params.top +// << " w = " << obj_meta->rect_params.width +// << " h = " << obj_meta->rect_params.height +// << " score = " << obj_meta->confidence +// << " Object ID: " << obj_meta->object_id +// << std::endl; +// } +// } +// display_meta = nvds_acquire_display_meta_from_pool(batch_meta); +// NvOSD_TextParams *txt_params = &display_meta->text_params[0]; +// display_meta->num_labels = 1; +// txt_params->display_text = (gchar *)g_malloc0(MAX_DISPLAY_LEN); +// offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, +// "Person = %d ", person_count); +// (void)offset; + +// /* Now set the offsets where the string should appear */ +// txt_params->x_offset = 10; +// txt_params->y_offset = 12; + +// /* Font , font-color and font-size */ +// txt_params->font_params.font_name = (gchar *)"Serif"; +// txt_params->font_params.font_size = 10; +// txt_params->font_params.font_color.red = 1.0; +// txt_params->font_params.font_color.green = 1.0; +// txt_params->font_params.font_color.blue = 1.0; +// txt_params->font_params.font_color.alpha = 1.0; + +// /* Text background color */ +// txt_params->set_bg_clr = 1; +// txt_params->text_bg_clr.red = 0.0; +// txt_params->text_bg_clr.green = 0.0; +// txt_params->text_bg_clr.blue = 0.0; +// txt_params->text_bg_clr.alpha = 1.0; + +// nvds_add_display_meta_to_frame(frame_meta, display_meta); +// } +// g_print( +// "In OSD sink " +// "Frame Number = %d " +// "Person Count = %d\n", +// frame_number, person_count); + +// frame_number++; +// return GST_PAD_PROBE_OK; +// } + /* This is the buffer probe function that we have registered on the src pad * of the PGIE's next queue element. PGIE element in the pipeline shall attach * its NvDsInferTensorMeta to each frame metadata on GstBuffer, here we will @@ -393,35 +651,942 @@ bool FaceNvInferServerManager::create_face_nv_infer_server(int num_sources) { // return GST_PAD_PROBE_OK; // } -// // add custom infromation to metadata by: set_metadata_ptr, copy_user_meta, -// // release_user_meta -// void *FaceNvInferServerManager::set_metadata_ptr(float *arr) { -// int i = 0; -// float *user_metadata = (float *)g_malloc0(57 * sizeof(float)); +/* This is the buffer probe function that we have registered on the src pad + * of the PGIE's next queue element. PGIE element in the pipeline shall attach + * its NvDsInferTensorMeta to each frame metadata on GstBuffer, here we will + * iterate & parse the tensor data to get detection bounding boxes. The result + * would be attached as object-meta(NvDsObjectMeta) into the same frame + * metadata. + */ +// GstPadProbeReturn FaceNvInferServerManager::pgie_pad_buffer_probe( +// GstPad *pad, GstPadProbeInfo *info, gpointer u_data) { +// (void)pad; +// gboolean *use_new_mux = (gboolean *)u_data; +// guint stream_width = 0, stream_height = 0; -// for (i = 0; i < 57; i++) { -// user_metadata[i] = arr[i]; +// static NvDsInferNetworkInfo networkInfo{PGIE_NET_WIDTH, PGIE_NET_HEIGHT, +// 3}; NvDsBatchMeta *batch_meta = +// gst_buffer_get_nvds_batch_meta(GST_BUFFER(info->data)); + +// /* Iterate each frame metadata in batch */ +// for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != +// NULL; +// l_frame = l_frame->next) { +// NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data; +// // to solve track not showing up issue +// nvds_acquire_meta_lock(batch_meta); +// frame_meta->bInferDone = TRUE; +// nvds_release_meta_lock(batch_meta); +// if (*use_new_mux) { +// stream_width = frame_meta->source_frame_width; +// stream_height = frame_meta->source_frame_height; +// } else { +// stream_width = MUXER_OUTPUT_WIDTH; +// stream_height = MUXER_OUTPUT_HEIGHT; +// } +// (void)stream_height; +// (void)stream_width; + +// // float source_id = (float)frame_meta->source_id; +// /* Iterate user metadata in frames to search PGIE's tensor metadata +// */ for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list; +// l_user != NULL; l_user = l_user->next) { +// NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; +// if (user_meta->base_meta.meta_type != +// NVDSINFER_TENSOR_OUTPUT_META) +// continue; + +// /* convert to tensor metadata */ +// NvDsInferTensorMeta *meta = +// (NvDsInferTensorMeta *)user_meta->user_meta_data; +// for (unsigned int i = 0; i < meta->num_output_layers; i++) { +// NvDsInferLayerInfo *info = &meta->output_layers_info[i]; +// info->buffer = meta->out_buf_ptrs_host[i]; +// if (use_device_mem && meta->out_buf_ptrs_dev[i]) { +// cudaMemcpy(meta->out_buf_ptrs_host[i], +// meta->out_buf_ptrs_dev[i], +// info->inferDims.numElements * 4, +// cudaMemcpyDeviceToHost); +// } +// } +// /* Parse output tensor and fill detection results into +// objectList. +// */ +// std::vector outputLayersInfo( +// meta->output_layers_info, +// meta->output_layers_info + meta->num_output_layers); +// #if NVDS_VERSION_MAJOR >= 5 +// if (nvds_lib_major_version >= 5) { +// if (meta->network_info.width != networkInfo.width || +// meta->network_info.height != networkInfo.height || +// meta->network_info.channels != networkInfo.channels) { +// g_error("failed to check pgie network info\n"); +// } +// } +// #endif + +// // std::cout << "frame number: " << frame_meta->frame_num +// // << " frame id: " << frame_meta->source_id << +// std::endl; + +// float *outputBuffer = (float *)outputLayersInfo[0].buffer; +// (void)outputBuffer; +// // NvDsInferDims dims = outputLayersInfo[0].inferDims; + +// for (size_t jkl = 0; jkl < outputLayersInfo.size(); jkl++) { +// const NvDsInferLayerInfo &layer = outputLayersInfo[jkl]; + +// unsigned int numDims = layer.inferDims.numDims; +// unsigned int numElements = layer.inferDims.numElements; +// (void)numElements; +// (void)numDims; + +// // std::cout << "Layer " << jkl << " (" << layer.layerName << +// // "):\n"; std::cout << " Num Dims: " << numDims << "\n"; +// // std::cout << " Num Elements: " << numElements << "\n"; +// // std::cout << " Dims: ["; +// // for (unsigned int mno = 0; mno < numDims; ++mno) { +// // std::cout << layer.inferDims.d[mno]; +// // // layer.inferDims.d[0] = 100; +// // // layer.inferDims.d[1] = 57; +// // if (mno < numDims - 1) +// // std::cout << ", "; +// // } +// // std::cout << "]\n"; +// } +// const NvDsInferLayerInfo &layer = +// outputLayersInfo[0]; // or loop over all + +// uint detected_persons = 0; +// float *data = static_cast(layer.buffer); +// for (unsigned int jkl = 0; jkl < 100; +// jkl += 4) { // 100 persons for each frame +// if (data[jkl * 57 + 4] > threshold_body_detection) { +// detected_persons++; +// // std::cout +// // << "nvinferserver first for x = " << data[jkl * 57 +// + +// // 0] +// // << " y = " << data[jkl * 57 + 1] +// // << " w = " << data[jkl * 57 + 2] +// // << " h = " << data[jkl * 57 + 3] +// // << " score = " << data[jkl * 57 + 4] << std::endl; +// for (unsigned int mno = 0; mno < 57; ++mno) { +// float value = data[jkl * 57 + mno]; +// (void)value; +// // std::cout << "data[" << jkl << "][" << mno +// // << "] = " << value << std::endl; +// } +// } +// } + +// for (uint index = 0; index < detected_persons; index++) { +// NvDsObjectMeta *obj_meta = +// nvds_acquire_obj_meta_from_pool(batch_meta); +// obj_meta->unique_component_id = meta->unique_id; +// obj_meta->confidence = data[index * 57 + 4]; +// // obj_meta->object_id = UNTRACKED_OBJECT_ID; +// obj_meta->class_id = 0; +// NvOSD_RectParams &rect_params = obj_meta->rect_params; +// NvOSD_TextParams &text_params = obj_meta->text_params; +// /* Assign bounding box coordinates. */ +// rect_params.left = int(data[index * 57 + 0] * +// MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH); +// rect_params.top = int(data[index * 57 + 1] * +// MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT); +// rect_params.width = +// int((data[index * 57 + 2] - data[index * 57 + 0]) * +// MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH); +// rect_params.height = +// int((data[index * 57 + 3] - data[index * 57 + 1]) * +// MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT); + +// // std::cout << "nvinferserver second for x = " << +// // rect_params.left +// // << " y = " << rect_params.top +// // << " w = " << rect_params.width +// // << " h = " << rect_params.height +// // << " score = " << obj_meta->confidence << +// // std::endl; + +// /* Border of width 3. */ +// rect_params.border_width = 3; +// rect_params.has_bg_color = 0; +// rect_params.border_color = NvOSD_ColorParams{1, 0, 0, 1}; +// /* display_text requires heap allocated memory. */ +// text_params.display_text = g_strdup(pgie_class_str[0]); +// /* Display text above the left top corner of the object. */ +// text_params.x_offset = rect_params.left; +// text_params.y_offset = rect_params.top - 10; +// /* Set black background for the text. */ +// text_params.set_bg_clr = 1; +// text_params.text_bg_clr = NvOSD_ColorParams{0, 0, 0, 1}; +// /* Font face, size and color. */ +// text_params.font_params.font_name = (gchar *)"Serif"; +// text_params.font_params.font_size = 11; +// text_params.font_params.font_color = +// NvOSD_ColorParams#include +// "face_nv_infer_server_manager.hpp" + +/* This is the buffer probe function that we have registered on the src pad + * of the PGIE's next queue element. PGIE element in the pipeline shall attach + * its NvDsInferTensorMeta to each frame metadata on GstBuffer, here we will + * iterate & parse the tensor data to get detection bounding boxes. The result + * would be attached as object-meta(NvDsObjectMeta) into the same frame + * metadata. + */ +// GstPadProbeReturn FaceNvInferServerManager::pgie_pad_buffer_probe( +// GstPad *pad, GstPadProbeInfo *info, gpointer u_data) { +// (void)pad; +// gboolean *use_new_mux = (gboolean *)u_data; +// guint stream_width = 0, stream_height = 0; + +// static NvDsInferNetworkInfo networkInfo{PGIE_NET_WIDTH, PGIE_NET_HEIGHT, +// 3}; NvDsBatchMeta *batch_meta = +// gst_buffer_get_nvds_batch_meta(GST_BUFFER(info->data)); + +// /* Iterate each frame metadata in batch */ +// for (NvDsMetaList *l_frame = batch_meta->frame_meta_list; l_frame != +// NULL; +// l_frame = l_frame->next) { +// NvDsFrameMeta *frame_meta = (NvDsFrameMeta *)l_frame->data; +// // to solve track not showing up issue +// nvds_acquire_meta_lock(batch_meta); +// frame_meta->bInferDone = TRUE; +// nvds_release_meta_lock(batch_meta); +// if (*use_new_mux) { +// stream_width = frame_meta->source_frame_width; +// stream_height = frame_meta->source_frame_height; +// } else { +// stream_width = MUXER_OUTPUT_WIDTH; +// stream_height = MUXER_OUTPUT_HEIGHT; +// } +// (void)stream_height; +// (void)stream_width; + +// // float source_id = (float)frame_meta->source_id; +// /* Iterate user metadata in frames to search PGIE's tensor metadata +// */ for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list; +// l_user != NULL; l_user = l_user->next) { +// NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; +// if (user_meta->base_meta.meta_type != +// NVDSINFER_TENSOR_OUTPUT_META) +// continue; + +// /* convert to tensor metadata */ +// NvDsInferTensorMeta *meta = +// (NvDsInferTensorMeta *)user_meta->user_meta_data; +// for (unsigned int i = 0; i < meta->num_output_layers; i++) { +// NvDsInferLayerInfo *info = &meta->output_layers_info[i]; +// info->buffer = meta->out_buf_ptrs_host[i]; +// if (use_device_mem && meta->out_buf_ptrs_dev[i]) { +// cudaMemcpy(meta->out_buf_ptrs_host[i], +// meta->out_buf_ptrs_dev[i], +// info->inferDims.numElements * 4, +// cudaMemcpyDeviceToHost); +// } +// } +// /* Parse output tensor and fill detection results into +// objectList. +// */ +// std::vector outputLayersInfo( +// meta->output_layers_info, +// meta->output_layers_info + meta->num_output_layers); +// #if NVDS_VERSION_MAJOR >= 5 +// if (nvds_lib_major_version >= 5) { +// if (meta->network_info.width != networkInfo.width || +// meta->network_info.height != networkInfo.height || +// meta->network_info.channels != networkInfo.channels) { +// g_error("failed to check pgie network info\n"); +// } +// } +// #endif + +// // std::cout << "frame number: " << frame_meta->frame_num +// // << " frame id: " << frame_meta->source_id << +// std::endl; + +// float *outputBuffer = (float *)outputLayersInfo[0].buffer; +// (void)outputBuffer; +// // NvDsInferDims dims = outputLayersInfo[0].inferDims; + +// for (size_t jkl = 0; jkl < outputLayersInfo.size(); jkl++) { +// const NvDsInferLayerInfo &layer = outputLayersInfo[jkl]; + +// unsigned int numDims = layer.inferDims.numDims; +// unsigned int numElements = layer.inferDims.numElements; +// (void)numElements; +// (void)numDims; + +// // std::cout << "Layer " << jkl << " (" << layer.layerName << +// // "):\n"; std::cout << " Num Dims: " << numDims << "\n"; +// // std::cout << " Num Elements: " << numElements << "\n"; +// // std::cout << " Dims: ["; +// // for (unsigned int mno = 0; mno < numDims; ++mno) { +// // std::cout << layer.inferDims.d[mno]; +// // // layer.inferDims.d[0] = 100; +// // // layer.inferDims.d[1] = 57; +// // if (mno < numDims - 1) +// // std::cout << ", "; +// // } +// // std::cout << "]\n"; +// } +// const NvDsInferLayerInfo &layer = +// outputLayersInfo[0]; // or loop over all + +// uint detected_persons = 0; +// float *data = static_cast(layer.buffer); +// for (unsigned int jkl = 0; jkl < 100; +// jkl += 4) { // 100 persons for each frame +// if (data[jkl * 57 + 4] > threshold_body_detection) { +// detected_persons++; +// // std::cout +// // << "nvinferserver first for x = " << data[jkl * 57 +// + +// // 0] +// // << " y = " << data[jkl * 57 + 1] +// // << " w = " << data[jkl * 57 + 2] +// // << " h = " << data[jkl * 57 + 3] +// // << " score = " << data[jkl * 57 + 4] << std::endl; +// for (unsigned int mno = 0; mno < 57; ++mno) { +// float value = data[jkl * 57 + mno]; +// (void)value; +// // std::cout << "data[" << jkl << "][" << mno +// // << "] = " << value << std::endl; +// } +// } +// } + +// for (uint index = 0; index < detected_persons; index++) { +// NvDsObjectMeta *obj_meta = +// nvds_acquire_obj_meta_from_pool(batch_meta); +// obj_meta->unique_component_id = meta->unique_id; +// obj_meta->confidence = data[index * 57 + 4]; +// // obj_meta->object_id = UNTRACKED_OBJECT_ID; +// obj_meta->class_id = 0; +// NvOSD_RectParams &rect_params = obj_meta->rect_params; +// NvOSD_TextParams &text_params = obj_meta->text_params; +// /* Assign bounding box coordinates. */ +// rect_params.left = int(data[index * 57 + 0] * +// MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH); +// rect_params.top = int(data[index * 57 + 1] * +// MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT); +// rect_params.width = +// int((data[index * 57 + 2] - data[index * 57 + 0]) * +// MUXER_OUTPUT_WIDTH / PGIE_NET_WIDTH); +// rect_params.height = +// int((data[index * 57 + 3] - data[index * 57 + 1]) * +// MUXER_OUTPUT_HEIGHT / PGIE_NET_HEIGHT); + +// // std::cout << "nvinferserver second for x = " << +// // rect_params.left +// // << " y = " << rect_params.top +// // << " w = " << rect_params.width +// // << " h = " << rect_params.height +// // << " score = " << obj_meta->confidence << +// // std::endl; + +// /* Border of width 3. */ +// rect_params.border_width = 3; +// rect_params.has_bg_color = 0; +// rect_params.border_color = NvOSD_ColorParams{1, 0, 0, 1}; +// /* display_text requires heap allocated memory. */ +// text_params.display_text = g_strdup(pgie_class_str[0]); +// /* Display text above the left top corner of the object. */ +// text_params.x_offset = rect_params.left; +// text_params.y_offset = rect_params.top - 10; +// /* Set black background for the text. */ +// text_params.set_bg_clr = 1; +// text_params.text_bg_clr = NvOSD_ColorParams{0, 0, 0, 1}; +// /* Font face, size and color. */ +// text_params.font_params.font_name = (gchar *)"Serif"; +// text_params.font_params.font_size = 11; +// text_params.font_params.font_color = +// NvOSD_ColorParams{1, 1, 1, 1}; +// // adding landmarks to obj_meta as user_meta +// NvDsUserMeta *um1 = +// nvds_acquire_user_meta_from_pool(batch_meta); +// um1->user_meta_data = set_metadata_ptr( +// &(data[index * 57])); // Add landmarks here +// um1->base_meta.meta_type = +// NVDS_USER_OBJECT_META_LANDMARKS_AND_SOURCE_ID; +// um1->base_meta.copy_func = (NvDsMetaCopyFunc)copy_user_meta; +// um1->base_meta.release_func = +// (NvDsMetaReleaseFunc)release_user_meta; +// nvds_add_user_meta_to_obj(obj_meta, um1); +// nvds_add_obj_meta_to_frame(frame_meta, obj_meta, NULL); +// } +// } // } -// // user_metadata[51] = source_id; -// return (void *)user_metadata; +// // use_device_mem = 1 - use_device_mem; +// return GST_PAD_PROBE_OK; // } -// gpointer FaceNvInferServerManager::copy_user_meta(gpointer data, -// gpointer user_data) { -// (void)user_data; -// NvDsUserMeta *user_meta = (NvDsUserMeta *)data; -// gfloat *src_user_metadata = (gfloat *)user_meta->user_meta_data; -// gfloat *dst_user_metadata = (gfloat *)g_malloc0(57 * sizeof(gfloat)); -// memcpy(dst_user_metadata, src_user_metadata, 57 * sizeof(gfloat)); -// return (gpointer)dst_user_metadata; -// } +float FaceNvInferServerManager::numpy_clip(float value, float min_val, + float max_val) { + return std::clamp(value, min_val, max_val); +} -// void FaceNvInferServerManager::release_user_meta(gpointer data, -// gpointer user_data) { -// (void)user_data; -// NvDsUserMeta *user_meta = (NvDsUserMeta *)data; -// if (user_meta->user_meta_data) { -// g_free(user_meta->user_meta_data); -// user_meta->user_meta_data = NULL; +NvOSD_RectParams *FaceNvInferServerManager::allign_postprocess( + NvOSD_RectParams &parent_bbox, float *face_location) { + int target_w = 160; + int target_h = 160; + float x0 = parent_bbox.left; + float y0 = parent_bbox.top; + float w0 = parent_bbox.width; + float h0 = parent_bbox.height; + // std::cout << "x = " << x0 + // << " y = " << y0 + // << " w = " << w0 + // << " h = " << h0 + // << std::endl; + float scale = std::min(target_w / w0, target_h / h0); + int new_w = int(w0 * scale); + int new_h = int(h0 * scale); + int pad_w = target_w - new_w; + int pad_h = target_h - new_h; + + // std::cout << "scale = " << scale + // << " new_w = " << new_w + // << " new_h = " << new_h + // << " pad_w = " << pad_w + // << " pad_h = " << pad_h + // << std::endl; + int left = (int)pad_w / 2; + // int right = pad_w - left; + int top = (int)pad_h / 2; + // int bottom = pad_h - top; + // int _pad =(left, top); + // int _scale = scale; + // std::cout << "left = " << left + // << " top = " << top + // << std::endl; + + float x1n = numpy_clip(face_location[0], 0.0f, 1.0f); + float y1n = numpy_clip(face_location[1], 0.0f, 1.0f); + float x2n = numpy_clip(face_location[2], 0.0f, 1.0f); + float y2n = numpy_clip(face_location[3], 0.0f, 1.0f); + + // std::cout << "x1n = " << x1n + // << " y1n = " << y1n + // << " x2n = " << x2n + // << " y2n = " << y2n + // << std::endl; + + float x1_p = x1n * target_w; + float y1_p = y1n * target_h; + float x2_p = x2n * target_w; + float y2_p = y2n * target_h; + // std::cout << "x1_p = " << x1_p + // << " y1_p = " << y1_p + // << " x2_p = " << x2_p + // << " y2_p = " << y2_p + // << std::endl; + + int pad_x = left; + int pad_y = top; + // std::cout << "pad_x = " << pad_x + // << " pad_y = " << pad_y + // << std::endl; + float x1 = (x1_p - pad_x) / scale; + float y1 = (y1_p - pad_y) / scale; + float x2 = (x2_p - pad_x) / scale; + float y2 = (y2_p - pad_y) / scale; + // std::cout << "x1 = " << x1 + // << " y1 = " << y1 + // << " x2 = " << x2 + // << " y2 = " << y2 + // << std::endl; + + x1 += x0; + y1 += y0; + x2 += x0; + y2 += y0; + // std::cout << "x1 = " << x1 + // << " y1 = " << y1 + // << " x2 = " << x2 + // << " y2 = " << y2 + // << std::endl; + + NvOSD_RectParams *updated_bbox = new NvOSD_RectParams(); + updated_bbox->left = x1; + updated_bbox->top = y1; + updated_bbox->width = x2 - x1; + updated_bbox->height = y2 - y1; + // std::cout << "x = " << updated_bbox->left + // << " y = " << updated_bbox->top + // << " w = " << updated_bbox->width + // << " h = " << updated_bbox->height + // << std::endl; + // box = [int(x1), int(y1), int(x2), int(y2)]; + // std::quick_exit(0); + return updated_bbox; +} + +void FaceNvInferServerManager::add_face_body(int object_id, float face_score) { + bool find = false; + for (std::vector::iterator iter = face_body_list.begin(); + iter != face_body_list.end(); iter++) { + if (((*iter).object_id == object_id)) { + find = true; + if (((*iter).face_score > face_score)) { + // std::cout<<(*iter).face_score; + (*iter).face_score = face_score; + // std::cout<<" "<<(*iter).face_score<rect_params.left, + // body_obj->rect_params.top, + // body_obj->rect_params.width, + // body_obj->rect_params.height); + } else { + // printf( + // "sgie_pad_buffer_probe HHH Face bbox: %.2f %.2f %.2f + // " + // "%.2f\n", + // body_obj->rect_params.left, + // body_obj->rect_params.top, + // body_obj->rect_params.width, + // body_obj->rect_params.height); + } + } + // else{ + // // std::cout<<"obj_meta->class_id =" + // <class_id<obj_user_meta_list; + l_user != NULL; l_user = l_user->next) { + NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; + if (user_meta->base_meta.meta_type != + NVDSINFER_TENSOR_OUTPUT_META) + continue; + + /* convert to tensor metadata */ + NvDsInferTensorMeta *meta = + (NvDsInferTensorMeta *)user_meta->user_meta_data; + + // from here + // std::cout << "meta->num_output_layers: " + // << meta->num_output_layers << std::endl; + for (unsigned int i = 0; i < meta->num_output_layers; i++) { + NvDsInferLayerInfo *info = &meta->output_layers_info[i]; + info->buffer = meta->out_buf_ptrs_host[i]; + if (use_device_mem && meta->out_buf_ptrs_dev[i]) { + cudaMemcpy(meta->out_buf_ptrs_host[i], + meta->out_buf_ptrs_dev[i], + info->inferDims.numElements * 4, + cudaMemcpyDeviceToHost); + } + } + /* Parse output tensor and fill detection results into + * objectList.*/ + std::vector outputLayersInfo( + meta->output_layers_info, + meta->output_layers_info + meta->num_output_layers); + // std::cout << "meta->network_info.width: " + // << meta->network_info.width + // << " meta->network_info.height: " + // << meta->network_info.height + // << " meta->network_info.channels: " + // << meta->network_info.channels << std::endl; +#if NVDS_VERSION_MAJOR >= 5 + if (nvds_lib_major_version >= 5) { + if (meta->network_info.width != networkInfo.width || + meta->network_info.height != networkInfo.height || + meta->network_info.channels != networkInfo.channels) { + g_error("failed to check sgie network info\n"); + } + } +#endif + + // std::cout << "frame number: " << frame_meta->frame_num + // << " frame id: " << frame_meta->source_id + // << " outputLayersInfo.size(): " + // << outputLayersInfo.size() << std::endl; + + float *outputBuffer = (float *)outputLayersInfo[0].buffer; + (void)outputBuffer; + NvDsInferDims dims = outputLayersInfo[0].inferDims; + (void)dims; + + float score_face = 0; + float *face_location = (float *)g_malloc0(4 * sizeof(float)); + for (size_t jkl = 0; jkl < outputLayersInfo.size(); jkl++) { + const NvDsInferLayerInfo &layer = outputLayersInfo[jkl]; + + unsigned int numDims = layer.inferDims.numDims; + unsigned int numElements = layer.inferDims.numElements; + // (void)numElements; + // (void)numDims; + + // std::cout << "Layer " << jkl << " (" << layer.layerName + // << "):\n"; + // std::cout << " Num Dims: " << numDims << "\n"; + // std::cout << " Num Elements: " << numElements << "\n"; + // std::cout << " Dims: ["; + for (unsigned int mno = 0; mno < numDims; ++mno) { + // std::cout << layer.inferDims.d[mno]; + if (mno < numDims - 1) { + // std::cout << ", "; + } + } + // std::cout << "]\n"; + + float *data_face = static_cast(layer.buffer); + if (!(strcmp(layer.layerName, "embedding") == 0)) { + for (unsigned int xyz = 0; xyz < numElements; xyz++) { + // std::cout << "data_face[" << xyz + // << "]= " << data_face[xyz] << + // std::endl; + } + if ((strcmp(layer.layerName, "bbox") == 0)) { + for (int l = 0; l < 4; l++) { + face_location[l] = data_face[l]; + // std::cout << "face_location[" << l + // << "]= " << data_face[l] << + // std::endl; + } + } + if ((strcmp(layer.layerName, "score") == 0)) { + score_face = data_face[0]; + // std::cout << "score_face= " << score_face << + // std::endl; if (score_face>0.9){ + // high_confidence_faces++; + // std::cout << "high_confidence_faces= " << + // high_confidence_faces << std::endl; + // std::quick_exit(0); + // } + } + } + // else{ + // for (int l = 0; l < 512; l++) { + // std::cout << "face_location[" << l + // << "]= " << data_face[l] << std::endl; + // } + // } + } + + if (score_face > threshold_face_detection) { + // std::cout<<"obj_meta->rect_params left = + // "<rect_params.left + // <<" top = "<rect_params.top + // <<" width = "<rect_params.width + // <<" height = + // "<rect_params.height<rect_params, face_location); + // std::cout<<"updated_bbox left = "<left + // <<" top = "<top + // <<" width = "<width + // <<" height = "<height<obj_label, "FINAL_FACE", + sizeof(final_face_obj->obj_label) - 1); + final_face_obj + ->obj_label[sizeof(final_face_obj->obj_label) - 1] = + '\0'; // Ensure null termination + final_face_obj->unique_component_id = + FINAL_FACE_COMPONENT_ID; // Use a new component ID + final_face_obj->confidence = score_face; + + add_face_body(obj_meta->object_id, score_face); + // std::cout << "body_obj->object_id = " << + // body_obj->object_id + // << " score_face= "<object_id); + // printf("Imprecise Face ID: %lu, Precise Face ID: %lu\n", + // obj_meta->object_id, final_face_obj->object_id); + /* Display text above the left top corner of the object.*/ + text_params.x_offset = rect_params.left; + text_params.y_offset = rect_params.top + 30; + /* Set black background for the text. */ + text_params.set_bg_clr = 1; + text_params.text_bg_clr = NvOSD_ColorParams{0, 0, 0, 1}; + /* Font face, size and color. */ + text_params.font_params.font_name = (gchar *)"Serif"; + text_params.font_params.font_size = 11; + text_params.font_params.font_color = + NvOSD_ColorParams{1, 1, 1, 1}; + // adding landmarks to obj_meta as user_meta + // NvDsUserMeta *um1 = + // nvds_acquire_user_meta_from_pool(batch_meta); + // um1->user_meta_data = set_metadata_ptr( + // &(data[index * 57])); // Add landmarks here + // um1->base_meta.meta_type = + // NVDS_USER_OBJECT_META_LANDMARKS_AND_SOURCE_ID; + // um1->base_meta.copy_func + // =(NvDsMetaCopyFunc)copy_user_meta; + // um1->base_meta.release_func = + // (NvDsMetaReleaseFunc)release_user_meta; + // nvds_add_user_meta_to_obj(obj_meta, um1); + nvds_add_obj_meta_to_frame(frame_meta, final_face_obj, + obj_meta); + // nvds_add_obj_meta_to_frame(frame_meta, final_face_obj, + // NULL); + } + + // for (size_t jkl = 0; jkl < outputLayersInfo.size(); jkl++) { + // const NvDsInferLayerInfo &layer = outputLayersInfo[jkl]; + // unsigned int numDims = layer.inferDims.numDims; + // unsigned int numElements = layer.inferDims.numElements; + + // std::cout << "Layer " << jkl << " (" << layer.layerName + // << + // "):\n"; std::cout << " Num Dims: " << numDims << "\n"; + // std::cout << " Num Elements: " << numElements << "\n"; + // std::cout << " Dims: ["; + // for (unsigned int mno = 0; mno < numDims; ++mno) { + // std::cout << layer.inferDims.d[mno]; + // if (mno < numDims - 1) + // std::cout << ", "; + // } + // std::cout << "]\n"; + + // float *data_face = static_cast(layer.buffer); + // } + + // std::quick_exit(0); + // const NvDsInferLayerInfo &layer = + // outputLayersInfo[0]; // or loop over all + + // std::cout<<"meta->num_output_layers: "<< + // meta->num_output_layers<num_output_layers; i++) { + // NvDsInferLayerInfo *info = &meta->output_layers_info[i]; + // info->buffer = meta->out_buf_ptrs_host[i]; + // if (use_device_mem && meta->out_buf_ptrs_dev[i]) { + // cudaMemcpy (meta->out_buf_ptrs_host[i], + // meta->out_buf_ptrs_dev[i], + // info->inferDims.numElements * 4, + // cudaMemcpyDeviceToHost); + // } + // } + + NvDsInferDimsCHW dims_chw; + + getDimsCHWFromDims(dims_chw, + meta->output_layers_info[0].inferDims); + unsigned int numClasses = dims_chw.c; + // std::cout << "numClasses: " << numClasses << std::endl; + (void)numClasses; + // float *outputCoverageBuffer = + // (float *) meta->output_layers_info[0].buffer; + // float maxProbability = 0; + // bool attrFound = false; + // NvDsInferAttribute attr; + + /* Iterate through all the probabilities that the object belongs + * to each class. Find the maximum probability and the + * corresponding class which meets the minimum threshold. */ + // for (unsigned int c = 0; c < numClasses; c++) { + // float probability = outputCoverageBuffer[c]; + // if (probability > 0.51 && probability > maxProbability) { + // maxProbability = probability; + // attrFound = true; + // attr.attributeIndex = 0; + // attr.attributeValue = c; + // attr.attributeConfidence = probability; + // } + // } + + /* Generate classifer metadata and attach to obj_meta */ + // if (attrFound) { + // NvDsClassifierMeta *classifier_meta = + // nvds_acquire_classifier_meta_from_pool (batch_meta); + + // classifier_meta->unique_component_id = meta->unique_id; + + // NvDsLabelInfo *label_info = + // nvds_acquire_label_info_meta_from_pool (batch_meta); + // label_info->result_class_id = attr.attributeValue; + // label_info->result_prob = attr.attributeConfidence; + + // /* Fill label name */ + // switch (meta->unique_id) { + // case sgie1_unique_id: + // strcpy (label_info->result_label, + // sgie1_classes_str[label_info->result_class_id]); + // break; + // // case sgie2_unique_id: + // // strcpy (label_info->result_label, + // // sgie2_classes_str[label_info->result_class_id]); + // // break; + // default: + // break; + // } + // gchar *temp = obj_meta->text_params.display_text; + // obj_meta->text_params.display_text = + // g_strconcat (temp, " ", label_info->result_label, + // nullptr); + // g_free (temp); + + // nvds_add_label_info_meta_to_classifier (classifier_meta, + // label_info); nvds_add_classifier_meta_to_object (obj_meta, + // classifier_meta); + // } + } + } + } + + // use_device_mem = 1 - use_device_mem; + return GST_PAD_PROBE_OK; +} + +// add custom infromation to metadata by: set_metadata_ptr, copy_user_meta, +// release_user_meta +void *FaceNvInferServerManager::set_metadata_ptr(float *arr) { + int i = 0; + float *user_metadata = (float *)g_malloc0(512 * sizeof(float)); + + for (i = 0; i < 512; i++) { + user_metadata[i] = arr[i]; + } + return (void *)user_metadata; +} + +gpointer FaceNvInferServerManager::copy_user_meta(gpointer data, + gpointer user_data) { + (void)user_data; + NvDsUserMeta *user_meta = (NvDsUserMeta *)data; + gfloat *src_user_metadata = (gfloat *)user_meta->user_meta_data; + gfloat *dst_user_metadata = (gfloat *)g_malloc0(512 * sizeof(gfloat)); + memcpy(dst_user_metadata, src_user_metadata, 512 * sizeof(gfloat)); + return (gpointer)dst_user_metadata; +} + +void FaceNvInferServerManager::release_user_meta(gpointer data, + gpointer user_data) { + (void)user_data; + NvDsUserMeta *user_meta = (NvDsUserMeta *)data; + if (user_meta->user_meta_data) { + g_free(user_meta->user_meta_data); + user_meta->user_meta_data = NULL; + } +} + +// }{1, 1, 1, 1}; +// // adding landmarks to obj_meta as user_meta +// NvDsUserMeta *um1 = +// nvds_acquire_user_meta_from_pool(batch_meta); +// um1->user_meta_data = set_metadata_ptr( +// &(data[index * 57])); // Add landmarks here +// um1->base_meta.meta_type = +// NVDS_USER_OBJECT_META_LANDMARKS_AND_SOURCE_ID; +// um1->base_meta.copy_func = (NvDsMetaCopyFunc)copy_user_meta; +// um1->base_meta.release_func = +// (NvDsMetaReleaseFunc)release_user_meta; +// nvds_add_user_meta_to_obj(obj_meta, um1); +// nvds_add_obj_meta_to_frame(frame_meta, obj_meta, NULL); +// } +// } // } +// // use_device_mem = 1 - use_device_mem; +// return GST_PAD_PROBE_OK; // } \ No newline at end of file diff --git a/src/face_nv_infer_server_manager.hpp b/src/face_nv_infer_server_manager.hpp index 29a8a5a..01cc348 100644 --- a/src/face_nv_infer_server_manager.hpp +++ b/src/face_nv_infer_server_manager.hpp @@ -3,15 +3,24 @@ #include #include -#include "config_manager.hpp" // #include "gstnvdsinfer.h" -// #include "gstnvdsmeta.h" +#include "gstnvdsmeta.h" #include "nvds_version.h" // #include "nvdsinfer_custom_impl.h" +#include "nvdsmeta.h" +#include "nvdsmeta_schema.h" +#include "custom_gstnvdsinfer.hpp" +#include "config_manager.hpp" + class FaceNvInferServerManager { private: public: + struct FACE_BODY { + int object_id = 0; + float face_score = 0; + }; + static std::vector face_body_list; GstElement *face_detector = NULL; int face_batch_size; @@ -41,4 +50,17 @@ class FaceNvInferServerManager { // static void *set_metadata_ptr(float *); // static gpointer copy_user_meta(gpointer, gpointer); // static void release_user_meta(gpointer, gpointer); + + static GstPadProbeReturn sgie_pad_buffer_probe(GstPad *, GstPadProbeInfo + *, + gpointer); + // static GstPadProbeReturn osd_sink_pad_buffer_probe_new(GstPad *, + // GstPadProbeInfo + // *, gpointer); + static void *set_metadata_ptr(float *); + static gpointer copy_user_meta(gpointer, gpointer); + static void release_user_meta(gpointer, gpointer); + static NvOSD_RectParams * allign_postprocess(NvOSD_RectParams &, float*); + static float numpy_clip(float, float, float); + static void add_face_body(int, float); }; \ No newline at end of file diff --git a/src/nv_infer_server_manager.cpp b/src/nv_infer_server_manager.cpp index 19f16b3..4114134 100644 --- a/src/nv_infer_server_manager.cpp +++ b/src/nv_infer_server_manager.cpp @@ -22,8 +22,9 @@ float NvInferServerManager::threshold_body_detection = 0; unsigned int NvInferServerManager::nvds_lib_major_version = NVDS_VERSION_MAJOR; unsigned int NvInferServerManager::nvds_lib_minor_version = NVDS_VERSION_MINOR; -const gchar pgie_class_str[PGIE_DETECTED_CLASS_NUM][32] = {"Person"}; -const gchar imprecise_face_str[PGIE_DETECTED_CLASS_NUM][32] = {"ImpreciseFace"}; +const gchar pgie_class_str[PGIE_DETECTED_CLASS_NUM][32] = {"Person_NVINFER"}; +const gchar imprecise_face_str[PGIE_DETECTED_CLASS_NUM][32] = { + "ImpreciseFace_NVINFER"}; /* nvds_lib_major_version and nvds_lib_minor_version is the version number of * deepstream sdk */ @@ -244,6 +245,7 @@ GstPadProbeReturn NvInferServerManager::pgie_pad_buffer_probe( } (void)stream_height; (void)stream_width; + uint detected_persons = 0; /* Iterate user metadata in frames to search PGIE's tensor metadata */ for (NvDsMetaList *l_user = frame_meta->frame_user_meta_list; @@ -251,15 +253,46 @@ GstPadProbeReturn NvInferServerManager::pgie_pad_buffer_probe( NvDsUserMeta *user_meta = (NvDsUserMeta *)l_user->data; if (user_meta->base_meta.meta_type != NVDSINFER_TENSOR_OUTPUT_META) continue; - extract_tensor_metadata(user_meta, networkInfo, batch_meta, - frame_meta); + detected_persons = extract_tensor_metadata(user_meta, networkInfo, + batch_meta, frame_meta); } + + NvDsDisplayMeta *display_meta = NULL; + display_meta = nvds_acquire_display_meta_from_pool(batch_meta); + NvOSD_TextParams *txt_params = &display_meta->text_params[0]; + display_meta->num_labels = 1; + txt_params->display_text = (gchar *)g_malloc0(MAX_DISPLAY_LEN); + int offset = 0; + offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, + "Person_NVInfer = %d ", detected_persons); + (void)offset; + + /* Now set the offsets where the string should appear */ + txt_params->x_offset = 10; + txt_params->y_offset = 12; + + /* Font , font-color and font-size */ + txt_params->font_params.font_name = (gchar *)"Serif"; + txt_params->font_params.font_size = 10; + txt_params->font_params.font_color.red = 1.0; + txt_params->font_params.font_color.green = 1.0; + txt_params->font_params.font_color.blue = 1.0; + txt_params->font_params.font_color.alpha = 1.0; + + /* Text background color */ + txt_params->set_bg_clr = 1; + txt_params->text_bg_clr.red = 0.0; + txt_params->text_bg_clr.green = 0.0; + txt_params->text_bg_clr.blue = 0.0; + txt_params->text_bg_clr.alpha = 1.0; + + nvds_add_display_meta_to_frame(frame_meta, display_meta); } // use_device_mem = 1 - use_device_mem; return GST_PAD_PROBE_OK; } -void NvInferServerManager::extract_tensor_metadata( +uint NvInferServerManager::extract_tensor_metadata( NvDsUserMeta *user_meta, NvDsInferNetworkInfo networkInfo, NvDsBatchMeta *batch_meta, NvDsFrameMeta *frame_meta) { /* convert to tensor metadata */ @@ -325,6 +358,7 @@ void NvInferServerManager::extract_tensor_metadata( } update_frame_with_face_body_meta(detected_persons, batch_meta, data, frame_meta); + return detected_persons; } void NvInferServerManager::update_frame_with_face_body_meta( @@ -395,14 +429,14 @@ void NvInferServerManager::update_frame_with_face_body_meta( /* Border of width 3. */ rect_params_imprecise_face.border_width = 3; rect_params_imprecise_face.has_bg_color = 0; - rect_params_imprecise_face.border_color = NvOSD_ColorParams{0, 0, 1, 1}; + rect_params_imprecise_face.border_color = + NvOSD_ColorParams{0, 0, 1, 1}; // Blue box /* display_text requires heap allocated memory. */ text_params_imprecise_face.display_text = g_strdup(imprecise_face_str[0]); // g_strdup(pgie_class_str[0]); /* Display text above the left top corner of the object. */ text_params_imprecise_face.x_offset = rect_params_imprecise_face.left; - text_params_imprecise_face.y_offset = - rect_params_imprecise_face.top - 10; + text_params_imprecise_face.y_offset = rect_params_imprecise_face.top; /* Set black background for the text. */ text_params_imprecise_face.set_bg_clr = 1; text_params_imprecise_face.text_bg_clr = NvOSD_ColorParams{0, 0, 0, 1}; @@ -413,6 +447,7 @@ void NvInferServerManager::update_frame_with_face_body_meta( NvOSD_ColorParams{1, 1, 1, 1}; // adding landmarks to imprecise_face_obj_meta as user_meta NvDsUserMeta *um1 = nvds_acquire_user_meta_from_pool(batch_meta); + assert(um1 != NULL); um1->user_meta_data = set_metadata_ptr(&(data[index * 57])); // Add landmarks here um1->base_meta.meta_type = @@ -451,12 +486,15 @@ void NvInferServerManager::update_frame_with_face_body_meta( /* Border of width 3. */ rect_params_body.border_width = 3; rect_params_body.has_bg_color = 0; - rect_params_body.border_color = NvOSD_ColorParams{1, 0, 0, 1}; + rect_params_body.border_color = + NvOSD_ColorParams{1, 0, 0, 1}; // Red box /* display_text requires heap allocated memory. */ text_params_body.display_text = g_strdup(pgie_class_str[0]); + // text_params.display_text = g_strdup_printf("ImpreciseFace %lu", + // face_obj->object_id); /* Display text above the left top corner of the object. */ text_params_body.x_offset = rect_params_body.left - 30; - text_params_body.y_offset = rect_params_body.top - 50; + text_params_body.y_offset = rect_params_body.top - 30; /* Set black background for the text. */ text_params_body.set_bg_clr = 1; text_params_body.text_bg_clr = NvOSD_ColorParams{0, 0, 0, 1}; diff --git a/src/nv_infer_server_manager.hpp b/src/nv_infer_server_manager.hpp index 927c7b8..3dc3d4d 100644 --- a/src/nv_infer_server_manager.hpp +++ b/src/nv_infer_server_manager.hpp @@ -4,10 +4,11 @@ #include #include "config_manager.hpp" -#include "gstnvdsinfer.h" +// #include "gstnvdsinfer.h" #include "gstnvdsmeta.h" #include "nvds_version.h" #include "nvdsinfer_custom_impl.h" +#include "custom_gstnvdsinfer.hpp" class NvInferServerManager { private: @@ -49,7 +50,7 @@ class NvInferServerManager { static void release_user_meta(gpointer, gpointer); static void update_frame_with_face_body_meta(uint, NvDsBatchMeta *, float *, NvDsFrameMeta *); - static void extract_tensor_metadata(NvDsUserMeta *, NvDsInferNetworkInfo, + static uint extract_tensor_metadata(NvDsUserMeta *, NvDsInferNetworkInfo, NvDsBatchMeta *, NvDsFrameMeta *); static Point2D find_left_down_corner_shoulder(float *, uint); }; \ No newline at end of file diff --git a/src/nv_tracker_manager.cpp b/src/nv_tracker_manager.cpp index 55be652..a738e65 100644 --- a/src/nv_tracker_manager.cpp +++ b/src/nv_tracker_manager.cpp @@ -22,7 +22,8 @@ FaceCandidTrace *NvTrackerManager::face_candidate_trace = new FaceCandidTrace(); // nullptr; // Definition gint NvTrackerManager::frame_number = 0; -const gchar face_class_str[FACE_DETECTED_CLASS_NUM][32] = {"FACE_TRACKER"}; +const gchar face_class_str[FACE_DETECTED_CLASS_NUM][32] = { + "ImpreciseFace_TRACKER"}; NvTrackerManager::NvTrackerManager() { const auto &config = ConfigManager::get_instance().get_config(); @@ -237,7 +238,7 @@ GstPadProbeReturn NvTrackerManager::tracker_src_pad_buffer_probe( face_obj->rect_params.has_bg_color = 0; face_obj->rect_params.border_width = 2; face_obj->rect_params.border_color = - NvOSD_ColorParams{1.0, 0.0, 0.0, 1.0}; // Red box + NvOSD_ColorParams{0.0, 0.0, 1.0, 1.0}; // Blue box // FaceCandidate *face_candidate = new FaceCandidate(); FaceCandidTrace::FaceCandidate *face_candidate = @@ -253,6 +254,7 @@ GstPadProbeReturn NvTrackerManager::tracker_src_pad_buffer_probe( user_meta_data); // manager->face_score; face_obj->confidence = face_candidate->face_score; // 1.0; face_candidate->object_id = obj_meta->object_id; + face_obj->object_id = obj_meta->object_id; face_candidate->source_id = frame_meta->source_id; bool add_status = face_candidate_trace->add(face_candidate); @@ -263,6 +265,26 @@ GstPadProbeReturn NvTrackerManager::tracker_src_pad_buffer_probe( face_obj->class_id = 41; } + NvOSD_TextParams &text_params = face_obj->text_params; + NvOSD_RectParams &rect_params = face_obj->rect_params; + /* display_text requires heap allocated memory. */ + // Instead of letting OSD auto-generate text, set your own + text_params.display_text = g_strdup_printf( + "ImpreciseFace_Tracker %lu", face_obj->object_id); + // printf("Imprecise Face ID: %lu, Precise Face ID: %lu\n", + // obj_meta->object_id, final_face_obj->object_id); + /* Display text above the left top corner of the object.*/ + text_params.x_offset = rect_params.left; + text_params.y_offset = rect_params.top + 30; + /* Set black background for the text. */ + text_params.set_bg_clr = 1; + text_params.text_bg_clr = NvOSD_ColorParams{0, 0, 0, 1}; + /* Font face, size and color. */ + text_params.font_params.font_name = (gchar *)"Serif"; + text_params.font_params.font_size = 11; + text_params.font_params.font_color = + NvOSD_ColorParams{1, 1, 1, 1}; + // std::cout << "In Tracker sink " // << " source_id = " << frame_meta->source_id // << " object_id = " << obj_meta->object_id @@ -373,12 +395,12 @@ GstPadProbeReturn NvTrackerManager::tracker_src_pad_buffer_probe( display_meta->num_labels = 1; txt_params->display_text = (gchar *)g_malloc0(MAX_DISPLAY_LEN); offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, - "Person = %d ", person_count); + "Person_TRACKER = %d ", person_count); (void)offset; /* Now set the offsets where the string should appear */ txt_params->x_offset = 10; - txt_params->y_offset = 12; + txt_params->y_offset = 52; /* Font , font-color and font-size */ txt_params->font_params.font_name = (gchar *)"Serif"; diff --git a/src/pipeline_manager.cpp b/src/pipeline_manager.cpp index ea1a8f3..62f0651 100644 --- a/src/pipeline_manager.cpp +++ b/src/pipeline_manager.cpp @@ -393,6 +393,15 @@ bool PipelineManager::create_pipeline_elements(int num_sources, nv_infer_server_manager->pgie_pad_buffer_probe, &use_new_mux, NULL); + /* Add probe to get informed of the meta data generated, we add probe to + * the source pad of SGIE's next queue element, since by that time, SGIE's + * buffer would have had got tensor metadata. */ + sgie_src_pad = gst_element_get_static_pad( + face_nv_infer_server_manager->face_detector, "src"); + gst_pad_add_probe(sgie_src_pad, GST_PAD_PROBE_TYPE_BUFFER, + face_nv_infer_server_manager->sgie_pad_buffer_probe, + &use_new_mux, NULL); + auto start = std::chrono::system_clock::now(); status_playing = playing_pipeline(num_sources, url_camera); if (status_playing == false) { diff --git a/src/pipeline_manager.hpp b/src/pipeline_manager.hpp index c5401cf..c9c7f31 100644 --- a/src/pipeline_manager.hpp +++ b/src/pipeline_manager.hpp @@ -71,6 +71,7 @@ class PipelineManager { const gchar *new_mux_str; gboolean use_new_mux; GstPad *pgie_src_pad = NULL; + GstPad *sgie_src_pad = NULL; static std::chrono::time_point last_time_osd_sink; static std::chrono::time_point