From 394402b37c3db1a61f0846f09bb7b6907ed3522c Mon Sep 17 00:00:00 2001
From: Barzan Hayati <barzanhayati@gmail.com>
Date: Wed, 17 Sep 2025 14:57:48 +0000
Subject: [PATCH] Attach embedding vector in sgie fetch in nvosd

---
 src/face_nv_infer_server_manager.cpp | 59 ++++++++++++++++++++--------
 src/nv_osd_manager.cpp               | 26 +++++++++++-
 2 files changed, 68 insertions(+), 17 deletions(-)

diff --git a/src/face_nv_infer_server_manager.cpp b/src/face_nv_infer_server_manager.cpp
index 1ccee04..527cd5b 100644
--- a/src/face_nv_infer_server_manager.cpp
+++ b/src/face_nv_infer_server_manager.cpp
@@ -4,6 +4,9 @@
 // (nvds_get_user_meta_type("NVIDIA.NVINFER.USER_META"))
 #define NVDS_USER_OBJECT_META_LANDMARKS_AND_SOURCE_ID \
     (nvds_get_user_meta_type(const_cast<gchar *>("NVIDIA.NVINFER.USER_META")))
+#define NVDS_USER_EMBEDDING_VECTOR_META \
+    (nvds_get_user_meta_type(           \
+        const_cast<gchar *>("NVIDIA.NVINFER.EMBEDDING_VECTOR.USER_META")))
 
 #define MAX_DISPLAY_LEN 64
 #define PGIE_CLASS_ID_PERSON 0
@@ -12,6 +15,7 @@
 #define BODY_COMPONENT_ID 1
 #define IMPRECISE_FACE_COMPONENT_ID 2
 #define FINAL_FACE_COMPONENT_ID 3
+#define EMBEDDING_VECTOR_SIZE 512
 
 gint FaceNvInferServerManager::frame_number = 0;
 unsigned int FaceNvInferServerManager::FACE_NET_WIDTH = 1;
@@ -1364,6 +1368,8 @@ GstPadProbeReturn FaceNvInferServerManager::sgie_pad_buffer_probe(
 
                 float score_face = 0;
                 float *face_location = (float *)g_malloc0(4 * sizeof(float));
+                float *embedding_vector =
+                    (float *)g_malloc0(EMBEDDING_VECTOR_SIZE * sizeof(float));
                 for (size_t jkl = 0; jkl < outputLayersInfo.size(); jkl++) {
                     const NvDsInferLayerInfo &layer = outputLayersInfo[jkl];
 
@@ -1416,25 +1422,27 @@ GstPadProbeReturn FaceNvInferServerManager::sgie_pad_buffer_probe(
                             0.0f)  // if the first element is zero, then its
                                    // possible all element are zero
                         {
-                            is_zero_embedding_vector = all_zero(data_face, 512);
+                            is_zero_embedding_vector =
+                                all_zero(data_face, EMBEDDING_VECTOR_SIZE);
                             // is_zero_embedding_vector = 1 where approximately
                             // 3 times more than is_zero_embedding_vector 0
                         } else {
-                            is_zero_embedding_vector =
-                                std::all_of(data_face, data_face + 512,
-                                            [](float v) { return v == 0.0f; });
+                            is_zero_embedding_vector = std::all_of(
+                                data_face, data_face + EMBEDDING_VECTOR_SIZE,
+                                [](float v) { return v == 0.0f; });
                             // is_zero_embedding_vector = 0;
                         }
-                        // std::cout<<"is_zero_embedding_vector =
-                        // "<<is_zero_embedding_vector<<std::endl; for (int l =
-                        // 0; l < 512; l++) {
-                        //     std::cout << "face_location[" << l
-                        //             << "]= " << data_face[l] << std::endl;
-                        // }
+
+                        if (is_zero_embedding_vector == false) {
+                            // copy the values
+                            std::memcpy(embedding_vector, data_face,
+                                        EMBEDDING_VECTOR_SIZE * sizeof(float));
+                        }
                     }
                 }
 
-                if (score_face > threshold_face_detection) {
+                if (score_face > threshold_face_detection &&
+                    is_zero_embedding_vector == false) {
                     NvOSD_RectParams *updated_bbox = allign_postprocess(
                         obj_meta->rect_params, face_location);
 
@@ -1541,6 +1549,23 @@ GstPadProbeReturn FaceNvInferServerManager::sgie_pad_buffer_probe(
                             encode_objects_attach_meta(ctx, ip_surf, frame_meta,
                                                        final_face_obj);
                         }
+
+                        // adding embedding vector to final_face_obj as
+                        // user_meta
+                        NvDsUserMeta *user_meta_embedding_vector =
+                            nvds_acquire_user_meta_from_pool(batch_meta);
+                        assert(user_meta_embedding_vector != NULL);
+                        user_meta_embedding_vector->user_meta_data =
+                            set_metadata_ptr(
+                                embedding_vector);  // Add embedding vector
+                        user_meta_embedding_vector->base_meta.meta_type =
+                            NVDS_USER_EMBEDDING_VECTOR_META;
+                        user_meta_embedding_vector->base_meta.copy_func =
+                            (NvDsMetaCopyFunc)copy_user_meta;
+                        user_meta_embedding_vector->base_meta.release_func =
+                            (NvDsMetaReleaseFunc)release_user_meta;
+                        nvds_add_user_meta_to_obj(final_face_obj,
+                                                  user_meta_embedding_vector);
                     }
                     nvds_add_obj_meta_to_frame(frame_meta, final_face_obj,
                                                obj_meta);
@@ -1643,9 +1668,9 @@ GstPadProbeReturn FaceNvInferServerManager::sgie_pad_buffer_probe(
 // release_user_meta
 void *FaceNvInferServerManager::set_metadata_ptr(float *arr) {
     int i = 0;
-    float *user_metadata = (float *)g_malloc0(512 * sizeof(float));
-
-    for (i = 0; i < 512; i++) {
+    float *user_metadata =
+        (float *)g_malloc0(EMBEDDING_VECTOR_SIZE * sizeof(float));
+    for (i = 0; i < EMBEDDING_VECTOR_SIZE; i++) {
         user_metadata[i] = arr[i];
     }
     return (void *)user_metadata;
@@ -1656,8 +1681,10 @@ gpointer FaceNvInferServerManager::copy_user_meta(gpointer data,
     (void)user_data;
     NvDsUserMeta *user_meta = (NvDsUserMeta *)data;
     gfloat *src_user_metadata = (gfloat *)user_meta->user_meta_data;
-    gfloat *dst_user_metadata = (gfloat *)g_malloc0(512 * sizeof(gfloat));
-    memcpy(dst_user_metadata, src_user_metadata, 512 * sizeof(gfloat));
+    gfloat *dst_user_metadata =
+        (gfloat *)g_malloc0(EMBEDDING_VECTOR_SIZE * sizeof(gfloat));
+    memcpy(dst_user_metadata, src_user_metadata,
+           EMBEDDING_VECTOR_SIZE * sizeof(gfloat));
     return (gpointer)dst_user_metadata;
 }
 
diff --git a/src/nv_osd_manager.cpp b/src/nv_osd_manager.cpp
index 0c392e8..cd72858 100644
--- a/src/nv_osd_manager.cpp
+++ b/src/nv_osd_manager.cpp
@@ -1,5 +1,8 @@
 #include "nv_osd_manager.hpp"
 
+#define NVDS_USER_EMBEDDING_VECTOR_META \
+    (nvds_get_user_meta_type(           \
+        const_cast<gchar *>("NVIDIA.NVINFER.EMBEDDING_VECTOR.USER_META")))
 // #define ENABLE_DUMP_FILE
 #ifdef ENABLE_DUMP_FILE
 FILE *fp;
@@ -38,6 +41,7 @@ char fileObjNameString[1024];
 #define MAX_TIME_STAMP_LEN 32
 #define PGIE_CLASS_ID_PERSON 0
 #define FACE_CLASS_ID 1
+#define EMBEDDING_VECTOR_SIZE 512
 
 gint msg2p_meta =
     0;  //"Type of message schema (0=Full, 1=minimal, 2=protobuf), default=0
@@ -508,7 +512,25 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_metadata_probe(
              * component implementing detection / recognition logic.
              * Here it demonstrates how to use / attach that meta data.
              */
-            if (is_first_object && !(frame_number % frame_interval)) {
+
+            NvDsUserMeta *user_meta = NULL;
+            NvDsMetaList *l_user_meta = NULL;
+            float *user_meta_data = NULL;
+            bool is_meta_type_NVOSD_embedding_vector = false;
+            for (l_user_meta = obj_meta->obj_user_meta_list;
+                 l_user_meta != NULL; l_user_meta = l_user_meta->next) {
+                user_meta = (NvDsUserMeta *)(l_user_meta->data);
+                user_meta_data = (float *)user_meta->user_meta_data;
+                if (user_meta->base_meta.meta_type ==
+                    NVDS_USER_EMBEDDING_VECTOR_META) {
+                    std::cout << "NVOSD_EMBEDDING_VECTOR[" << 0
+                              << "]= " << user_meta_data[0] << std::endl;
+                    is_meta_type_NVOSD_embedding_vector = true;
+                }
+            }
+
+            if (is_first_object && !(frame_number % frame_interval) &&
+                is_meta_type_NVOSD_embedding_vector == true) {
                 /* Frequency of messages to be send will be based on use case.
                  * Here message is being sent for first object every
                  * frame_interval(default=30).
@@ -523,6 +545,8 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_metadata_probe(
                 msg_meta->frameId = frame_number;
                 msg_meta->trackingId = obj_meta->object_id;
                 msg_meta->confidence = obj_meta->confidence;
+                msg_meta->embedding.embedding_vector = user_meta_data;
+                msg_meta->embedding.embedding_length = EMBEDDING_VECTOR_SIZE;
                 generate_event_msg_meta(msg_meta, obj_meta->class_id, obj_meta);
 
                 NvDsUserMeta *user_event_meta =