Send encoded images via event message custom meta

src/nv_message_converter.cpp

@@ -15,9 +15,9 @@ bool NvMessageConverter::create_message_converter() {
                  payload_generation_library.c_str(), NULL);
     g_object_set(G_OBJECT(msgconv), "config", msgconv_config_file.c_str(),
                  NULL);
-    g_object_set(G_OBJECT(msgconv), "payload-type", 0,
+    g_object_set(G_OBJECT(msgconv), "payload-type", 1,
                  NULL);  // 0 = DeepStream schema, 1 = minimal schema
-    g_object_set(G_OBJECT(msgconv), "msg2p-newapi", 0,
+    g_object_set(G_OBJECT(msgconv), "msg2p-newapi", 1,
                  NULL);  // use new API; If you want to send images, please set
                          // the "payload-type: 1" and "msg2p-newapi: 1"
     // msg2p-newapi: TRUE for DeepStream 6.x+ (recommended).
@@ -283,7 +283,7 @@ GstPadProbeReturn NvMessageConverter::nvmsgconv_probe_cb_src(
     if (gst_buffer_map(buf, &map, GST_MAP_READ)) {
         // nvmsgconv outputs application/json
         std::string json_str(reinterpret_cast<char *>(map.data), map.size);
-        g_print("nvmsgconv JSON:\n%s\n", json_str.c_str());
+        // g_print("nvmsgconv JSON:\n%s\n", json_str.c_str());
         gst_buffer_unmap(buf, &map);
     }
     return GST_PAD_PROBE_OK;

src/nv_osd_manager.cpp

@@ -44,7 +44,7 @@ char fileObjNameString[1024];
 #define EMBEDDING_VECTOR_SIZE 512
 
 gint msg2p_meta =
-    0;  //"Type of message schema (0=Full, 1=minimal, 2=protobuf), default=0
+    1;  //"Type of message schema (0=Full, 1=minimal, 2=protobuf), default=0
 
 gint NvOsdManager::frame_number = 0;
 bool NvOsdManager::write_full_frame_to_disk = false;
@@ -115,6 +115,19 @@ void NvOsdManager::save_full_frame(NvDsFrameMeta *frame_meta) {
     }
 }
 
+NvDsObjEncOutParams *NvOsdManager::get_full_frame(NvDsFrameMeta *frame_meta) {
+    NvDsObjEncOutParams *enc_jpeg_image = NULL;
+    NvDsUserMetaList *usrMetaList = frame_meta->frame_user_meta_list;
+    while (usrMetaList != NULL) {
+        NvDsUserMeta *usrMetaData = (NvDsUserMeta *)usrMetaList->data;
+        if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
+            enc_jpeg_image = (NvDsObjEncOutParams *)usrMetaData->user_meta_data;
+        }
+        usrMetaList = usrMetaList->next;
+    }
+    return enc_jpeg_image;
+}
+
 void NvOsdManager::save_cropped_objects(NvDsFrameMeta *frame_meta,
                                         NvDsObjectMeta *obj_meta,
                                         guint num_rects) {
@@ -147,6 +160,22 @@ void NvOsdManager::save_cropped_objects(NvDsFrameMeta *frame_meta,
     }
 }
 
+NvDsObjEncOutParams *NvOsdManager::get_cropped_objects(
+    NvDsObjectMeta *obj_meta) {
+    NvDsObjEncOutParams *enc_jpeg_image = NULL;
+    NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list;
+    while (usrMetaList != NULL) {
+        NvDsUserMeta *usrMetaData = (NvDsUserMeta *)usrMetaList->data;
+        if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
+            enc_jpeg_image = (NvDsObjEncOutParams *)usrMetaData->user_meta_data;
+            usrMetaList = NULL;
+        } else {
+            usrMetaList = usrMetaList->next;
+        }
+    }
+    return enc_jpeg_image;
+}
+
 /* This is the buffer probe function that we have registered on the sink pad
  * of the OSD element. All the infer elements in the pipeline shall attach
  * their metadata to the GstBuffer, here we will iterate & process the metadata
@@ -432,10 +461,24 @@ void NvOsdManager::generate_event_msg_meta(gpointer data, gint class_id,
     }
 }
 
-void NvOsdManager::event_message_meta(NvDsBatchMeta *batch_meta,
+void NvOsdManager::event_message_meta(
-                                      NvDsFrameMeta *frame_meta,
+    NvDsBatchMeta *batch_meta, NvDsFrameMeta *frame_meta,
-                                      NvDsObjectMeta *obj_meta,
+    NvDsObjectMeta *obj_meta, float *user_meta_data,
-                                      float *user_meta_data) {
+    std::vector<NvDsObjEncOutParams> encoded_images) {
+    NvDsObjEncOutParams *face_frame = &encoded_images.front();
+    NvDsObjEncOutParams *full_frame = &encoded_images.back();
+    if (encoded_images.size() == 3) {
+        NvDsObjEncOutParams *body_frame = &encoded_images[1];
+        (void)body_frame;
+    }
+
+    gchar *face_encoded_data =
+        g_base64_encode(face_frame->outBuffer, face_frame->outLen);
+    gchar *full_frame_encoded_data =
+        g_base64_encode(full_frame->outBuffer, full_frame->outLen);
+    // gchar *combined = g_strconcat(face_encoded_data, ";",
+    // full_frame_encoded_data, NULL);
+
     NvDsEventMsgMeta *msg_meta =
         (NvDsEventMsgMeta *)g_malloc0(sizeof(NvDsEventMsgMeta));
     msg_meta->bbox.top = obj_meta->rect_params.top;
@@ -447,6 +490,15 @@ void NvOsdManager::event_message_meta(NvDsBatchMeta *batch_meta,
     msg_meta->confidence = obj_meta->confidence;
     msg_meta->embedding.embedding_vector = user_meta_data;
     msg_meta->embedding.embedding_length = EMBEDDING_VECTOR_SIZE;
+    // msg_meta->otherAttrs = combined;
+    msg_meta->otherAttrs =
+        g_strdup_printf("{\"face_frame\":\"%s\",\"full_frame\":\"%s\"}",
+                        face_encoded_data, full_frame_encoded_data);
+    // msg_meta->otherAttrs = g_strdup_printf(
+    // "[\"customMessage\":\"%s\"]",
+    // "face_encoded_data");
+    // msg_meta->otherAttrs = g_strdup("test123;test456");
+
     generate_event_msg_meta(msg_meta, obj_meta->class_id, obj_meta);
 
     NvDsUserMeta *user_event_meta =
@@ -539,6 +591,44 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_metadata_probe(
              * Here it demonstrates how to use / attach that meta data.
              */
 
+            std::vector<NvDsObjEncOutParams> encoded_images;
+            NvDsObjEncOutParams *enc_jpeg_image = NULL;
+            NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list;
+            int num_encode = 0;
+            bool is_meta_type_NVDS_CROP_IMAGE_META = false;
+            while (usrMetaList != NULL) {
+                NvDsUserMeta *user_meta = (NvDsUserMeta *)usrMetaList->data;
+                if (user_meta->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
+                    enc_jpeg_image =
+                        (NvDsObjEncOutParams *)user_meta->user_meta_data;
+                    encoded_images.push_back(*enc_jpeg_image);
+                    num_encode++;
+                    // usrMetaList = NULL;
+                    is_meta_type_NVDS_CROP_IMAGE_META = true;
+                }
+                // else {
+                //     usrMetaList = usrMetaList->next;
+                // }
+                usrMetaList = usrMetaList->next;
+            }
+
+            // // Print results
+            // for (const auto &item : encoded_images) {
+            //     std::cout << " (size=" << item.outLen << ")\n";
+            // }
+
+            if (is_meta_type_NVDS_CROP_IMAGE_META == true) {
+                enc_jpeg_image = get_full_frame(frame_meta);
+                encoded_images.push_back(*enc_jpeg_image);
+            }
+
+            // Sort by size (ascending)
+            std::sort(
+                encoded_images.begin(), encoded_images.end(),
+                [](const NvDsObjEncOutParams &a, const NvDsObjEncOutParams &b) {
+                    return a.outLen < b.outLen;
+                });
+
             NvDsUserMeta *user_meta = NULL;
             NvDsMetaList *l_user_meta = NULL;
             float *user_meta_data = NULL;
@@ -546,18 +636,17 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_metadata_probe(
             for (l_user_meta = obj_meta->obj_user_meta_list;
                  l_user_meta != NULL; l_user_meta = l_user_meta->next) {
                 user_meta = (NvDsUserMeta *)(l_user_meta->data);
-                user_meta_data = (float *)user_meta->user_meta_data;
                 if (user_meta->base_meta.meta_type ==
                     NVDS_USER_EMBEDDING_VECTOR_META) {
-                    std::cout << "NVOSD_EMBEDDING_VECTOR[" << 0
-                              << "]= " << user_meta_data[0] << std::endl;
                     is_meta_type_NVOSD_embedding_vector = true;
+                    user_meta_data = (float *)user_meta->user_meta_data;
                 }
             }
 
-            if (is_meta_type_NVOSD_embedding_vector == true) {
+            if (is_meta_type_NVOSD_embedding_vector == true &&
+                encoded_images.size() >= 2) {
                 event_message_meta(batch_meta, frame_meta, obj_meta,
-                                   user_meta_data);
+                                   user_meta_data, encoded_images);
             }
         }
     }
@@ -598,6 +687,135 @@ void NvOsdManager::meta_free_func_custom(gpointer data, gpointer user_data) {
     g_free(user_meta->user_meta_data);
 }
 
+void NvOsdManager::event_message_custom_meta(
+    NvDsBatchMeta *batch_meta, NvDsFrameMeta *frame_meta,
+    NvDsObjectMeta *obj_meta, float *user_meta_data,
+    std::vector<NvDsObjEncOutParams> encoded_images, guint source_id) {
+    gchar *ts = (gchar *)g_malloc0(MAX_TIME_STAMP_LEN + 1);
+    gchar *width, *height, *top, *left, *object_id, *confidence,
+        *embedding_length, *json_embedding_vector, *src_id;
+    gchar *message_data;
+    NvDsObjEncOutParams *face_frame = &encoded_images.front();
+    NvDsObjEncOutParams *full_frame = &encoded_images.back();
+    if (encoded_images.size() == 3) {
+        NvDsObjEncOutParams *body_frame = &encoded_images[1];
+        (void)body_frame;
+    }
+
+    START_PROFILE;
+    gchar *face_encoded_data =
+        g_base64_encode(face_frame->outBuffer, face_frame->outLen);
+    gchar *full_frame_encoded_data =
+        g_base64_encode(full_frame->outBuffer, full_frame->outLen);
+    // gchar *combined = g_strconcat(face_encoded_data, ";",
+    // full_frame_encoded_data, NULL);
+
+    // encoded_data = g_base64_encode(enc_jpeg_image->outBuffer,
+    //                                 enc_jpeg_image->outLen);
+    generate_ts_rfc3339(ts, MAX_TIME_STAMP_LEN);
+    confidence = g_strdup_printf("%f", obj_meta->confidence);
+    object_id = g_strdup_printf("%lu", obj_meta->object_id);
+    src_id = g_strdup_printf("%d", source_id);
+    top = g_strdup_printf("%f", obj_meta->rect_params.top);
+    left = g_strdup_printf("%f", obj_meta->rect_params.left);
+    width = g_strdup_printf("%f", obj_meta->rect_params.width);
+    height = g_strdup_printf("%f", obj_meta->rect_params.height);
+    embedding_length = g_strdup_printf("%d", EMBEDDING_VECTOR_SIZE);
+
+    // Create a nlohmann::json object
+    nlohmann::json embedding_vector_json;
+    embedding_vector_json["embedding_vector"] = std::vector<float>(
+        user_meta_data, user_meta_data + EMBEDDING_VECTOR_SIZE);
+    std::string json_str_embedding_vector = embedding_vector_json.dump(4);
+    json_embedding_vector = g_strdup(json_str_embedding_vector.c_str());
+
+    /* Image message fields are separated by ";".
+     * Specific Format:
+     * "image;image_format;image_widthximage_height;time;encoded
+     * data;" For Example:
+     * "image;jpg;640x480;2023-07-31T10:20:13;xxxxxxxxxxx"
+     */
+
+    message_data =
+        g_strconcat("image;jpg;",                  // fixed prefix
+                    ";", ts,                       // timestamp
+                    ";", face_encoded_data,        // face image
+                    ";", full_frame_encoded_data,  // full frame image
+                    ";", confidence, ";", src_id, ";", object_id, ";", top, ";",
+                    left, ";", width, ";", height, ";", embedding_length, ";",
+                    json_embedding_vector,  // embedding JSON
+                    NULL);
+    // message_data =
+    //     g_strconcat("image;jpg;", width, "x", height, ";", ts,
+    //                 ";", face_encoded_data, ";", NULL);
+    STOP_PROFILE("Base64 Encode Time ");
+    NvDsCustomMsgInfo *msg_custom_meta =
+        (NvDsCustomMsgInfo *)g_malloc0(sizeof(NvDsCustomMsgInfo));
+    msg_custom_meta->size = strlen(message_data);
+    msg_custom_meta->message = g_strdup(message_data);
+    NvDsUserMeta *user_event_meta_custom =
+        nvds_acquire_user_meta_from_pool(batch_meta);
+    if (user_event_meta_custom) {
+        user_event_meta_custom->user_meta_data = (void *)msg_custom_meta;
+        user_event_meta_custom->base_meta.meta_type = NVDS_CUSTOM_MSG_BLOB;
+        user_event_meta_custom->base_meta.copy_func =
+            (NvDsMetaCopyFunc)meta_copy_func_custom;
+        user_event_meta_custom->base_meta.release_func =
+            (NvDsMetaReleaseFunc)meta_free_func_custom;
+        nvds_add_user_meta_to_frame(frame_meta, user_event_meta_custom);
+        std::cout << "*** send custom message for source id = " << source_id
+                  << " and object_id = " << obj_meta->object_id << " at " << ts
+                  << " ***" << std::endl;
+    } else {
+        g_print(
+            "Error in attaching event meta custom to "
+            "buffer\n");
+        // std::quick_exit(0);
+    }
+
+#ifdef ENABLE_DUMP_FILE
+    gsize size = 0;
+    snprintf(fileObjNameString, 1024, "%s_%d_%d_%s.jpg", ts, frame_number,
+             frame_meta->batch_id, obj_meta->obj_label);
+    guchar *decoded_data = g_base64_decode(face_encoded_data, &size);
+    fp = fopen(fileObjNameString, "wb");
+    if (fp) {
+        fwrite(decoded_data, size, 1, fp);
+        fclose(fp);
+    } else {
+        g_printerr("Could not open file!\n");
+    }
+    g_free(face_encoded_data);
+
+    gsize size = 0;
+    snprintf(fileObjNameString, 1024, "%s_%d_%d_%s.jpg", ts, frame_number,
+             frame_meta->batch_id, obj_meta->obj_label);
+    guchar *decoded_data = g_base64_decode(full_frame_encoded_data, &size);
+    fp = fopen(fileObjNameString, "wb");
+    if (fp) {
+        fwrite(decoded_data, size, 1, fp);
+        fclose(fp);
+    } else {
+        g_printerr("Could not open file!\n");
+    }
+    g_free(full_frame_encoded_data);
+#endif
+
+    g_free(ts);
+    // g_free(message_data);  // after sending/processing
+    g_free(width);
+    g_free(height);
+    g_free(top);
+    g_free(left);
+    g_free(object_id);
+    g_free(src_id);
+    g_free(confidence);
+    g_free(embedding_length);
+    g_free(json_embedding_vector);
+    g_free(face_encoded_data);
+    g_free(full_frame_encoded_data);
+}
+
 GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_image_probe(
     GstPad *pad, GstPadProbeInfo *info, gpointer u_data) {
     (void)pad;
@@ -605,15 +823,10 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_image_probe(
     GstBuffer *buf = (GstBuffer *)info->data;
     NvDsFrameMeta *frame_meta = NULL;
     NvDsMetaList *l_frame, *l_obj;
-    gchar *encoded_data;
-    gchar *message_data;
-    gchar *width, *height;
-    gchar *ts = (gchar *)g_malloc0(MAX_TIME_STAMP_LEN + 1);
 
     NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
     if (!batch_meta) {
         // No batch meta attached.
-        g_free(ts);
         return GST_PAD_PROBE_OK;
     }
 
@@ -639,84 +852,65 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_image_probe(
              * Here images is being sent for first object every
              * frame_interval(default=30).
              */
+            std::vector<NvDsObjEncOutParams> encoded_images;
+            NvDsObjEncOutParams *enc_jpeg_image = NULL;
+            int num_encode = 0;
+            bool is_meta_type_NVDS_CROP_IMAGE_META = false;
             NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list;
             while (usrMetaList != NULL) {
-                NvDsUserMeta *user_event_meta_custom =
-                    nvds_acquire_user_meta_from_pool(batch_meta);
-                NvDsCustomMsgInfo *msg_custom_meta =
-                    (NvDsCustomMsgInfo *)g_malloc0(sizeof(NvDsCustomMsgInfo));
-
                 NvDsUserMeta *usrMetaData = (NvDsUserMeta *)usrMetaList->data;
                 if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
-                    NvDsObjEncOutParams *enc_jpeg_image =
+                    enc_jpeg_image =
                         (NvDsObjEncOutParams *)usrMetaData->user_meta_data;
-                    START_PROFILE;
+                    encoded_images.push_back(*enc_jpeg_image);
-                    encoded_data = g_base64_encode(enc_jpeg_image->outBuffer,
+                    num_encode++;
-                                                   enc_jpeg_image->outLen);
+                    is_meta_type_NVDS_CROP_IMAGE_META = true;
-                    generate_ts_rfc3339(ts, MAX_TIME_STAMP_LEN);
+                    // usrMetaList = NULL;
-                    width = g_strdup_printf(
-                        "%f",
-                        obj_meta->detector_bbox_info.org_bbox_coords.width);
-                    height = g_strdup_printf(
-                        "%f",
-                        obj_meta->detector_bbox_info.org_bbox_coords.height);
-                    /* Image message fields are separated by ";".
-                     * Specific Format:
-                     * "image;image_format;image_widthximage_height;time;encoded
-                     * data;" For Example:
-                     * "image;jpg;640x480;2023-07-31T10:20:13;xxxxxxxxxxx"
-                     */
-                    message_data =
-                        g_strconcat("image;jpg;", width, "x", height, ";", ts,
-                                    ";", encoded_data, ";", NULL);
-                    STOP_PROFILE("Base64 Encode Time ");
-                    msg_custom_meta->size = strlen(message_data);
-                    msg_custom_meta->message = g_strdup(message_data);
-                    if (user_event_meta_custom) {
-                        user_event_meta_custom->user_meta_data =
-                            (void *)msg_custom_meta;
-                        user_event_meta_custom->base_meta.meta_type =
-                            NVDS_CUSTOM_MSG_BLOB;
-                        user_event_meta_custom->base_meta.copy_func =
-                            (NvDsMetaCopyFunc)meta_copy_func_custom;
-                        user_event_meta_custom->base_meta.release_func =
-                            (NvDsMetaReleaseFunc)meta_free_func_custom;
-                        nvds_add_user_meta_to_frame(frame_meta,
-                                                    user_event_meta_custom);
-                    } else {
-                        g_print(
-                            "Error in attaching event meta custom to "
-                            "buffer\n");
                 }
-
+                // else {
-#ifdef ENABLE_DUMP_FILE
+                //     usrMetaList = usrMetaList->next;
-                    gsize size = 0;
+                // }
-                    snprintf(fileObjNameString, 1024, "%s_%d_%d_%s.jpg", ts,
-                             frame_number, frame_meta->batch_id,
-                             obj_meta->obj_label);
-                    guchar *decoded_data = g_base64_decode(encoded_data, &size);
-                    fp = fopen(fileObjNameString, "wb");
-                    if (fp) {
-                        fwrite(decoded_data, size, 1, fp);
-                        fclose(fp);
-                    } else {
-                        g_printerr("Could not open file!\n");
-                    }
-                    g_free(decoded_data);
-#endif
-                    g_free(encoded_data);
-                    g_free(message_data);
-                    g_free(width);
-                    g_free(height);
-                    usrMetaList = NULL;
-                } else {
                 usrMetaList = usrMetaList->next;
             }
-            }
+
-        }
+            // // Print results
+            // for (const auto &item : encoded_images) {
+            //     std::cout << " (size=" << item.outLen << ")\n";
+            // }
+
+            if (is_meta_type_NVDS_CROP_IMAGE_META == true) {
+                enc_jpeg_image = get_full_frame(frame_meta);
+                encoded_images.push_back(*enc_jpeg_image);
             }
 
-    g_free(ts);
+            // Sort by size (ascending)
+            std::sort(
+                encoded_images.begin(), encoded_images.end(),
+                [](const NvDsObjEncOutParams &a, const NvDsObjEncOutParams &b) {
+                    return a.outLen < b.outLen;
+                });
+
+            NvDsUserMeta *user_meta = NULL;
+            NvDsMetaList *l_user_meta = NULL;
+            float *user_meta_data = NULL;
+            bool is_meta_type_NVOSD_embedding_vector = false;
+            for (l_user_meta = obj_meta->obj_user_meta_list;
+                 l_user_meta != NULL; l_user_meta = l_user_meta->next) {
+                user_meta = (NvDsUserMeta *)(l_user_meta->data);
+                if (user_meta->base_meta.meta_type ==
+                    NVDS_USER_EMBEDDING_VECTOR_META) {
+                    is_meta_type_NVOSD_embedding_vector = true;
+                    user_meta_data = (float *)user_meta->user_meta_data;
+                }
+            }
+            if (is_meta_type_NVOSD_embedding_vector == true &&
+                encoded_images.size() >= 2) {
+                event_message_custom_meta(batch_meta, frame_meta, obj_meta,
+                                          user_meta_data, encoded_images,
+                                          frame_meta->source_id);
+            }
+        }
+    }
     frame_number++;
 
     return GST_PAD_PROBE_OK;

src/nv_osd_manager.hpp

@@ -11,6 +11,10 @@
 class NvOsdManager {
    private:
    public:
+    //    struct Item {
+    //     std::string name;
+    //     int size;
+    //     };
     GstElement *nvosd = NULL;
     static bool write_full_frame_to_disk, write_cropped_objects_to_disk;
     NvOsdManager();
@@ -41,5 +45,12 @@ class NvOsdManager {
     static void generate_face_meta(gpointer);
     static void generate_person_meta(gpointer);
     static void event_message_meta(NvDsBatchMeta *, NvDsFrameMeta *,
-                                   NvDsObjectMeta *, float *);
+                                   NvDsObjectMeta *, float *,
+                                   std::vector<NvDsObjEncOutParams>);
+    static void event_message_custom_meta(NvDsBatchMeta *, NvDsFrameMeta *,
+                                          NvDsObjectMeta *, float *,
+                                          std::vector<NvDsObjEncOutParams>,
+                                          guint);
+    static NvDsObjEncOutParams *get_full_frame(NvDsFrameMeta *);
+    static NvDsObjEncOutParams *get_cropped_objects(NvDsObjectMeta *);
 };