From 77a61986f4d18d6b1b846fb589d8b2ded4621317 Mon Sep 17 00:00:00 2001 From: Barzan Hayati Date: Sat, 20 Sep 2025 13:08:23 +0000 Subject: [PATCH] Send encoded images via event message custom meta --- src/nv_message_converter.cpp | 6 +- src/nv_osd_manager.cpp | 362 +++++++++++++++++++++++++++-------- src/nv_osd_manager.hpp | 13 +- 3 files changed, 293 insertions(+), 88 deletions(-) diff --git a/src/nv_message_converter.cpp b/src/nv_message_converter.cpp index 708eeb5..bd206f3 100644 --- a/src/nv_message_converter.cpp +++ b/src/nv_message_converter.cpp @@ -15,9 +15,9 @@ bool NvMessageConverter::create_message_converter() { payload_generation_library.c_str(), NULL); g_object_set(G_OBJECT(msgconv), "config", msgconv_config_file.c_str(), NULL); - g_object_set(G_OBJECT(msgconv), "payload-type", 0, + g_object_set(G_OBJECT(msgconv), "payload-type", 1, NULL); // 0 = DeepStream schema, 1 = minimal schema - g_object_set(G_OBJECT(msgconv), "msg2p-newapi", 0, + g_object_set(G_OBJECT(msgconv), "msg2p-newapi", 1, NULL); // use new API; If you want to send images, please set // the "payload-type: 1" and "msg2p-newapi: 1" // msg2p-newapi: TRUE for DeepStream 6.x+ (recommended). @@ -283,7 +283,7 @@ GstPadProbeReturn NvMessageConverter::nvmsgconv_probe_cb_src( if (gst_buffer_map(buf, &map, GST_MAP_READ)) { // nvmsgconv outputs application/json std::string json_str(reinterpret_cast(map.data), map.size); - g_print("nvmsgconv JSON:\n%s\n", json_str.c_str()); + // g_print("nvmsgconv JSON:\n%s\n", json_str.c_str()); gst_buffer_unmap(buf, &map); } return GST_PAD_PROBE_OK; diff --git a/src/nv_osd_manager.cpp b/src/nv_osd_manager.cpp index bb940f4..b8b4f53 100644 --- a/src/nv_osd_manager.cpp +++ b/src/nv_osd_manager.cpp @@ -44,7 +44,7 @@ char fileObjNameString[1024]; #define EMBEDDING_VECTOR_SIZE 512 gint msg2p_meta = - 0; //"Type of message schema (0=Full, 1=minimal, 2=protobuf), default=0 + 1; //"Type of message schema (0=Full, 1=minimal, 2=protobuf), default=0 gint NvOsdManager::frame_number = 0; bool NvOsdManager::write_full_frame_to_disk = false; @@ -115,6 +115,19 @@ void NvOsdManager::save_full_frame(NvDsFrameMeta *frame_meta) { } } +NvDsObjEncOutParams *NvOsdManager::get_full_frame(NvDsFrameMeta *frame_meta) { + NvDsObjEncOutParams *enc_jpeg_image = NULL; + NvDsUserMetaList *usrMetaList = frame_meta->frame_user_meta_list; + while (usrMetaList != NULL) { + NvDsUserMeta *usrMetaData = (NvDsUserMeta *)usrMetaList->data; + if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) { + enc_jpeg_image = (NvDsObjEncOutParams *)usrMetaData->user_meta_data; + } + usrMetaList = usrMetaList->next; + } + return enc_jpeg_image; +} + void NvOsdManager::save_cropped_objects(NvDsFrameMeta *frame_meta, NvDsObjectMeta *obj_meta, guint num_rects) { @@ -147,6 +160,22 @@ void NvOsdManager::save_cropped_objects(NvDsFrameMeta *frame_meta, } } +NvDsObjEncOutParams *NvOsdManager::get_cropped_objects( + NvDsObjectMeta *obj_meta) { + NvDsObjEncOutParams *enc_jpeg_image = NULL; + NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list; + while (usrMetaList != NULL) { + NvDsUserMeta *usrMetaData = (NvDsUserMeta *)usrMetaList->data; + if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) { + enc_jpeg_image = (NvDsObjEncOutParams *)usrMetaData->user_meta_data; + usrMetaList = NULL; + } else { + usrMetaList = usrMetaList->next; + } + } + return enc_jpeg_image; +} + /* This is the buffer probe function that we have registered on the sink pad * of the OSD element. All the infer elements in the pipeline shall attach * their metadata to the GstBuffer, here we will iterate & process the metadata @@ -432,10 +461,24 @@ void NvOsdManager::generate_event_msg_meta(gpointer data, gint class_id, } } -void NvOsdManager::event_message_meta(NvDsBatchMeta *batch_meta, - NvDsFrameMeta *frame_meta, - NvDsObjectMeta *obj_meta, - float *user_meta_data) { +void NvOsdManager::event_message_meta( + NvDsBatchMeta *batch_meta, NvDsFrameMeta *frame_meta, + NvDsObjectMeta *obj_meta, float *user_meta_data, + std::vector encoded_images) { + NvDsObjEncOutParams *face_frame = &encoded_images.front(); + NvDsObjEncOutParams *full_frame = &encoded_images.back(); + if (encoded_images.size() == 3) { + NvDsObjEncOutParams *body_frame = &encoded_images[1]; + (void)body_frame; + } + + gchar *face_encoded_data = + g_base64_encode(face_frame->outBuffer, face_frame->outLen); + gchar *full_frame_encoded_data = + g_base64_encode(full_frame->outBuffer, full_frame->outLen); + // gchar *combined = g_strconcat(face_encoded_data, ";", + // full_frame_encoded_data, NULL); + NvDsEventMsgMeta *msg_meta = (NvDsEventMsgMeta *)g_malloc0(sizeof(NvDsEventMsgMeta)); msg_meta->bbox.top = obj_meta->rect_params.top; @@ -447,6 +490,15 @@ void NvOsdManager::event_message_meta(NvDsBatchMeta *batch_meta, msg_meta->confidence = obj_meta->confidence; msg_meta->embedding.embedding_vector = user_meta_data; msg_meta->embedding.embedding_length = EMBEDDING_VECTOR_SIZE; + // msg_meta->otherAttrs = combined; + msg_meta->otherAttrs = + g_strdup_printf("{\"face_frame\":\"%s\",\"full_frame\":\"%s\"}", + face_encoded_data, full_frame_encoded_data); + // msg_meta->otherAttrs = g_strdup_printf( + // "[\"customMessage\":\"%s\"]", + // "face_encoded_data"); + // msg_meta->otherAttrs = g_strdup("test123;test456"); + generate_event_msg_meta(msg_meta, obj_meta->class_id, obj_meta); NvDsUserMeta *user_event_meta = @@ -539,6 +591,44 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_metadata_probe( * Here it demonstrates how to use / attach that meta data. */ + std::vector encoded_images; + NvDsObjEncOutParams *enc_jpeg_image = NULL; + NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list; + int num_encode = 0; + bool is_meta_type_NVDS_CROP_IMAGE_META = false; + while (usrMetaList != NULL) { + NvDsUserMeta *user_meta = (NvDsUserMeta *)usrMetaList->data; + if (user_meta->base_meta.meta_type == NVDS_CROP_IMAGE_META) { + enc_jpeg_image = + (NvDsObjEncOutParams *)user_meta->user_meta_data; + encoded_images.push_back(*enc_jpeg_image); + num_encode++; + // usrMetaList = NULL; + is_meta_type_NVDS_CROP_IMAGE_META = true; + } + // else { + // usrMetaList = usrMetaList->next; + // } + usrMetaList = usrMetaList->next; + } + + // // Print results + // for (const auto &item : encoded_images) { + // std::cout << " (size=" << item.outLen << ")\n"; + // } + + if (is_meta_type_NVDS_CROP_IMAGE_META == true) { + enc_jpeg_image = get_full_frame(frame_meta); + encoded_images.push_back(*enc_jpeg_image); + } + + // Sort by size (ascending) + std::sort( + encoded_images.begin(), encoded_images.end(), + [](const NvDsObjEncOutParams &a, const NvDsObjEncOutParams &b) { + return a.outLen < b.outLen; + }); + NvDsUserMeta *user_meta = NULL; NvDsMetaList *l_user_meta = NULL; float *user_meta_data = NULL; @@ -546,18 +636,17 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_metadata_probe( for (l_user_meta = obj_meta->obj_user_meta_list; l_user_meta != NULL; l_user_meta = l_user_meta->next) { user_meta = (NvDsUserMeta *)(l_user_meta->data); - user_meta_data = (float *)user_meta->user_meta_data; if (user_meta->base_meta.meta_type == NVDS_USER_EMBEDDING_VECTOR_META) { - std::cout << "NVOSD_EMBEDDING_VECTOR[" << 0 - << "]= " << user_meta_data[0] << std::endl; is_meta_type_NVOSD_embedding_vector = true; + user_meta_data = (float *)user_meta->user_meta_data; } } - if (is_meta_type_NVOSD_embedding_vector == true) { + if (is_meta_type_NVOSD_embedding_vector == true && + encoded_images.size() >= 2) { event_message_meta(batch_meta, frame_meta, obj_meta, - user_meta_data); + user_meta_data, encoded_images); } } } @@ -598,6 +687,135 @@ void NvOsdManager::meta_free_func_custom(gpointer data, gpointer user_data) { g_free(user_meta->user_meta_data); } +void NvOsdManager::event_message_custom_meta( + NvDsBatchMeta *batch_meta, NvDsFrameMeta *frame_meta, + NvDsObjectMeta *obj_meta, float *user_meta_data, + std::vector encoded_images, guint source_id) { + gchar *ts = (gchar *)g_malloc0(MAX_TIME_STAMP_LEN + 1); + gchar *width, *height, *top, *left, *object_id, *confidence, + *embedding_length, *json_embedding_vector, *src_id; + gchar *message_data; + NvDsObjEncOutParams *face_frame = &encoded_images.front(); + NvDsObjEncOutParams *full_frame = &encoded_images.back(); + if (encoded_images.size() == 3) { + NvDsObjEncOutParams *body_frame = &encoded_images[1]; + (void)body_frame; + } + + START_PROFILE; + gchar *face_encoded_data = + g_base64_encode(face_frame->outBuffer, face_frame->outLen); + gchar *full_frame_encoded_data = + g_base64_encode(full_frame->outBuffer, full_frame->outLen); + // gchar *combined = g_strconcat(face_encoded_data, ";", + // full_frame_encoded_data, NULL); + + // encoded_data = g_base64_encode(enc_jpeg_image->outBuffer, + // enc_jpeg_image->outLen); + generate_ts_rfc3339(ts, MAX_TIME_STAMP_LEN); + confidence = g_strdup_printf("%f", obj_meta->confidence); + object_id = g_strdup_printf("%lu", obj_meta->object_id); + src_id = g_strdup_printf("%d", source_id); + top = g_strdup_printf("%f", obj_meta->rect_params.top); + left = g_strdup_printf("%f", obj_meta->rect_params.left); + width = g_strdup_printf("%f", obj_meta->rect_params.width); + height = g_strdup_printf("%f", obj_meta->rect_params.height); + embedding_length = g_strdup_printf("%d", EMBEDDING_VECTOR_SIZE); + + // Create a nlohmann::json object + nlohmann::json embedding_vector_json; + embedding_vector_json["embedding_vector"] = std::vector( + user_meta_data, user_meta_data + EMBEDDING_VECTOR_SIZE); + std::string json_str_embedding_vector = embedding_vector_json.dump(4); + json_embedding_vector = g_strdup(json_str_embedding_vector.c_str()); + + /* Image message fields are separated by ";". + * Specific Format: + * "image;image_format;image_widthximage_height;time;encoded + * data;" For Example: + * "image;jpg;640x480;2023-07-31T10:20:13;xxxxxxxxxxx" + */ + + message_data = + g_strconcat("image;jpg;", // fixed prefix + ";", ts, // timestamp + ";", face_encoded_data, // face image + ";", full_frame_encoded_data, // full frame image + ";", confidence, ";", src_id, ";", object_id, ";", top, ";", + left, ";", width, ";", height, ";", embedding_length, ";", + json_embedding_vector, // embedding JSON + NULL); + // message_data = + // g_strconcat("image;jpg;", width, "x", height, ";", ts, + // ";", face_encoded_data, ";", NULL); + STOP_PROFILE("Base64 Encode Time "); + NvDsCustomMsgInfo *msg_custom_meta = + (NvDsCustomMsgInfo *)g_malloc0(sizeof(NvDsCustomMsgInfo)); + msg_custom_meta->size = strlen(message_data); + msg_custom_meta->message = g_strdup(message_data); + NvDsUserMeta *user_event_meta_custom = + nvds_acquire_user_meta_from_pool(batch_meta); + if (user_event_meta_custom) { + user_event_meta_custom->user_meta_data = (void *)msg_custom_meta; + user_event_meta_custom->base_meta.meta_type = NVDS_CUSTOM_MSG_BLOB; + user_event_meta_custom->base_meta.copy_func = + (NvDsMetaCopyFunc)meta_copy_func_custom; + user_event_meta_custom->base_meta.release_func = + (NvDsMetaReleaseFunc)meta_free_func_custom; + nvds_add_user_meta_to_frame(frame_meta, user_event_meta_custom); + std::cout << "*** send custom message for source id = " << source_id + << " and object_id = " << obj_meta->object_id << " at " << ts + << " ***" << std::endl; + } else { + g_print( + "Error in attaching event meta custom to " + "buffer\n"); + // std::quick_exit(0); + } + +#ifdef ENABLE_DUMP_FILE + gsize size = 0; + snprintf(fileObjNameString, 1024, "%s_%d_%d_%s.jpg", ts, frame_number, + frame_meta->batch_id, obj_meta->obj_label); + guchar *decoded_data = g_base64_decode(face_encoded_data, &size); + fp = fopen(fileObjNameString, "wb"); + if (fp) { + fwrite(decoded_data, size, 1, fp); + fclose(fp); + } else { + g_printerr("Could not open file!\n"); + } + g_free(face_encoded_data); + + gsize size = 0; + snprintf(fileObjNameString, 1024, "%s_%d_%d_%s.jpg", ts, frame_number, + frame_meta->batch_id, obj_meta->obj_label); + guchar *decoded_data = g_base64_decode(full_frame_encoded_data, &size); + fp = fopen(fileObjNameString, "wb"); + if (fp) { + fwrite(decoded_data, size, 1, fp); + fclose(fp); + } else { + g_printerr("Could not open file!\n"); + } + g_free(full_frame_encoded_data); +#endif + + g_free(ts); + // g_free(message_data); // after sending/processing + g_free(width); + g_free(height); + g_free(top); + g_free(left); + g_free(object_id); + g_free(src_id); + g_free(confidence); + g_free(embedding_length); + g_free(json_embedding_vector); + g_free(face_encoded_data); + g_free(full_frame_encoded_data); +} + GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_image_probe( GstPad *pad, GstPadProbeInfo *info, gpointer u_data) { (void)pad; @@ -605,15 +823,10 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_image_probe( GstBuffer *buf = (GstBuffer *)info->data; NvDsFrameMeta *frame_meta = NULL; NvDsMetaList *l_frame, *l_obj; - gchar *encoded_data; - gchar *message_data; - gchar *width, *height; - gchar *ts = (gchar *)g_malloc0(MAX_TIME_STAMP_LEN + 1); NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf); if (!batch_meta) { // No batch meta attached. - g_free(ts); return GST_PAD_PROBE_OK; } @@ -639,84 +852,65 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_image_probe( * Here images is being sent for first object every * frame_interval(default=30). */ + std::vector encoded_images; + NvDsObjEncOutParams *enc_jpeg_image = NULL; + int num_encode = 0; + bool is_meta_type_NVDS_CROP_IMAGE_META = false; NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list; while (usrMetaList != NULL) { - NvDsUserMeta *user_event_meta_custom = - nvds_acquire_user_meta_from_pool(batch_meta); - NvDsCustomMsgInfo *msg_custom_meta = - (NvDsCustomMsgInfo *)g_malloc0(sizeof(NvDsCustomMsgInfo)); - NvDsUserMeta *usrMetaData = (NvDsUserMeta *)usrMetaList->data; if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) { - NvDsObjEncOutParams *enc_jpeg_image = + enc_jpeg_image = (NvDsObjEncOutParams *)usrMetaData->user_meta_data; - START_PROFILE; - encoded_data = g_base64_encode(enc_jpeg_image->outBuffer, - enc_jpeg_image->outLen); - generate_ts_rfc3339(ts, MAX_TIME_STAMP_LEN); - width = g_strdup_printf( - "%f", - obj_meta->detector_bbox_info.org_bbox_coords.width); - height = g_strdup_printf( - "%f", - obj_meta->detector_bbox_info.org_bbox_coords.height); - /* Image message fields are separated by ";". - * Specific Format: - * "image;image_format;image_widthximage_height;time;encoded - * data;" For Example: - * "image;jpg;640x480;2023-07-31T10:20:13;xxxxxxxxxxx" - */ - message_data = - g_strconcat("image;jpg;", width, "x", height, ";", ts, - ";", encoded_data, ";", NULL); - STOP_PROFILE("Base64 Encode Time "); - msg_custom_meta->size = strlen(message_data); - msg_custom_meta->message = g_strdup(message_data); - if (user_event_meta_custom) { - user_event_meta_custom->user_meta_data = - (void *)msg_custom_meta; - user_event_meta_custom->base_meta.meta_type = - NVDS_CUSTOM_MSG_BLOB; - user_event_meta_custom->base_meta.copy_func = - (NvDsMetaCopyFunc)meta_copy_func_custom; - user_event_meta_custom->base_meta.release_func = - (NvDsMetaReleaseFunc)meta_free_func_custom; - nvds_add_user_meta_to_frame(frame_meta, - user_event_meta_custom); - } else { - g_print( - "Error in attaching event meta custom to " - "buffer\n"); - } - -#ifdef ENABLE_DUMP_FILE - gsize size = 0; - snprintf(fileObjNameString, 1024, "%s_%d_%d_%s.jpg", ts, - frame_number, frame_meta->batch_id, - obj_meta->obj_label); - guchar *decoded_data = g_base64_decode(encoded_data, &size); - fp = fopen(fileObjNameString, "wb"); - if (fp) { - fwrite(decoded_data, size, 1, fp); - fclose(fp); - } else { - g_printerr("Could not open file!\n"); - } - g_free(decoded_data); -#endif - g_free(encoded_data); - g_free(message_data); - g_free(width); - g_free(height); - usrMetaList = NULL; - } else { - usrMetaList = usrMetaList->next; + encoded_images.push_back(*enc_jpeg_image); + num_encode++; + is_meta_type_NVDS_CROP_IMAGE_META = true; + // usrMetaList = NULL; } + // else { + // usrMetaList = usrMetaList->next; + // } + usrMetaList = usrMetaList->next; + } + + // // Print results + // for (const auto &item : encoded_images) { + // std::cout << " (size=" << item.outLen << ")\n"; + // } + + if (is_meta_type_NVDS_CROP_IMAGE_META == true) { + enc_jpeg_image = get_full_frame(frame_meta); + encoded_images.push_back(*enc_jpeg_image); + } + + // Sort by size (ascending) + std::sort( + encoded_images.begin(), encoded_images.end(), + [](const NvDsObjEncOutParams &a, const NvDsObjEncOutParams &b) { + return a.outLen < b.outLen; + }); + + NvDsUserMeta *user_meta = NULL; + NvDsMetaList *l_user_meta = NULL; + float *user_meta_data = NULL; + bool is_meta_type_NVOSD_embedding_vector = false; + for (l_user_meta = obj_meta->obj_user_meta_list; + l_user_meta != NULL; l_user_meta = l_user_meta->next) { + user_meta = (NvDsUserMeta *)(l_user_meta->data); + if (user_meta->base_meta.meta_type == + NVDS_USER_EMBEDDING_VECTOR_META) { + is_meta_type_NVOSD_embedding_vector = true; + user_meta_data = (float *)user_meta->user_meta_data; + } + } + if (is_meta_type_NVOSD_embedding_vector == true && + encoded_images.size() >= 2) { + event_message_custom_meta(batch_meta, frame_meta, obj_meta, + user_meta_data, encoded_images, + frame_meta->source_id); } } } - - g_free(ts); frame_number++; return GST_PAD_PROBE_OK; diff --git a/src/nv_osd_manager.hpp b/src/nv_osd_manager.hpp index f095f2c..011ce0a 100644 --- a/src/nv_osd_manager.hpp +++ b/src/nv_osd_manager.hpp @@ -11,6 +11,10 @@ class NvOsdManager { private: public: + // struct Item { + // std::string name; + // int size; + // }; GstElement *nvosd = NULL; static bool write_full_frame_to_disk, write_cropped_objects_to_disk; NvOsdManager(); @@ -41,5 +45,12 @@ class NvOsdManager { static void generate_face_meta(gpointer); static void generate_person_meta(gpointer); static void event_message_meta(NvDsBatchMeta *, NvDsFrameMeta *, - NvDsObjectMeta *, float *); + NvDsObjectMeta *, float *, + std::vector); + static void event_message_custom_meta(NvDsBatchMeta *, NvDsFrameMeta *, + NvDsObjectMeta *, float *, + std::vector, + guint); + static NvDsObjEncOutParams *get_full_frame(NvDsFrameMeta *); + static NvDsObjEncOutParams *get_cropped_objects(NvDsObjectMeta *); }; \ No newline at end of file