haiati/FaceRecognition
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Send encoded images via event message custom meta

Browse Source
This commit is contained in:
Barzan Hayati 2025-09-20 13:08:23 +00:00
parent 6522fbd46e
commit 77a61986f4
3 changed files with 293 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/nv_message_converter.cpp
View File

@ -15,9 +15,9 @@ bool NvMessageConverter::create_message_converter() {
payload_generation_library.c_str(), NULL);
g_object_set(G_OBJECT(msgconv), "config", msgconv_config_file.c_str(),
NULL);
g_object_set(G_OBJECT(msgconv), "payload-type", 0,
g_object_set(G_OBJECT(msgconv), "payload-type", 1,
NULL); // 0 = DeepStream schema, 1 = minimal schema
g_object_set(G_OBJECT(msgconv), "msg2p-newapi", 0,
g_object_set(G_OBJECT(msgconv), "msg2p-newapi", 1,
NULL); // use new API; If you want to send images, please set
// the "payload-type: 1" and "msg2p-newapi: 1"
// msg2p-newapi: TRUE for DeepStream 6.x+ (recommended).
@ -283,7 +283,7 @@ GstPadProbeReturn NvMessageConverter::nvmsgconv_probe_cb_src(
if (gst_buffer_map(buf, &map, GST_MAP_READ)) {
// nvmsgconv outputs application/json
std::string json_str(reinterpret_cast<char *>(map.data), map.size);
g_print("nvmsgconv JSON:\n%s\n", json_str.c_str());
// g_print("nvmsgconv JSON:\n%s\n", json_str.c_str());
gst_buffer_unmap(buf, &map);
}
return GST_PAD_PROBE_OK;

362
src/nv_osd_manager.cpp
View File

@ -44,7 +44,7 @@ char fileObjNameString[1024];
#define EMBEDDING_VECTOR_SIZE 512
gint msg2p_meta =
0; //"Type of message schema (0=Full, 1=minimal, 2=protobuf), default=0
1; //"Type of message schema (0=Full, 1=minimal, 2=protobuf), default=0
gint NvOsdManager::frame_number = 0;
bool NvOsdManager::write_full_frame_to_disk = false;
@ -115,6 +115,19 @@ void NvOsdManager::save_full_frame(NvDsFrameMeta *frame_meta) {
}
}
NvDsObjEncOutParams *NvOsdManager::get_full_frame(NvDsFrameMeta *frame_meta) {
NvDsObjEncOutParams *enc_jpeg_image = NULL;
NvDsUserMetaList *usrMetaList = frame_meta->frame_user_meta_list;
while (usrMetaList != NULL) {
NvDsUserMeta *usrMetaData = (NvDsUserMeta *)usrMetaList->data;
if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
enc_jpeg_image = (NvDsObjEncOutParams *)usrMetaData->user_meta_data;
}
usrMetaList = usrMetaList->next;
}
return enc_jpeg_image;
}
void NvOsdManager::save_cropped_objects(NvDsFrameMeta *frame_meta,
NvDsObjectMeta *obj_meta,
guint num_rects) {
@ -147,6 +160,22 @@ void NvOsdManager::save_cropped_objects(NvDsFrameMeta *frame_meta,
}
}
NvDsObjEncOutParams *NvOsdManager::get_cropped_objects(
NvDsObjectMeta *obj_meta) {
NvDsObjEncOutParams *enc_jpeg_image = NULL;
NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list;
while (usrMetaList != NULL) {
NvDsUserMeta *usrMetaData = (NvDsUserMeta *)usrMetaList->data;
if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
enc_jpeg_image = (NvDsObjEncOutParams *)usrMetaData->user_meta_data;
usrMetaList = NULL;
} else {
usrMetaList = usrMetaList->next;
}
}
return enc_jpeg_image;
}
/* This is the buffer probe function that we have registered on the sink pad
* of the OSD element. All the infer elements in the pipeline shall attach
* their metadata to the GstBuffer, here we will iterate & process the metadata
@ -432,10 +461,24 @@ void NvOsdManager::generate_event_msg_meta(gpointer data, gint class_id,
}
}
void NvOsdManager::event_message_meta(NvDsBatchMeta *batch_meta,
NvDsFrameMeta *frame_meta,
NvDsObjectMeta *obj_meta,
float *user_meta_data) {
void NvOsdManager::event_message_meta(
NvDsBatchMeta *batch_meta, NvDsFrameMeta *frame_meta,
NvDsObjectMeta *obj_meta, float *user_meta_data,
std::vector<NvDsObjEncOutParams> encoded_images) {
NvDsObjEncOutParams *face_frame = &encoded_images.front();
NvDsObjEncOutParams *full_frame = &encoded_images.back();
if (encoded_images.size() == 3) {
NvDsObjEncOutParams *body_frame = &encoded_images[1];
(void)body_frame;
}
gchar *face_encoded_data =
g_base64_encode(face_frame->outBuffer, face_frame->outLen);
gchar *full_frame_encoded_data =
g_base64_encode(full_frame->outBuffer, full_frame->outLen);
// gchar *combined = g_strconcat(face_encoded_data, ";",
// full_frame_encoded_data, NULL);
NvDsEventMsgMeta *msg_meta =
(NvDsEventMsgMeta *)g_malloc0(sizeof(NvDsEventMsgMeta));
msg_meta->bbox.top = obj_meta->rect_params.top;
@ -447,6 +490,15 @@ void NvOsdManager::event_message_meta(NvDsBatchMeta *batch_meta,
msg_meta->confidence = obj_meta->confidence;
msg_meta->embedding.embedding_vector = user_meta_data;
msg_meta->embedding.embedding_length = EMBEDDING_VECTOR_SIZE;
// msg_meta->otherAttrs = combined;
msg_meta->otherAttrs =
g_strdup_printf("{\"face_frame\":\"%s\",\"full_frame\":\"%s\"}",
face_encoded_data, full_frame_encoded_data);
// msg_meta->otherAttrs = g_strdup_printf(
// "[\"customMessage\":\"%s\"]",
// "face_encoded_data");
// msg_meta->otherAttrs = g_strdup("test123;test456");
generate_event_msg_meta(msg_meta, obj_meta->class_id, obj_meta);
NvDsUserMeta *user_event_meta =
@ -539,6 +591,44 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_metadata_probe(
* Here it demonstrates how to use / attach that meta data.
*/
std::vector<NvDsObjEncOutParams> encoded_images;
NvDsObjEncOutParams *enc_jpeg_image = NULL;
NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list;
int num_encode = 0;
bool is_meta_type_NVDS_CROP_IMAGE_META = false;
while (usrMetaList != NULL) {
NvDsUserMeta *user_meta = (NvDsUserMeta *)usrMetaList->data;
if (user_meta->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
enc_jpeg_image =
(NvDsObjEncOutParams *)user_meta->user_meta_data;
encoded_images.push_back(*enc_jpeg_image);
num_encode++;
// usrMetaList = NULL;
is_meta_type_NVDS_CROP_IMAGE_META = true;
}
// else {
// usrMetaList = usrMetaList->next;
// }
usrMetaList = usrMetaList->next;
}
// // Print results
// for (const auto &item : encoded_images) {
// std::cout << " (size=" << item.outLen << ")\n";
// }
if (is_meta_type_NVDS_CROP_IMAGE_META == true) {
enc_jpeg_image = get_full_frame(frame_meta);
encoded_images.push_back(*enc_jpeg_image);
}
// Sort by size (ascending)
std::sort(
encoded_images.begin(), encoded_images.end(),
[](const NvDsObjEncOutParams &a, const NvDsObjEncOutParams &b) {
return a.outLen < b.outLen;
});
NvDsUserMeta *user_meta = NULL;
NvDsMetaList *l_user_meta = NULL;
float *user_meta_data = NULL;
@ -546,18 +636,17 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_metadata_probe(
for (l_user_meta = obj_meta->obj_user_meta_list;
l_user_meta != NULL; l_user_meta = l_user_meta->next) {
user_meta = (NvDsUserMeta *)(l_user_meta->data);
user_meta_data = (float *)user_meta->user_meta_data;
if (user_meta->base_meta.meta_type ==
NVDS_USER_EMBEDDING_VECTOR_META) {
std::cout << "NVOSD_EMBEDDING_VECTOR[" << 0
<< "]= " << user_meta_data[0] << std::endl;
is_meta_type_NVOSD_embedding_vector = true;
user_meta_data = (float *)user_meta->user_meta_data;
}
}
if (is_meta_type_NVOSD_embedding_vector == true) {
if (is_meta_type_NVOSD_embedding_vector == true &&
encoded_images.size() >= 2) {
event_message_meta(batch_meta, frame_meta, obj_meta,
user_meta_data);
user_meta_data, encoded_images);
}
}
}
@ -598,6 +687,135 @@ void NvOsdManager::meta_free_func_custom(gpointer data, gpointer user_data) {
g_free(user_meta->user_meta_data);
}
void NvOsdManager::event_message_custom_meta(
NvDsBatchMeta *batch_meta, NvDsFrameMeta *frame_meta,
NvDsObjectMeta *obj_meta, float *user_meta_data,
std::vector<NvDsObjEncOutParams> encoded_images, guint source_id) {
gchar *ts = (gchar *)g_malloc0(MAX_TIME_STAMP_LEN + 1);
gchar *width, *height, *top, *left, *object_id, *confidence,
*embedding_length, *json_embedding_vector, *src_id;
gchar *message_data;
NvDsObjEncOutParams *face_frame = &encoded_images.front();
NvDsObjEncOutParams *full_frame = &encoded_images.back();
if (encoded_images.size() == 3) {
NvDsObjEncOutParams *body_frame = &encoded_images[1];
(void)body_frame;
}
START_PROFILE;
gchar *face_encoded_data =
g_base64_encode(face_frame->outBuffer, face_frame->outLen);
gchar *full_frame_encoded_data =
g_base64_encode(full_frame->outBuffer, full_frame->outLen);
// gchar *combined = g_strconcat(face_encoded_data, ";",
// full_frame_encoded_data, NULL);
// encoded_data = g_base64_encode(enc_jpeg_image->outBuffer,
// enc_jpeg_image->outLen);
generate_ts_rfc3339(ts, MAX_TIME_STAMP_LEN);
confidence = g_strdup_printf("%f", obj_meta->confidence);
object_id = g_strdup_printf("%lu", obj_meta->object_id);
src_id = g_strdup_printf("%d", source_id);
top = g_strdup_printf("%f", obj_meta->rect_params.top);
left = g_strdup_printf("%f", obj_meta->rect_params.left);
width = g_strdup_printf("%f", obj_meta->rect_params.width);
height = g_strdup_printf("%f", obj_meta->rect_params.height);
embedding_length = g_strdup_printf("%d", EMBEDDING_VECTOR_SIZE);
// Create a nlohmann::json object
nlohmann::json embedding_vector_json;
embedding_vector_json["embedding_vector"] = std::vector<float>(
user_meta_data, user_meta_data + EMBEDDING_VECTOR_SIZE);
std::string json_str_embedding_vector = embedding_vector_json.dump(4);
json_embedding_vector = g_strdup(json_str_embedding_vector.c_str());
/* Image message fields are separated by ";".
* Specific Format:
* "image;image_format;image_widthximage_height;time;encoded
* data;" For Example:
* "image;jpg;640x480;2023-07-31T10:20:13;xxxxxxxxxxx"
*/
message_data =
g_strconcat("image;jpg;", // fixed prefix
";", ts, // timestamp
";", face_encoded_data, // face image
";", full_frame_encoded_data, // full frame image
";", confidence, ";", src_id, ";", object_id, ";", top, ";",
left, ";", width, ";", height, ";", embedding_length, ";",
json_embedding_vector, // embedding JSON
NULL);
// message_data =
// g_strconcat("image;jpg;", width, "x", height, ";", ts,
// ";", face_encoded_data, ";", NULL);
STOP_PROFILE("Base64 Encode Time ");
NvDsCustomMsgInfo *msg_custom_meta =
(NvDsCustomMsgInfo *)g_malloc0(sizeof(NvDsCustomMsgInfo));
msg_custom_meta->size = strlen(message_data);
msg_custom_meta->message = g_strdup(message_data);
NvDsUserMeta *user_event_meta_custom =
nvds_acquire_user_meta_from_pool(batch_meta);
if (user_event_meta_custom) {
user_event_meta_custom->user_meta_data = (void *)msg_custom_meta;
user_event_meta_custom->base_meta.meta_type = NVDS_CUSTOM_MSG_BLOB;
user_event_meta_custom->base_meta.copy_func =
(NvDsMetaCopyFunc)meta_copy_func_custom;
user_event_meta_custom->base_meta.release_func =
(NvDsMetaReleaseFunc)meta_free_func_custom;
nvds_add_user_meta_to_frame(frame_meta, user_event_meta_custom);
std::cout << "*** send custom message for source id = " << source_id
<< " and object_id = " << obj_meta->object_id << " at " << ts
<< " ***" << std::endl;
} else {
g_print(
"Error in attaching event meta custom to "
"buffer\n");
// std::quick_exit(0);
}
#ifdef ENABLE_DUMP_FILE
gsize size = 0;
snprintf(fileObjNameString, 1024, "%s_%d_%d_%s.jpg", ts, frame_number,
frame_meta->batch_id, obj_meta->obj_label);
guchar *decoded_data = g_base64_decode(face_encoded_data, &size);
fp = fopen(fileObjNameString, "wb");
if (fp) {
fwrite(decoded_data, size, 1, fp);
fclose(fp);
} else {
g_printerr("Could not open file!\n");
}
g_free(face_encoded_data);
gsize size = 0;
snprintf(fileObjNameString, 1024, "%s_%d_%d_%s.jpg", ts, frame_number,
frame_meta->batch_id, obj_meta->obj_label);
guchar *decoded_data = g_base64_decode(full_frame_encoded_data, &size);
fp = fopen(fileObjNameString, "wb");
if (fp) {
fwrite(decoded_data, size, 1, fp);
fclose(fp);
} else {
g_printerr("Could not open file!\n");
}
g_free(full_frame_encoded_data);
#endif
g_free(ts);
// g_free(message_data); // after sending/processing
g_free(width);
g_free(height);
g_free(top);
g_free(left);
g_free(object_id);
g_free(src_id);
g_free(confidence);
g_free(embedding_length);
g_free(json_embedding_vector);
g_free(face_encoded_data);
g_free(full_frame_encoded_data);
}
GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_image_probe(
GstPad *pad, GstPadProbeInfo *info, gpointer u_data) {
(void)pad;
@ -605,15 +823,10 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_image_probe(
GstBuffer *buf = (GstBuffer *)info->data;
NvDsFrameMeta *frame_meta = NULL;
NvDsMetaList *l_frame, *l_obj;
gchar *encoded_data;
gchar *message_data;
gchar *width, *height;
gchar *ts = (gchar *)g_malloc0(MAX_TIME_STAMP_LEN + 1);
NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta(buf);
if (!batch_meta) {
// No batch meta attached.
g_free(ts);
return GST_PAD_PROBE_OK;
}
@ -639,84 +852,65 @@ GstPadProbeReturn NvOsdManager::osd_src_pad_buffer_image_probe(
* Here images is being sent for first object every
* frame_interval(default=30).
*/
std::vector<NvDsObjEncOutParams> encoded_images;
NvDsObjEncOutParams *enc_jpeg_image = NULL;
int num_encode = 0;
bool is_meta_type_NVDS_CROP_IMAGE_META = false;
NvDsUserMetaList *usrMetaList = obj_meta->obj_user_meta_list;
while (usrMetaList != NULL) {
NvDsUserMeta *user_event_meta_custom =
nvds_acquire_user_meta_from_pool(batch_meta);
NvDsCustomMsgInfo *msg_custom_meta =
(NvDsCustomMsgInfo *)g_malloc0(sizeof(NvDsCustomMsgInfo));
NvDsUserMeta *usrMetaData = (NvDsUserMeta *)usrMetaList->data;
if (usrMetaData->base_meta.meta_type == NVDS_CROP_IMAGE_META) {
NvDsObjEncOutParams *enc_jpeg_image =
enc_jpeg_image =
(NvDsObjEncOutParams *)usrMetaData->user_meta_data;
START_PROFILE;
encoded_data = g_base64_encode(enc_jpeg_image->outBuffer,
enc_jpeg_image->outLen);
generate_ts_rfc3339(ts, MAX_TIME_STAMP_LEN);
width = g_strdup_printf(
"%f",
obj_meta->detector_bbox_info.org_bbox_coords.width);
height = g_strdup_printf(
"%f",
obj_meta->detector_bbox_info.org_bbox_coords.height);
/* Image message fields are separated by ";".
* Specific Format:
* "image;image_format;image_widthximage_height;time;encoded
* data;" For Example:
* "image;jpg;640x480;2023-07-31T10:20:13;xxxxxxxxxxx"
*/
message_data =
g_strconcat("image;jpg;", width, "x", height, ";", ts,
";", encoded_data, ";", NULL);
STOP_PROFILE("Base64 Encode Time ");
msg_custom_meta->size = strlen(message_data);
msg_custom_meta->message = g_strdup(message_data);
if (user_event_meta_custom) {
user_event_meta_custom->user_meta_data =
(void *)msg_custom_meta;
user_event_meta_custom->base_meta.meta_type =
NVDS_CUSTOM_MSG_BLOB;
user_event_meta_custom->base_meta.copy_func =
(NvDsMetaCopyFunc)meta_copy_func_custom;
user_event_meta_custom->base_meta.release_func =
(NvDsMetaReleaseFunc)meta_free_func_custom;
nvds_add_user_meta_to_frame(frame_meta,
user_event_meta_custom);
} else {
g_print(
"Error in attaching event meta custom to "
"buffer\n");
}
#ifdef ENABLE_DUMP_FILE
gsize size = 0;
snprintf(fileObjNameString, 1024, "%s_%d_%d_%s.jpg", ts,
frame_number, frame_meta->batch_id,
obj_meta->obj_label);
guchar *decoded_data = g_base64_decode(encoded_data, &size);
fp = fopen(fileObjNameString, "wb");
if (fp) {
fwrite(decoded_data, size, 1, fp);
fclose(fp);
} else {
g_printerr("Could not open file!\n");
}
g_free(decoded_data);
#endif
g_free(encoded_data);
g_free(message_data);
g_free(width);
g_free(height);
usrMetaList = NULL;
} else {
usrMetaList = usrMetaList->next;
encoded_images.push_back(*enc_jpeg_image);
num_encode++;
is_meta_type_NVDS_CROP_IMAGE_META = true;
// usrMetaList = NULL;
}
// else {
// usrMetaList = usrMetaList->next;
// }
usrMetaList = usrMetaList->next;
}
// // Print results
// for (const auto &item : encoded_images) {
// std::cout << " (size=" << item.outLen << ")\n";
// }
if (is_meta_type_NVDS_CROP_IMAGE_META == true) {
enc_jpeg_image = get_full_frame(frame_meta);
encoded_images.push_back(*enc_jpeg_image);
}
// Sort by size (ascending)
std::sort(
encoded_images.begin(), encoded_images.end(),
[](const NvDsObjEncOutParams &a, const NvDsObjEncOutParams &b) {
return a.outLen < b.outLen;
});
NvDsUserMeta *user_meta = NULL;
NvDsMetaList *l_user_meta = NULL;
float *user_meta_data = NULL;
bool is_meta_type_NVOSD_embedding_vector = false;
for (l_user_meta = obj_meta->obj_user_meta_list;
l_user_meta != NULL; l_user_meta = l_user_meta->next) {
user_meta = (NvDsUserMeta *)(l_user_meta->data);
if (user_meta->base_meta.meta_type ==
NVDS_USER_EMBEDDING_VECTOR_META) {
is_meta_type_NVOSD_embedding_vector = true;
user_meta_data = (float *)user_meta->user_meta_data;
}
}
if (is_meta_type_NVOSD_embedding_vector == true &&
encoded_images.size() >= 2) {
event_message_custom_meta(batch_meta, frame_meta, obj_meta,
user_meta_data, encoded_images,
frame_meta->source_id);
}
}
}
g_free(ts);
frame_number++;
return GST_PAD_PROBE_OK;

13
src/nv_osd_manager.hpp
View File

@ -11,6 +11,10 @@
class NvOsdManager {
private:
public:
// struct Item {
// std::string name;
// int size;
// };
GstElement *nvosd = NULL;
static bool write_full_frame_to_disk, write_cropped_objects_to_disk;
NvOsdManager();
@ -41,5 +45,12 @@ class NvOsdManager {
static void generate_face_meta(gpointer);
static void generate_person_meta(gpointer);
static void event_message_meta(NvDsBatchMeta *, NvDsFrameMeta *,
NvDsObjectMeta *, float *);
NvDsObjectMeta *, float *,
std::vector<NvDsObjEncOutParams>);
static void event_message_custom_meta(NvDsBatchMeta *, NvDsFrameMeta *,
NvDsObjectMeta *, float *,
std::vector<NvDsObjEncOutParams>,
guint);
static NvDsObjEncOutParams *get_full_frame(NvDsFrameMeta *);
static NvDsObjEncOutParams *get_cropped_objects(NvDsObjectMeta *);
};