Line data Source code
1 : /**
2 : * GStreamer
3 : * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
4 : * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
5 : * Copyright (C) 2018 Jijoong Moon <jijoong.moon@samsung.com>
6 : *
7 : *
8 : * This library is free software; you can redistribute it and/or
9 : * modify it under the terms of the GNU Library General Public
10 : * License as published by the Free Software Foundation;
11 : * version 2.1 of the License.
12 : *
13 : * This library is distributed in the hope that it will be useful,
14 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 : * Library General Public License for more details.
17 : *
18 : */
19 : /**
20 : * @file gsttensor_merge.c
21 : * @date 03 July 2018
22 : * @brief GStreamer plugin to merge tensors (as a filter for other general neural network filters)
23 : * @see https://github.com/nnstreamer/nnstreamer
24 : * @author Jijoong Moon <jijoong.moon@samsung.com>
25 : * @bug No known bugs except for NYI items
26 : *
27 : */
28 :
29 : /**
30 : * SECTION:element-tensormerge
31 : *
32 : * A Merger that merge tensor stream to tensor stream for NN frameworks.
33 : * The output is always in the format of other/tensor
34 : *
35 : * <refsect2>
36 : * <title>Example launch line</title>
37 : * |[
38 : * gst-launch -v -m tensor_merge name=merge ! fakesink
39 : * filesrc location=b.png ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,width=100,height=100,framerate=0/1 ! tensor_converter ! merge.sink_0
40 : * filesrc location=b.png ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,width=100,height=100,framerate=0/1 ! tensor_converter ! merge.sink_1
41 : * filesrc location=b.png ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,width=100,height=100,framerate=0/1 ! tensor_converter ! merge.sink_2
42 : * ]|
43 : *
44 : * |[
45 : * gst-launch -v -m tensor_merge name=merge ! filesink location=merge.log
46 : * multifilesrc location="testsequence_%1d.png" index=0 caps="image/png, framerate=(fraction)30/1" ! pngdec ! tensor_converter ! merge.sink_0
47 : * multifilesrc location="testsequence_%1d.png" index=0 caps="image/png, framerate=(fraction)30/1" ! pngdec ! tensor_converter ! merge.sink_1
48 : * multifilesrc location="testsequence_%1d.png" index=0 caps="image/png, framerate=(fraction)30/1" ! pngdec ! tensor_converter ! merge.sink_2
49 : *
50 : * </refsect2>
51 : *
52 : */
53 :
54 :
55 : #ifdef HAVE_CONFIG_H
56 : #include <config.h>
57 : #endif
58 :
59 : #include <string.h>
60 : #include <gst/gst.h>
61 : #include <glib.h>
62 : #include <nnstreamer_util.h>
63 :
64 : #include "gsttensor_merge.h"
65 :
66 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_merge_debug);
67 : #define GST_CAT_DEFAULT gst_tensor_merge_debug
68 :
69 : /**
70 : * @brief Macro for debug mode.
71 : */
72 : #ifndef DBG
73 : #define DBG (!tensor_merge->silent)
74 : #endif
75 :
76 : enum
77 : {
78 : PROP_0,
79 : PROP_MODE,
80 : PROP_OPTION,
81 : PROP_SYNC_MODE,
82 : PROP_SYNC_OPTION,
83 : PROP_SILENT,
84 : };
85 :
86 : static const gchar *gst_tensor_merge_mode_string[] = {
87 : [GTT_LINEAR] = "linear",
88 : [GTT_END] = "error",
89 : };
90 :
91 : static const gchar *gst_tensor_merge_linear_string[] = {
92 : [LINEAR_FIRST] = "0",
93 : [LINEAR_SECOND] = "1",
94 : [LINEAR_THIRD] = "2",
95 : [LINEAR_FOURTH] = "3",
96 : [LINEAR_END] = NULL,
97 : };
98 :
99 : /**
100 : * @brief Template caps string.
101 : */
102 : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_WITH_NUM ("1")
103 :
104 : /**
105 : * @brief the capabilities of the inputs and outputs.
106 : * describe the real formats here.
107 : */
108 : static GstStaticPadTemplate src_templ = GST_STATIC_PAD_TEMPLATE ("src",
109 : GST_PAD_SRC,
110 : GST_PAD_ALWAYS,
111 : GST_STATIC_CAPS (CAPS_STRING));
112 :
113 : static GstStaticPadTemplate sink_templ = GST_STATIC_PAD_TEMPLATE ("sink_%u",
114 : GST_PAD_SINK,
115 : GST_PAD_REQUEST,
116 : GST_STATIC_CAPS (CAPS_STRING));
117 :
118 : static gboolean gst_tensor_merge_src_event (GstPad * pad, GstObject * parent,
119 : GstEvent * event);
120 : static GstPad *gst_tensor_merge_request_new_pad (GstElement * element,
121 : GstPadTemplate * templ, const gchar * name, const GstCaps * caps);
122 : static GstStateChangeReturn gst_tensor_merge_change_state (GstElement * element,
123 : GstStateChange transition);
124 : static gboolean gst_tensor_merge_sink_event (GstCollectPads * pads,
125 : GstCollectData * data, GstEvent * event, GstTensorMerge * tensor_merge);
126 : static GstFlowReturn gst_tensor_merge_collected (GstCollectPads * pads,
127 : GstTensorMerge * tensor_merge);
128 :
129 : static void gst_tensor_merge_set_property (GObject * object, guint prop_id,
130 : const GValue * value, GParamSpec * pspec);
131 : static void gst_tensor_merge_get_property (GObject * object, guint prop_id,
132 : GValue * value, GParamSpec * pspec);
133 : static void gst_tensor_merge_finalize (GObject * object);
134 :
135 : #define gst_tensor_merge_parent_class parent_class
136 1316 : G_DEFINE_TYPE (GstTensorMerge, gst_tensor_merge, GST_TYPE_ELEMENT);
137 :
138 : /**
139 : * @brief initialize the tensor_merge's class
140 : */
141 : static void
142 22 : gst_tensor_merge_class_init (GstTensorMergeClass * klass)
143 : {
144 : GObjectClass *gobject_class;
145 : GstElementClass *gstelement_class;
146 :
147 22 : GST_DEBUG_CATEGORY_INIT (gst_tensor_merge_debug, "tensor_merge", 0,
148 : "Element to merge multiple tensor stream to tensor stream");
149 :
150 22 : gobject_class = (GObjectClass *) klass;
151 22 : gstelement_class = (GstElementClass *) klass;
152 :
153 22 : parent_class = g_type_class_peek_parent (klass);
154 :
155 22 : gobject_class->finalize = gst_tensor_merge_finalize;
156 22 : gobject_class->get_property = gst_tensor_merge_get_property;
157 22 : gobject_class->set_property = gst_tensor_merge_set_property;
158 :
159 22 : g_object_class_install_property (gobject_class, PROP_SILENT,
160 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
161 : TRUE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
162 :
163 22 : g_object_class_install_property (gobject_class, PROP_MODE,
164 : g_param_spec_string ("mode", "Mode",
165 : "Tensor Merge mode. Currently, `linear` is available only.",
166 : gst_tensor_merge_mode_string[GTT_LINEAR], G_PARAM_READWRITE));
167 22 : g_object_class_install_property (gobject_class, PROP_OPTION,
168 : g_param_spec_string ("option", "Option",
169 : "Option for the tensor Merge mode.\n"
170 : "\t\t\t 1) linear mode: it will become the index of the rank.\n"
171 : "\t\t\t (e.g. want to merge 3:640:480 & 3:640:480 to 3:1280:480, option=1)\n"
172 : "\t\t\t 2) TBU", "", G_PARAM_READWRITE));
173 :
174 22 : g_object_class_install_property (gobject_class, PROP_SYNC_MODE,
175 : g_param_spec_string ("sync-mode", "Sync Mode",
176 : "Time synchronization mode\n"
177 : "\t\t\tSee also: https://github.com/nnstreamer/nnstreamer/blob/main/Documentation/synchronization-policies-at-mux-merge.md",
178 : "", G_PARAM_READWRITE));
179 :
180 22 : g_object_class_install_property (gobject_class, PROP_SYNC_OPTION,
181 : g_param_spec_string ("sync-option", "Sync Option",
182 : "Option for the time synchronization mode", "", G_PARAM_READWRITE));
183 :
184 22 : gstelement_class->request_new_pad =
185 22 : GST_DEBUG_FUNCPTR (gst_tensor_merge_request_new_pad);
186 22 : gstelement_class->change_state =
187 22 : GST_DEBUG_FUNCPTR (gst_tensor_merge_change_state);
188 :
189 22 : gst_element_class_add_pad_template (gstelement_class,
190 : gst_static_pad_template_get (&sink_templ));
191 22 : gst_element_class_add_pad_template (gstelement_class,
192 : gst_static_pad_template_get (&src_templ));
193 :
194 22 : gst_element_class_set_details_simple (gstelement_class,
195 : "TensorMerge",
196 : "Muxer/Tensor",
197 : "Merge multiple tensor stream to tensor stream",
198 : "Jijoong Moon <jijoong.moon@samsung.com>");
199 :
200 22 : }
201 :
202 : /**
203 : * @brief initialize the new element
204 : * instantiate pads and add them to element
205 : * set pad callback functions
206 : * initialize instance structure
207 : */
208 : static void
209 27 : gst_tensor_merge_init (GstTensorMerge * tensor_merge)
210 : {
211 27 : GstElementClass *klass = GST_ELEMENT_GET_CLASS (tensor_merge);
212 :
213 27 : tensor_merge->srcpad =
214 27 : gst_pad_new_from_template (gst_element_class_get_pad_template (klass,
215 : "src"), "src");
216 27 : gst_pad_set_event_function (tensor_merge->srcpad, gst_tensor_merge_src_event);
217 :
218 27 : gst_element_add_pad (GST_ELEMENT (tensor_merge), tensor_merge->srcpad);
219 :
220 27 : tensor_merge->collect = gst_collect_pads_new ();
221 27 : gst_collect_pads_set_event_function (tensor_merge->collect,
222 : (GstCollectPadsEventFunction)
223 27 : GST_DEBUG_FUNCPTR (gst_tensor_merge_sink_event), tensor_merge);
224 27 : gst_collect_pads_set_function (tensor_merge->collect,
225 27 : (GstCollectPadsFunction) GST_DEBUG_FUNCPTR (gst_tensor_merge_collected),
226 : tensor_merge);
227 :
228 27 : tensor_merge->silent = TRUE;
229 27 : tensor_merge->sync.mode = SYNC_NOSYNC;
230 27 : tensor_merge->sync.option = NULL;
231 27 : gst_tensors_config_init (&tensor_merge->tensors_config);
232 27 : tensor_merge->mode = GTT_LINEAR;
233 27 : tensor_merge->loaded = FALSE;
234 27 : tensor_merge->current_time = 0;
235 27 : tensor_merge->need_set_time = TRUE;
236 27 : }
237 :
238 : /**
239 : * @brief Get the corresponding mode from the string value
240 : * @param[in] str The string value for the mode
241 : * @return corresponding mode for the string. GTT_END for errors
242 : */
243 : static tensor_merge_mode
244 27 : gst_tensor_merge_get_mode (const gchar * str)
245 : {
246 : int i;
247 27 : for (i = 0; i < GTT_END; i++) {
248 27 : if (g_ascii_strcasecmp (gst_tensor_merge_mode_string[i], str) == 0)
249 27 : return i;
250 : }
251 0 : return GTT_END;
252 : }
253 :
254 : /**
255 : * @brief finalize vmethod
256 : */
257 : static void
258 27 : gst_tensor_merge_finalize (GObject * object)
259 : {
260 : GstTensorMerge *tensor_merge;
261 :
262 27 : tensor_merge = GST_TENSOR_MERGE (object);
263 :
264 27 : if (tensor_merge->collect) {
265 27 : gst_tensor_time_sync_flush (tensor_merge->collect);
266 27 : gst_object_unref (tensor_merge->collect);
267 27 : tensor_merge->collect = NULL;
268 : }
269 :
270 27 : if (tensor_merge->option) {
271 27 : g_free (tensor_merge->option);
272 27 : tensor_merge->option = NULL;
273 : }
274 :
275 27 : if (tensor_merge->sync.option) {
276 12 : g_free (tensor_merge->sync.option);
277 12 : tensor_merge->sync.option = NULL;
278 : }
279 :
280 27 : G_OBJECT_CLASS (parent_class)->finalize (object);
281 27 : }
282 :
283 : /**
284 : * @brief making new request pad (gst element vmethod)
285 : */
286 : static GstPad *
287 76 : gst_tensor_merge_request_new_pad (GstElement * element, GstPadTemplate * templ,
288 : const gchar * req_name, const GstCaps * caps)
289 : {
290 : GstPad *newpad;
291 : GstTensorMerge *tensor_merge;
292 : GSList *walk;
293 : guint length;
294 : gchar *name;
295 : UNUSED (req_name);
296 : UNUSED (caps);
297 :
298 76 : g_return_val_if_fail (templ != NULL, NULL);
299 76 : g_return_val_if_fail (GST_IS_TENSOR_MERGE (element), NULL);
300 :
301 76 : tensor_merge = GST_TENSOR_MERGE (element);
302 76 : walk = tensor_merge->collect->data;
303 76 : length = g_slist_length (walk);
304 :
305 76 : if (length >= NNS_TENSOR_SIZE_LIMIT) {
306 0 : GST_ERROR_OBJECT (tensor_merge,
307 : "supposed max number of tensors is " NNS_TENSOR_SIZE_LIMIT_STR);
308 0 : return NULL;
309 : }
310 :
311 76 : name = g_strdup_printf ("sink_%u", length);
312 76 : newpad = gst_pad_new_from_template (templ, name);
313 76 : g_free (name);
314 :
315 76 : if (newpad) {
316 : GstTensorCollectPadData *tensormergepad;
317 :
318 : tensormergepad = (GstTensorCollectPadData *)
319 76 : gst_collect_pads_add_pad (tensor_merge->collect, newpad,
320 : sizeof (GstTensorCollectPadData), NULL, TRUE);
321 :
322 76 : gst_pad_set_element_private (newpad, tensormergepad);
323 76 : gst_element_add_pad (element, newpad);
324 : } else {
325 0 : GST_WARNING_OBJECT (tensor_merge, "failed to create request pad");
326 : }
327 76 : return newpad;
328 : }
329 :
330 : /**
331 : * @brief src event vmethod
332 : */
333 : static gboolean
334 139 : gst_tensor_merge_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
335 : {
336 139 : g_return_val_if_fail (event != NULL, FALSE);
337 :
338 139 : switch (GST_EVENT_TYPE (event)) {
339 0 : case GST_EVENT_SEEK:
340 0 : gst_event_unref (event);
341 0 : return FALSE;
342 139 : default:
343 139 : break;
344 : }
345 :
346 139 : return gst_pad_event_default (pad, parent, event);
347 : }
348 :
349 : /**
350 : * @brief sink event vmethod
351 : */
352 : static gboolean
353 286 : gst_tensor_merge_sink_event (GstCollectPads * pads, GstCollectData * data,
354 : GstEvent * event, GstTensorMerge * tensor_merge)
355 : {
356 286 : g_return_val_if_fail (event != NULL, FALSE);
357 :
358 286 : switch (GST_EVENT_TYPE (event)) {
359 0 : case GST_EVENT_FLUSH_STOP:
360 0 : tensor_merge->need_segment = TRUE;
361 0 : tensor_merge->need_set_time = TRUE;
362 0 : gst_tensor_time_sync_flush (tensor_merge->collect);
363 0 : break;
364 286 : default:
365 286 : break;
366 : }
367 :
368 286 : return gst_collect_pads_event_default (pads, data, event, FALSE);
369 : }
370 :
371 : /**
372 : * @brief Generate out TensorsConfig with in TensorsConfig
373 : * @param tensor_merge tensor merger
374 : * @param in_config in tensors config data (multi tensors)
375 : * @param out_config out tensors config data (single tensor)
376 : * @return true / false
377 : */
378 : static gboolean
379 25 : gst_tensor_merge_get_merged_config (GstTensorMerge * tensor_merge,
380 : const GstTensorsConfig * in_config, GstTensorsConfig * out_config)
381 : {
382 : GstTensorsInfo *in_info;
383 : GstTensorInfo *_info;
384 25 : gboolean ret = FALSE;
385 : guint i;
386 : gint j;
387 : tensor_dim dim;
388 : tensor_type type;
389 :
390 25 : gst_tensors_config_init (out_config);
391 :
392 25 : in_info = (GstTensorsInfo *) &in_config->info;
393 25 : _info = gst_tensors_info_get_nth_info (in_info, 0);
394 :
395 25 : type = _info->type;
396 25 : memcpy (&dim, &_info->dimension, sizeof (tensor_dim));
397 :
398 72 : for (i = 1; i < in_config->info.num_tensors; i++) {
399 47 : _info = gst_tensors_info_get_nth_info (in_info, i);
400 :
401 47 : if (type != _info->type)
402 0 : GST_ELEMENT_ERROR (tensor_merge, CORE, NEGOTIATION, (NULL), (NULL));
403 : }
404 :
405 25 : switch (tensor_merge->mode) {
406 25 : case GTT_LINEAR:
407 : {
408 25 : int targetIdx = tensor_merge->data_linear.direction;
409 72 : for (i = 1; i < in_config->info.num_tensors; i++) {
410 47 : _info = gst_tensors_info_get_nth_info (in_info, i);
411 :
412 799 : for (j = 0; j < NNS_TENSOR_RANK_LIMIT; j++) {
413 752 : if (j == targetIdx) {
414 47 : dim[j] += _info->dimension[j];
415 : } else {
416 705 : if (dim[j] != _info->dimension[j])
417 0 : GST_ELEMENT_ERROR (tensor_merge, CORE, NEGOTIATION, (NULL),
418 : (NULL));
419 : }
420 : }
421 : }
422 :
423 25 : out_config->info.num_tensors = 1;
424 25 : out_config->rate_d = in_config->rate_d;
425 25 : out_config->rate_n = in_config->rate_n;
426 :
427 25 : _info = gst_tensors_info_get_nth_info (&out_config->info, 0);
428 25 : _info->type = type;
429 25 : memcpy (&_info->dimension, &dim, sizeof (tensor_dim));
430 25 : ret = TRUE;
431 : }
432 25 : break;
433 0 : default:
434 0 : ret = FALSE;
435 : }
436 :
437 25 : return ret;
438 : }
439 :
440 : /**
441 : * @brief Looping to generete output buffer for srcpad
442 : * @param tensor_merge tensor merger
443 : * @param tensor_buf output buffer for srcpad
444 : * @param is_eos boolean EOS ( End of Stream )
445 : * @return TRUE to push buffer to src pad
446 : */
447 : static gboolean
448 386 : gst_tensor_merge_collect_buffer (GstTensorMerge * tensor_merge,
449 : GstBuffer * tensors_buf, gboolean * is_eos)
450 : {
451 386 : if (tensor_merge->need_set_time) {
452 248 : if (gst_tensor_time_sync_get_current_time (tensor_merge->collect,
453 : &tensor_merge->sync, &tensor_merge->current_time, tensors_buf)) {
454 : /* end-of-stream */
455 37 : *is_eos = TRUE;
456 37 : return FALSE;
457 : }
458 :
459 211 : tensor_merge->need_set_time = FALSE;
460 : }
461 :
462 349 : return gst_tensor_time_sync_buffer_from_collectpad (tensor_merge->collect,
463 : &tensor_merge->sync, tensor_merge->current_time, tensors_buf,
464 : &tensor_merge->tensors_config, is_eos);
465 : }
466 :
467 : /**
468 : * @brief Generate Output GstMemory
469 : * @param tensor_merge tensor merger
470 : * @param tensors_buf collected tensors buffer
471 : * @param tensor_buf output tensor buffer
472 : * @return boolean
473 : */
474 : static GstFlowReturn
475 211 : gst_tensor_merge_generate_mem (GstTensorMerge * tensor_merge,
476 : GstBuffer * tensors_buf, GstBuffer * tensor_buf)
477 : {
478 211 : GstFlowReturn ret = GST_FLOW_OK;
479 : GstMapInfo mInfo[NNS_TENSOR_SIZE_LIMIT];
480 : GstMemory *mem[NNS_TENSOR_SIZE_LIMIT];
481 : GstMapInfo outInfo;
482 : GstMemory *outMem;
483 : uint8_t *inptr, *outptr;
484 211 : guint num_mem = tensor_merge->tensors_config.info.num_tensors;
485 : GstTensorsInfo *info;
486 : GstTensorInfo *_info;
487 : guint i, j, k, l;
488 : size_t c, s;
489 211 : gsize outSize = 0;
490 : gsize element_size;
491 : tensor_dim dim;
492 : tensor_type type;
493 :
494 211 : info = &tensor_merge->tensors_config.info;
495 211 : _info = gst_tensors_info_get_nth_info (info, 0);
496 :
497 211 : memcpy (&dim, &_info->dimension, sizeof (tensor_dim));
498 211 : type = _info->type;
499 211 : element_size = gst_tensor_get_element_size (type);
500 :
501 750 : for (i = 0; i < num_mem; i++) {
502 539 : mem[i] = gst_tensor_buffer_get_nth_memory (tensors_buf, i);
503 539 : if (!gst_memory_map (mem[i], &mInfo[i], GST_MAP_READ)) {
504 0 : ml_logf ("Cannot map input memory buffers (%d)\n", i);
505 0 : gst_memory_unref (mem[i]);
506 0 : num_mem = i;
507 0 : ret = GST_FLOW_ERROR;
508 0 : goto error_ret;
509 : }
510 539 : outSize += mInfo[i].size;
511 : }
512 :
513 211 : outMem = gst_allocator_alloc (NULL, outSize, NULL);
514 211 : if (!gst_memory_map (outMem, &outInfo, GST_MAP_WRITE)) {
515 0 : gst_memory_unref (outMem);
516 0 : ml_logf ("Cannot map output memory buffer\n");
517 0 : ret = GST_FLOW_ERROR;
518 0 : goto error_ret;
519 : }
520 211 : outptr = outInfo.data;
521 :
522 211 : switch (tensor_merge->mode) {
523 211 : case GTT_LINEAR:
524 : {
525 211 : switch (tensor_merge->data_linear.direction) {
526 1 : case LINEAR_FIRST:
527 : {
528 2 : for (l = 0; l < dim[3]; l++) {
529 101 : for (i = 0; i < dim[2]; i++) {
530 5100 : for (j = 0; j < dim[1]; j++) {
531 20000 : for (k = 0; k < num_mem; k++) {
532 15000 : _info = gst_tensors_info_get_nth_info (info, k);
533 :
534 15000 : c = _info->dimension[0];
535 15000 : s = element_size * c;
536 15000 : inptr =
537 15000 : mInfo[k].data + (l * dim[2] * dim[1] + i * dim[1] +
538 15000 : j) * s;
539 15000 : memcpy (outptr, inptr, s);
540 15000 : outptr += s;
541 : }
542 : }
543 : }
544 : }
545 1 : break;
546 : }
547 1 : case LINEAR_SECOND:
548 : {
549 2 : for (l = 0; l < dim[3]; l++) {
550 51 : for (i = 0; i < dim[2]; i++) {
551 200 : for (k = 0; k < num_mem; k++) {
552 150 : _info = gst_tensors_info_get_nth_info (info, k);
553 :
554 150 : c = 1;
555 450 : for (j = 0; j < LINEAR_SECOND + 1; j++)
556 300 : c *= _info->dimension[j];
557 :
558 150 : s = element_size * c;
559 :
560 150 : inptr = mInfo[k].data + (l * dim[2] + i) * s;
561 :
562 150 : memcpy (outptr, inptr, s);
563 150 : outptr += s;
564 : }
565 : }
566 : }
567 1 : break;
568 : }
569 119 : case LINEAR_THIRD:
570 : {
571 238 : for (l = 0; l < dim[3]; l++) {
572 471 : for (k = 0; k < num_mem; k++) {
573 352 : _info = gst_tensors_info_get_nth_info (info, k);
574 :
575 352 : c = 1;
576 1408 : for (j = 0; j < LINEAR_THIRD + 1; j++)
577 1056 : c *= _info->dimension[j];
578 :
579 352 : s = element_size * c;
580 :
581 352 : inptr = mInfo[k].data + l * s;
582 :
583 352 : memcpy (outptr, inptr, s);
584 352 : outptr += s;
585 : }
586 : }
587 119 : break;
588 : }
589 90 : case LINEAR_FOURTH:
590 : {
591 271 : for (k = 0; k < num_mem; k++) {
592 181 : _info = gst_tensors_info_get_nth_info (info, k);
593 :
594 181 : c = 1;
595 905 : for (j = 0; j < LINEAR_FOURTH + 1; j++)
596 724 : c *= _info->dimension[j];
597 :
598 181 : s = element_size * c;
599 :
600 181 : inptr = mInfo[k].data;
601 181 : memcpy (outptr, inptr, s);
602 181 : outptr += s;
603 : }
604 90 : break;
605 : }
606 0 : default:
607 0 : ret = GST_FLOW_ERROR;
608 : }
609 211 : break;
610 : }
611 0 : default:
612 0 : ret = GST_FLOW_ERROR;
613 : }
614 :
615 211 : gst_memory_unmap (outMem, &outInfo);
616 211 : gst_buffer_append_memory (tensor_buf, outMem);
617 211 : gst_buffer_copy_into (tensor_buf, tensors_buf, GST_BUFFER_COPY_TIMESTAMPS, 0,
618 : -1);
619 :
620 211 : error_ret:
621 750 : for (i = 0; i < num_mem; i++) {
622 539 : gst_memory_unmap (mem[i], &mInfo[i]);
623 539 : gst_memory_unref (mem[i]);
624 : }
625 211 : return ret;
626 : }
627 :
628 : /**
629 : * @brief Set src pad caps if src pad is not negotiated.
630 : */
631 : static gboolean
632 211 : gst_tensor_merge_set_src_caps (GstTensorMerge * tensor_merge)
633 : {
634 211 : if (!tensor_merge->negotiated) {
635 : GstCaps *newcaps;
636 : GstTensorsConfig config;
637 :
638 25 : if (!gst_tensor_merge_get_merged_config (tensor_merge,
639 25 : &tensor_merge->tensors_config, &config)) {
640 0 : goto nego_error;
641 : }
642 :
643 : /** Internal Logic Error? */
644 25 : g_assert (gst_tensors_config_validate (&config));
645 25 : newcaps = gst_tensor_pad_caps_from_config (tensor_merge->srcpad, &config);
646 :
647 25 : if (gst_pad_set_caps (tensor_merge->srcpad, newcaps)) {
648 25 : tensor_merge->negotiated = TRUE;
649 : }
650 :
651 25 : gst_caps_unref (newcaps);
652 : }
653 :
654 186 : nego_error:
655 211 : if (!tensor_merge->negotiated) {
656 0 : GST_WARNING_OBJECT (tensor_merge, "failed to set caps");
657 0 : GST_ELEMENT_ERROR (tensor_merge, CORE, NEGOTIATION, (NULL), (NULL));
658 : }
659 211 : return tensor_merge->negotiated;
660 : }
661 :
662 : /**
663 : * @brief Send segment event if necessary.
664 : */
665 : static void
666 211 : gst_tensor_merge_send_segment_event (GstTensorMerge * tensor_merge,
667 : GstClockTime pts, GstClockTime dts)
668 : {
669 211 : if (tensor_merge->need_segment) {
670 : GstSegment segment;
671 25 : GstClockTime time = 0;
672 :
673 25 : if (GST_CLOCK_TIME_IS_VALID (dts)) {
674 0 : time = dts;
675 25 : } else if (GST_CLOCK_TIME_IS_VALID (pts)) {
676 25 : time = pts;
677 : }
678 :
679 25 : gst_segment_init (&segment, GST_FORMAT_TIME);
680 25 : segment.start = time;
681 25 : gst_pad_push_event (tensor_merge->srcpad, gst_event_new_segment (&segment));
682 25 : tensor_merge->need_segment = FALSE;
683 : }
684 211 : }
685 :
686 : /**
687 : * @brief Gst Collect Pads Function which is called once collect pads done.
688 : * @param pads GstCollectPads
689 : * @param tensor_merge Merger
690 : * @return GstFlowReturn
691 : */
692 : static GstFlowReturn
693 386 : gst_tensor_merge_collected (GstCollectPads * pads,
694 : GstTensorMerge * tensor_merge)
695 : {
696 386 : GstFlowReturn ret = GST_FLOW_OK;
697 : GstBuffer *tensors_buf, *tensor_buf;
698 386 : gboolean isEOS = FALSE;
699 :
700 386 : GST_DEBUG_OBJECT (tensor_merge, " all pads are collected ");
701 :
702 386 : if (tensor_merge->need_stream_start) {
703 : /**
704 : * Cannot use gst-pad util to get stream ID (multiple sink pads).
705 : * Create stream ID using first sink pad.
706 : */
707 25 : GstCollectData *data = (GstCollectData *) pads->data->data;
708 25 : g_autofree gchar *sink_stream_id = gst_pad_get_stream_id (data->pad);
709 25 : g_autofree gchar *element_name = gst_element_get_name (tensor_merge);
710 25 : g_autofree gchar *pad_name = gst_pad_get_name (tensor_merge->srcpad);
711 50 : g_autofree gchar *stream_id = g_strdup_printf ("%s-%s-nnsmerge-%s-%08x",
712 25 : GST_STR_NULL (sink_stream_id), element_name, pad_name, g_random_int ());
713 :
714 25 : gst_pad_push_event (tensor_merge->srcpad, gst_event_new_stream_start (stream_id));
715 25 : tensor_merge->need_stream_start = FALSE;
716 : }
717 :
718 386 : if ((tensors_buf = gst_buffer_new ()) == NULL) {
719 0 : ml_logf ("gst_buffer_new() returns NULL. Out of memory?\n");
720 386 : return GST_FLOW_ERROR;
721 : }
722 :
723 386 : if (!gst_tensor_merge_collect_buffer (tensor_merge, tensors_buf, &isEOS)) {
724 175 : if (isEOS) {
725 37 : gst_pad_push_event (tensor_merge->srcpad, gst_event_new_eos ());
726 37 : ret = GST_FLOW_EOS;
727 : }
728 :
729 175 : goto beach;
730 : }
731 :
732 211 : if (!gst_tensor_merge_set_src_caps (tensor_merge)) {
733 0 : ret = GST_FLOW_NOT_NEGOTIATED;
734 0 : goto beach;
735 : }
736 :
737 211 : gst_tensor_merge_send_segment_event (tensor_merge,
738 : GST_BUFFER_PTS (tensors_buf), GST_BUFFER_DTS (tensors_buf));
739 :
740 211 : if ((tensor_buf = gst_buffer_new ()) == NULL) {
741 0 : ml_logf ("gst_buffer_new() returns NULL. Out of memory?\n");
742 0 : ret = GST_FLOW_ERROR;
743 0 : goto beach;
744 : }
745 :
746 211 : gst_tensor_merge_generate_mem (tensor_merge, tensors_buf, tensor_buf);
747 :
748 211 : ret = gst_pad_push (tensor_merge->srcpad, tensor_buf);
749 211 : tensor_merge->need_set_time = TRUE;
750 :
751 211 : if (ret != GST_FLOW_OK) {
752 0 : GST_WARNING_OBJECT (tensor_merge, "pushed outbuf, result = %s",
753 : gst_flow_get_name (ret));
754 : }
755 211 : beach:
756 386 : gst_buffer_unref (tensors_buf);
757 386 : return ret;
758 : }
759 :
760 : /**
761 : * @brief Ready --> Pasuse State Change
762 : */
763 : static void
764 25 : gst_tensor_merge_ready_to_paused (GstTensorMerge * tensor_merge)
765 : {
766 25 : tensor_merge->need_stream_start = TRUE;
767 25 : tensor_merge->need_segment = TRUE;
768 25 : tensor_merge->negotiated = FALSE;
769 25 : gst_collect_pads_start (tensor_merge->collect);
770 25 : }
771 :
772 : /**
773 : * @brief change state (gst element vmethod)
774 : */
775 : static GstStateChangeReturn
776 150 : gst_tensor_merge_change_state (GstElement * element, GstStateChange transition)
777 : {
778 : GstTensorMerge *tensor_merge;
779 : GstStateChangeReturn ret;
780 150 : tensor_merge = GST_TENSOR_MERGE (element);
781 150 : switch (transition) {
782 25 : case GST_STATE_CHANGE_READY_TO_PAUSED:
783 25 : gst_tensor_merge_ready_to_paused (tensor_merge);
784 25 : break;
785 25 : case GST_STATE_CHANGE_PAUSED_TO_READY:
786 25 : gst_collect_pads_stop (tensor_merge->collect);
787 25 : break;
788 100 : default:
789 100 : break;
790 : }
791 :
792 150 : ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
793 150 : if (ret == GST_STATE_CHANGE_FAILURE)
794 0 : return ret;
795 150 : switch (transition) {
796 25 : case GST_STATE_CHANGE_PAUSED_TO_READY:
797 25 : break;
798 125 : default:
799 125 : break;
800 : }
801 :
802 150 : return ret;
803 : }
804 :
805 : /**
806 : * @brief Setup internal data (data_* in GstTensor_Merge)
807 : * @param[in/out] filter "this" pointer. mode & option MUST BE set already.
808 : * @retval TRUE if ok or not configured, yet.
809 : * @retval FALSE if given input is configured invalid.
810 : */
811 : static gboolean
812 54 : gst_tensor_merge_set_option_data (GstTensorMerge * tensor_merge)
813 : {
814 54 : if (tensor_merge->mode == GTT_END || tensor_merge->option == NULL)
815 27 : return TRUE;
816 27 : switch (tensor_merge->mode) {
817 27 : case GTT_LINEAR:
818 : {
819 : gint idx;
820 :
821 : idx =
822 27 : find_key_strv (gst_tensor_merge_linear_string, tensor_merge->option);
823 27 : if (idx < 0)
824 0 : return FALSE;
825 :
826 27 : tensor_merge->data_linear.direction = (tensor_merge_linear_mode) idx;
827 27 : tensor_merge->loaded = TRUE;
828 : }
829 27 : break;
830 0 : default:
831 0 : GST_ERROR_OBJECT (tensor_merge, "Cannot identify mode\n");
832 0 : return FALSE;
833 : }
834 27 : return TRUE;
835 : }
836 :
837 : /**
838 : * @brief Get property (gst element vmethod)
839 : */
840 : static void
841 95 : gst_tensor_merge_set_property (GObject * object, guint prop_id,
842 : const GValue * value, GParamSpec * pspec)
843 : {
844 95 : GstTensorMerge *tensor_merge = GST_TENSOR_MERGE (object);
845 95 : switch (prop_id) {
846 12 : case PROP_SILENT:
847 12 : tensor_merge->silent = g_value_get_boolean (value);
848 12 : break;
849 27 : case PROP_MODE:
850 27 : tensor_merge->mode =
851 27 : gst_tensor_merge_get_mode (g_value_get_string (value));
852 27 : if (tensor_merge->mode == GTT_END) {
853 0 : ml_logw ("Given mode property is not recognized: %s\n",
854 : g_value_get_string (value));
855 0 : break;
856 : }
857 27 : if (!gst_tensor_merge_set_option_data (tensor_merge)) {
858 0 : tensor_merge->loaded = FALSE;
859 0 : ml_logw ("Given mode property is not consistent with its options.\n");
860 : }
861 27 : break;
862 27 : case PROP_OPTION:
863 27 : tensor_merge->option = g_value_dup_string (value);
864 27 : if (!gst_tensor_merge_set_option_data (tensor_merge)) {
865 0 : tensor_merge->loaded = FALSE;
866 0 : ml_logw ("Given option property is not consistent with its mode.\n");
867 : }
868 27 : break;
869 17 : case PROP_SYNC_MODE:
870 17 : tensor_merge->sync.mode =
871 17 : gst_tensor_time_sync_get_mode (g_value_get_string (value));
872 17 : if (tensor_merge->sync.mode == SYNC_END) {
873 0 : tensor_merge->sync.mode = SYNC_NOSYNC;
874 : }
875 17 : silent_debug (tensor_merge, "Mode = %d(%s)\n", tensor_merge->sync.mode,
876 : gst_tensor_time_sync_get_mode_string (tensor_merge->sync.mode));
877 17 : gst_tensor_time_sync_set_option_data (&tensor_merge->sync);
878 17 : break;
879 12 : case PROP_SYNC_OPTION:
880 12 : tensor_merge->sync.option = g_value_dup_string (value);
881 12 : silent_debug (tensor_merge, "Option = %s\n", tensor_merge->sync.option);
882 12 : gst_tensor_time_sync_set_option_data (&tensor_merge->sync);
883 12 : break;
884 0 : default:
885 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
886 0 : break;
887 : }
888 95 : }
889 :
890 : /**
891 : * @brief Get property (gst element vmethod)
892 : */
893 : static void
894 6 : gst_tensor_merge_get_property (GObject * object, guint prop_id,
895 : GValue * value, GParamSpec * pspec)
896 : {
897 6 : GstTensorMerge *tensor_merge = GST_TENSOR_MERGE (object);
898 6 : switch (prop_id) {
899 2 : case PROP_SILENT:
900 2 : g_value_set_boolean (value, tensor_merge->silent);
901 2 : break;
902 1 : case PROP_MODE:
903 1 : g_value_set_string (value,
904 1 : gst_tensor_merge_mode_string[tensor_merge->mode]);
905 1 : break;
906 1 : case PROP_OPTION:
907 1 : g_value_set_string (value, tensor_merge->option);
908 1 : break;
909 1 : case PROP_SYNC_MODE:
910 1 : g_value_set_string (value,
911 : gst_tensor_time_sync_get_mode_string (tensor_merge->sync.mode));
912 1 : break;
913 1 : case PROP_SYNC_OPTION:
914 1 : g_value_set_string (value, tensor_merge->sync.option);
915 1 : break;
916 0 : default:
917 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
918 0 : break;
919 : }
920 6 : }
|
