Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * NNStreamer Common Header's Contents (pipeline extension)
4 : * Copyright (C) 2020 MyungJoo Ham <myungjoo.ham@samsung.com>
5 : */
6 : /**
7 : * @file nnstreamer_plugin_api_impl.c
8 : * @date 14 Apr 2020
9 : * @brief Common data for NNStreamer, the GStreamer plugin for neural networks
10 : * @see https://github.com/nnstreamer/nnstreamer
11 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
12 : * @bug No known bugs except for NYI items
13 : *
14 : */
15 :
16 : #include <nnstreamer_util.h>
17 : #include <string.h>
18 : #include <tensor_common.h>
19 :
20 : static const gchar *gst_tensor_time_sync_mode_string[] = {
21 : [SYNC_NOSYNC] = "nosync",
22 : [SYNC_SLOWEST] = "slowest",
23 : [SYNC_BASEPAD] = "basepad",
24 : [SYNC_REFRESH] = "refresh",
25 : [SYNC_END] = NULL
26 : };
27 :
28 : /**
29 : * @brief The old rank of tensor.
30 : */
31 : #define NNS_TENSOR_RANK_LIMIT_PREV (4)
32 :
33 : #define NNS_TENSOR_EXTRA_MAGIC 0xf00dc0de
34 :
35 : /**
36 : * @brief Data structure to describe a "extra" tensor data.
37 : * This represents the information of the NNS_TENSOR_SIZE_LIMIT-th memory block for tensor stream.
38 : */
39 : typedef struct
40 : {
41 : uint32_t magic;
42 : uint32_t version;
43 : uint32_t num_extra_tensors;
44 : uint64_t reserved;
45 : GstTensorInfo infos[NNS_TENSOR_SIZE_EXTRA_LIMIT];
46 : } GstTensorExtraInfo;
47 :
48 : /**
49 : * @brief Check if given memory has extra tensors.
50 : * @param[in] map GstMapInfo of GstMemory to be checked.
51 : * @return TRUE if @map has extra tensors, otherwise FALSE.
52 : */
53 : static gboolean
54 2910 : gst_memory_map_is_extra_tensor (GstMapInfo * map)
55 : {
56 : GstTensorExtraInfo *extra_info;
57 : gboolean is_extra;
58 :
59 2910 : g_return_val_if_fail (map != NULL, FALSE);
60 :
61 2910 : if (map->size < sizeof (GstTensorExtraInfo))
62 380 : return FALSE;
63 :
64 2530 : extra_info = (GstTensorExtraInfo *) map->data;
65 :
66 : /* check magic in header (extra info) of the memory */
67 2530 : is_extra = (extra_info && extra_info->magic == NNS_TENSOR_EXTRA_MAGIC);
68 :
69 2530 : return is_extra;
70 : }
71 :
72 : /**
73 : * @brief Initialize GstTensorExtraInfo structure with given @a memory.
74 : * @param[in/out] extra GstTensorExtraInfo to be initialized.
75 : * @param[in] reserved_size The memory size of extra memory block.
76 : */
77 : static void
78 28 : gst_tensor_extra_info_init (GstTensorExtraInfo * extra, gsize reserved_size)
79 : {
80 : guint i;
81 :
82 28 : g_return_if_fail (extra != NULL);
83 :
84 28 : extra->magic = NNS_TENSOR_EXTRA_MAGIC;
85 28 : extra->version = 0;
86 28 : extra->num_extra_tensors = 0;
87 :
88 : /* set reserved size of NNS_TENSOR_SIZE_LIMIT-th memory */
89 28 : extra->reserved = reserved_size;
90 6748 : for (i = 0; i < NNS_TENSOR_SIZE_EXTRA_LIMIT; ++i) {
91 6720 : gst_tensor_info_init (&extra->infos[i]);
92 : }
93 : }
94 :
95 : /**
96 : * @brief Get the corresponding mode from the string value.
97 : * @param[in] str The string value for the mode.
98 : * @return Corresponding mode for the string. SYNC_END for errors.
99 : */
100 : tensor_time_sync_mode
101 66 : gst_tensor_time_sync_get_mode (const gchar * str)
102 : {
103 : gint index;
104 :
105 66 : index = find_key_strv (gst_tensor_time_sync_mode_string, str);
106 :
107 66 : return (index < 0) ? SYNC_END : index;
108 : }
109 :
110 : /**
111 : * @brief Get the time-sync mode string.
112 : * @return Corresponding mode string.
113 : */
114 : const gchar *
115 2 : gst_tensor_time_sync_get_mode_string (tensor_time_sync_mode mode)
116 : {
117 2 : return gst_tensor_time_sync_mode_string[mode];
118 : }
119 :
120 : /**
121 : * @brief Setup time sync option.
122 : * @param[in/out] filter "this" pointer. Sync mode & option MUST BE set already.
123 : * @return True if successfully set the option.
124 : */
125 : gboolean
126 95 : gst_tensor_time_sync_set_option_data (tensor_time_sync_data * sync)
127 : {
128 95 : g_return_val_if_fail (sync != NULL, FALSE);
129 :
130 95 : if (sync->mode == SYNC_END || sync->option == NULL)
131 66 : return FALSE;
132 :
133 29 : switch (sync->mode) {
134 0 : case SYNC_NOSYNC:
135 0 : break;
136 0 : case SYNC_SLOWEST:
137 0 : break;
138 29 : case SYNC_BASEPAD:
139 : {
140 29 : g_auto (GStrv) strv = g_strsplit (sync->option, ":", 2);
141 : guint sink_id;
142 : guint duration;
143 :
144 29 : if (strv[0] != NULL)
145 29 : sink_id = (guint) g_ascii_strtoull (strv[0], NULL, 10);
146 : else
147 0 : sink_id = 0;
148 :
149 29 : if (strv[1] != NULL)
150 27 : duration = (guint) g_ascii_strtoull (strv[1], NULL, 10);
151 : else
152 2 : duration = G_MAXINT;
153 :
154 29 : sync->data_basepad.sink_id = sink_id;
155 29 : sync->data_basepad.duration = duration;
156 29 : break;
157 : }
158 0 : default:
159 : /* unknown mode */
160 0 : GST_WARNING ("Unknown mode = %d", sync->mode);
161 0 : return FALSE;
162 : }
163 :
164 29 : return TRUE;
165 : }
166 :
167 : /**
168 : * @brief Internal function to detect EOS using the number of empty pads.
169 : * @param[in] collect Collect pad.
170 : * @param[in] sync Synchronization option.
171 : * @param[in] empty The number of empty pads (pad has no buffer).
172 : * @return True if EOS.
173 : */
174 : static gboolean
175 13257 : _gst_tensor_time_sync_is_eos (GstCollectPads * collect,
176 : tensor_time_sync_data * sync, guint empty)
177 : {
178 : guint total;
179 13257 : gboolean is_eos = FALSE;
180 :
181 13257 : total = g_slist_length (collect->data);
182 :
183 13257 : switch (sync->mode) {
184 11328 : case SYNC_REFRESH:
185 11328 : if (empty == total)
186 2 : is_eos = TRUE;
187 11328 : break;
188 1929 : default:
189 1929 : if (empty > 0)
190 215 : is_eos = TRUE;
191 1929 : break;
192 : }
193 :
194 13257 : return is_eos;
195 : }
196 :
197 : /**
198 : * @brief A function call to decide current timestamp among collected pads based on PTS.
199 : * It will decide current timestamp according to sync option.
200 : * GstMeta is also copied with same sync mode.
201 : */
202 : gboolean
203 6737 : gst_tensor_time_sync_get_current_time (GstCollectPads * collect,
204 : tensor_time_sync_data * sync, GstClockTime * current_time,
205 : GstBuffer * tensors_buf)
206 : {
207 6737 : GSList *walk = NULL;
208 : guint count, empty_pad;
209 :
210 6737 : g_return_val_if_fail (collect != NULL, FALSE);
211 6737 : g_return_val_if_fail (sync != NULL, FALSE);
212 6737 : g_return_val_if_fail (current_time != NULL, FALSE);
213 :
214 6737 : walk = collect->data;
215 6737 : count = empty_pad = 0;
216 :
217 21282 : while (walk) {
218 : GstCollectData *data;
219 : GstBuffer *buf;
220 14545 : gboolean need_update = FALSE;
221 :
222 14545 : data = (GstCollectData *) walk->data;
223 14545 : buf = gst_collect_pads_peek (collect, data);
224 14545 : walk = g_slist_next (walk);
225 :
226 14545 : if (buf) {
227 8335 : switch (sync->mode) {
228 7677 : case SYNC_NOSYNC:
229 : /* fall-through */
230 : case SYNC_SLOWEST:
231 : case SYNC_REFRESH:
232 7677 : if (*current_time < GST_BUFFER_PTS (buf))
233 6221 : need_update = TRUE;
234 7677 : break;
235 658 : case SYNC_BASEPAD:
236 658 : if (count == sync->data_basepad.sink_id)
237 296 : need_update = TRUE;
238 658 : break;
239 0 : default:
240 0 : break;
241 : }
242 8335 : if (need_update) {
243 6517 : *current_time = GST_BUFFER_PTS (buf);
244 6517 : gst_buffer_copy_into (tensors_buf, buf, GST_BUFFER_COPY_METADATA,
245 : 0, -1);
246 : }
247 8335 : gst_buffer_unref (buf);
248 : } else {
249 6210 : empty_pad++;
250 : }
251 :
252 14545 : count++;
253 : }
254 :
255 6737 : return _gst_tensor_time_sync_is_eos (collect, sync, empty_pad);
256 : }
257 :
258 : /**
259 : * @brief A function to be called while processing a flushing event.
260 : * It should clear old buffer and reset pad data.
261 : */
262 : void
263 152 : gst_tensor_time_sync_flush (GstCollectPads * collect)
264 : {
265 : GSList *walk;
266 : GstTensorCollectPadData *pad;
267 :
268 152 : g_return_if_fail (collect != NULL);
269 :
270 152 : walk = collect->data;
271 633 : while (walk) {
272 481 : pad = (GstTensorCollectPadData *) walk->data;
273 :
274 481 : if (pad->buffer) {
275 418 : gst_buffer_unref (pad->buffer);
276 418 : pad->buffer = NULL;
277 : }
278 :
279 481 : walk = g_slist_next (walk);
280 : }
281 : }
282 :
283 : /**
284 : * @brief Internal function to update buffer in pad data based on the sync mode.
285 : */
286 : static gboolean
287 3778 : _gst_tensor_time_sync_buffer_update (GstCollectPads * collect,
288 : GstCollectData * data, GstClockTime current, GstClockTime base,
289 : tensor_time_sync_data * sync)
290 : {
291 : GstTensorCollectPadData *pad;
292 : GstBuffer *buf;
293 :
294 3778 : pad = (GstTensorCollectPadData *) data;
295 :
296 3778 : buf = gst_collect_pads_peek (collect, data);
297 3778 : if (buf != NULL) {
298 3645 : if (GST_BUFFER_PTS (buf) < current) {
299 671 : gst_buffer_unref (buf);
300 671 : if (pad->buffer != NULL)
301 669 : gst_buffer_unref (pad->buffer);
302 671 : pad->buffer = gst_collect_pads_pop (collect, data);
303 671 : return FALSE;
304 : }
305 :
306 4399 : if ((sync->mode == SYNC_SLOWEST && pad->buffer != NULL &&
307 1425 : (ABS (GST_CLOCK_DIFF (current, GST_BUFFER_PTS (pad->buffer))) <
308 1425 : ABS (GST_CLOCK_DIFF (current, GST_BUFFER_PTS (buf))))) ||
309 3943 : (sync->mode == SYNC_BASEPAD && pad->buffer != NULL &&
310 1133 : (((GstClockTime) ABS (GST_CLOCK_DIFF (current,
311 : GST_BUFFER_PTS (buf)))) > base))) {
312 : /* keep last buffer */
313 : } else {
314 : /* update last buffer */
315 2240 : if (pad->buffer != NULL)
316 1824 : gst_buffer_unref (pad->buffer);
317 2240 : pad->buffer = gst_collect_pads_pop (collect, data);
318 : }
319 :
320 2974 : gst_buffer_unref (buf);
321 : }
322 :
323 3107 : return TRUE;
324 : }
325 :
326 : /**
327 : * @brief A function call to make tensors from collected pads.
328 : * It decide which buffer is going to be used according to sync option.
329 : * @return True to push buffer.
330 : */
331 : gboolean
332 7192 : gst_tensor_time_sync_buffer_from_collectpad (GstCollectPads * collect,
333 : tensor_time_sync_data * sync, GstClockTime current_time,
334 : GstBuffer * tensors_buf, GstTensorsConfig * configs, gboolean * is_eos)
335 : {
336 7192 : GSList *walk = NULL;
337 : GstCollectData *data;
338 : GstTensorCollectPadData *pad;
339 7192 : GstBuffer *buf = NULL;
340 : GstMemory *mem;
341 7192 : gint old_numerator = G_MAXINT;
342 7192 : gint old_denominator = G_MAXINT;
343 : guint counting, empty_pad;
344 : GstTensorsConfig in_configs;
345 7192 : GstClockTime base_time = 0;
346 : GstTensorInfo *_info;
347 : guint i, j;
348 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT];
349 : tensor_format in_formats[NNS_TENSOR_SIZE_LIMIT];
350 :
351 14384 : g_return_val_if_fail (collect != NULL, FALSE);
352 7192 : g_return_val_if_fail (sync != NULL, FALSE);
353 7192 : g_return_val_if_fail (tensors_buf != NULL, FALSE);
354 7192 : g_return_val_if_fail (configs != NULL, FALSE);
355 7192 : g_return_val_if_fail (is_eos != NULL, FALSE);
356 :
357 7192 : walk = collect->data;
358 7192 : counting = empty_pad = 0;
359 :
360 7192 : if (sync->mode == SYNC_BASEPAD) {
361 711 : walk = g_slist_nth (walk, sync->data_basepad.sink_id);
362 711 : if (walk == NULL) {
363 0 : GST_ERROR_OBJECT (collect, "Cannot get GstCollectData from GSList");
364 0 : return FALSE;
365 : }
366 :
367 711 : data = (GstCollectData *) walk->data;
368 711 : pad = (GstTensorCollectPadData *) data;
369 :
370 711 : buf = gst_collect_pads_peek (collect, data);
371 711 : if (buf != NULL) {
372 649 : if (pad->buffer != NULL)
373 624 : base_time =
374 624 : MIN ((GstClockTimeDiff) sync->data_basepad.duration,
375 : ABS (GST_CLOCK_DIFF (GST_BUFFER_PTS (buf),
376 : GST_BUFFER_PTS (pad->buffer))) - 1);
377 649 : gst_buffer_unref (buf);
378 : }
379 : }
380 :
381 7192 : walk = collect->data;
382 :
383 7192 : gst_tensors_config_init (&in_configs);
384 :
385 21882 : while (walk) {
386 15362 : gboolean configured = FALSE;
387 15362 : gboolean is_empty = FALSE;
388 :
389 15362 : data = (GstCollectData *) walk->data;
390 15362 : pad = (GstTensorCollectPadData *) data;
391 :
392 15362 : if (gst_pad_has_current_caps (data->pad)) {
393 15361 : GstCaps *caps = gst_pad_get_current_caps (data->pad);
394 :
395 15361 : gst_tensors_config_free (&in_configs);
396 15361 : configured = gst_tensors_config_from_caps (&in_configs, caps, TRUE);
397 :
398 15361 : gst_caps_unref (caps);
399 : }
400 :
401 : /**
402 : * This would be an internal logic error.
403 : * in_configs should be already confirmed valid at the negotiation phase
404 : * and this function should be called in a running pipeline.
405 : * If new sync mode is enabled (e.g., handle output when a pad gets new buffer),
406 : * this may cause unexpected exception.
407 : */
408 15362 : if (!configured) {
409 1 : return FALSE;
410 : }
411 :
412 15361 : if (in_configs.rate_d < old_denominator)
413 7191 : old_denominator = in_configs.rate_d;
414 15361 : if (in_configs.rate_n < old_numerator)
415 7399 : old_numerator = in_configs.rate_n;
416 :
417 15361 : walk = g_slist_next (walk);
418 :
419 15361 : switch (sync->mode) {
420 3778 : case SYNC_SLOWEST:
421 : /* fall-through */
422 : case SYNC_BASEPAD:
423 3778 : if (!_gst_tensor_time_sync_buffer_update (collect, data,
424 : current_time, base_time, sync))
425 671 : return FALSE;
426 3107 : buf = gst_buffer_ref (pad->buffer);
427 3107 : is_empty = (buf == NULL);
428 3107 : break;
429 257 : case SYNC_NOSYNC:
430 257 : buf = gst_collect_pads_pop (collect, data);
431 257 : is_empty = (buf == NULL);
432 257 : break;
433 11326 : case SYNC_REFRESH:
434 11326 : buf = gst_collect_pads_pop (collect, data);
435 11326 : if (buf != NULL) {
436 5664 : if (pad->buffer != NULL) {
437 5662 : gst_buffer_unref (pad->buffer);
438 : }
439 5664 : pad->buffer = gst_buffer_ref (buf);
440 : } else {
441 5662 : if (pad->buffer == NULL) {
442 0 : *is_eos = FALSE;
443 0 : ml_logd ("Not the all buffers are arrived yet.");
444 0 : return FALSE;
445 : }
446 5662 : is_empty = TRUE;
447 5662 : buf = gst_buffer_ref (pad->buffer);
448 : }
449 11326 : break;
450 0 : default:
451 0 : break;
452 : }
453 :
454 14690 : if (GST_IS_BUFFER (buf)) {
455 14690 : guint32 n_tensor = gst_tensor_buffer_get_count (buf);
456 14690 : buf = gst_tensor_buffer_from_config (buf, &in_configs);
457 :
458 : /** These are internal logic error. If given inputs are incorrect,
459 : the negotiation should have been failed before this stage. */
460 14690 : if (gst_tensors_config_is_static (&in_configs))
461 14664 : g_assert (n_tensor == in_configs.info.num_tensors);
462 14690 : g_assert ((counting + n_tensor) <= NNS_TENSOR_SIZE_LIMIT);
463 :
464 14690 : if (gst_tensors_config_is_flexible (&in_configs))
465 26 : configs->info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
466 :
467 29760 : for (i = 0; i < n_tensor; ++i) {
468 15070 : in_mem[counting] = gst_tensor_buffer_get_nth_memory (buf, i);
469 :
470 : /* set info */
471 15070 : gst_tensor_info_copy (gst_tensors_info_get_nth_info (&configs->info,
472 15070 : counting), gst_tensors_info_get_nth_info (&in_configs.info, i));
473 15070 : in_formats[counting] = in_configs.info.format;
474 15070 : counting++;
475 : }
476 :
477 14690 : gst_buffer_unref (buf);
478 : }
479 14690 : if (is_empty)
480 5662 : empty_pad++;
481 : }
482 :
483 : /* append memories to output buffer */
484 20795 : for (i = 0; i < counting; i++) {
485 14275 : _info = gst_tensors_info_get_nth_info (&configs->info, i);
486 14275 : mem = in_mem[i];
487 :
488 14275 : if (gst_tensors_config_is_flexible (configs)) {
489 : /* append header if input tensor is not flexible */
490 46 : if (in_formats[i] != _NNS_TENSOR_FORMAT_FLEXIBLE) {
491 : GstTensorMetaInfo meta;
492 :
493 20 : gst_tensor_info_convert_to_meta (_info, &meta);
494 20 : mem = gst_tensor_meta_info_append_header (&meta, in_mem[i]);
495 20 : gst_memory_unref (in_mem[i]);
496 : }
497 : }
498 :
499 14275 : if (!gst_tensor_buffer_append_memory (tensors_buf, mem, _info)) {
500 0 : for (j = i + 1; j < counting; j++)
501 0 : gst_memory_unref (in_mem[j]);
502 :
503 0 : nns_loge ("Failed to append memory to buffer.");
504 0 : return FALSE;
505 : }
506 : }
507 :
508 6520 : configs->info.num_tensors = counting;
509 6520 : configs->rate_d = old_denominator;
510 6520 : configs->rate_n = old_numerator;
511 :
512 6520 : GST_BUFFER_PTS (tensors_buf) = current_time;
513 :
514 6520 : gst_tensors_config_free (&in_configs);
515 :
516 : /* check eos */
517 6520 : *is_eos = _gst_tensor_time_sync_is_eos (collect, sync, empty_pad);
518 6520 : return !(*is_eos);
519 : }
520 :
521 : /**
522 : * @brief Configure gst-buffer with tensors information.
523 : * NNStreamer handles single memory chunk as single tensor.
524 : * If incoming buffer has invalid memories, separate it and generate new gst-buffer using tensors information.
525 : * Note that this function always takes the ownership of input buffer.
526 : * @param in input buffer
527 : * @param config tensors config structure
528 : * @return Newly allocated buffer. Null if failed. Caller should unref the buffer using gst_buffer_unref().
529 : */
530 : GstBuffer *
531 36881 : gst_tensor_buffer_from_config (GstBuffer * in, GstTensorsConfig * config)
532 : {
533 36881 : GstBuffer *out = NULL;
534 36881 : GstMemory *all = NULL;
535 : GstMapInfo map;
536 : guint i, num;
537 : gsize total, offset;
538 : gsize mem_size[NNS_TENSOR_MEMORY_MAX];
539 36881 : gboolean configured = FALSE;
540 36881 : gboolean is_extra = FALSE;
541 :
542 36881 : if (!GST_IS_BUFFER (in)) {
543 1 : nns_loge ("Failed to get tensor buffer, invalid input buffer.");
544 36881 : return NULL;
545 : }
546 :
547 36880 : if (!gst_tensors_config_validate (config)) {
548 2 : nns_loge ("Failed to get tensor buffer, invalid tensor configuration.");
549 2 : goto error;
550 : }
551 :
552 36878 : num = gst_buffer_n_memory (in);
553 36878 : total = gst_buffer_get_size (in);
554 :
555 : /* get memory size */
556 36878 : if (gst_tensors_config_is_static (config)) {
557 36802 : if (num == config->info.num_tensors) {
558 : /* Do nothing, pass input buffer. */
559 36795 : out = gst_buffer_ref (in);
560 36795 : goto done;
561 : }
562 :
563 7 : num = config->info.num_tensors;
564 7 : if ((is_extra = (num > NNS_TENSOR_MEMORY_MAX)))
565 5 : num = NNS_TENSOR_MEMORY_MAX;
566 93 : for (i = 0; i < num; i++)
567 86 : mem_size[i] = gst_tensors_info_get_size (&config->info, i);
568 7 : if (is_extra) {
569 5 : mem_size[num - 1] += sizeof (GstTensorExtraInfo);
570 25 : for (; i < config->info.num_tensors; i++)
571 20 : mem_size[num - 1] += gst_tensors_info_get_size (&config->info, i);
572 : }
573 : } else {
574 76 : if (num > 1) {
575 : /* Suppose it is already configured. */
576 20 : out = gst_buffer_ref (in);
577 20 : goto done;
578 : }
579 :
580 56 : if (!gst_buffer_map (in, &map, GST_MAP_READ)) {
581 0 : nns_loge ("Failed to get tensor buffer, cannot get the memory info.");
582 0 : goto error;
583 : }
584 :
585 56 : num = 0;
586 56 : offset = 0;
587 114 : while (offset < total) {
588 : GstTensorMetaInfo meta;
589 58 : gpointer h = map.data + offset;
590 :
591 58 : if (num >= NNS_TENSOR_MEMORY_MAX - 1) {
592 : /* Suppose remained memory may include extra tensors. */
593 0 : mem_size[num++] = total - offset;
594 0 : break;
595 : }
596 :
597 58 : gst_tensor_meta_info_parse_header (&meta, h);
598 58 : mem_size[num] = gst_tensor_meta_info_get_header_size (&meta);
599 58 : mem_size[num] += gst_tensor_meta_info_get_data_size (&meta);
600 :
601 58 : offset += mem_size[num];
602 58 : num++;
603 : }
604 :
605 56 : gst_buffer_unmap (in, &map);
606 :
607 56 : if (num == 1) {
608 : /* Do nothing, pass input buffer. */
609 55 : out = gst_buffer_ref (in);
610 55 : goto done;
611 : }
612 : }
613 :
614 : /* configure output buffer */
615 8 : out = gst_buffer_new ();
616 8 : all = gst_buffer_get_all_memory (in);
617 8 : offset = 0;
618 :
619 95 : for (i = 0; i < num; i++) {
620 : /* invalid memory size */
621 88 : if (offset + mem_size[i] > total) {
622 1 : nns_loge ("Failed to get tensor buffer, data size is mismatched.");
623 1 : goto error;
624 : }
625 :
626 87 : gst_buffer_append_memory (out, gst_memory_share (all, offset, mem_size[i]));
627 87 : offset += mem_size[i];
628 : }
629 :
630 7 : gst_buffer_copy_into (out, in, GST_BUFFER_COPY_METADATA, 0, -1);
631 :
632 36877 : done:
633 36877 : configured = TRUE;
634 36880 : error:
635 36880 : gst_buffer_unref (in);
636 :
637 36880 : if (all)
638 8 : gst_memory_unref (all);
639 :
640 36880 : if (!configured) {
641 3 : if (out) {
642 1 : gst_buffer_unref (out);
643 1 : out = NULL;
644 : }
645 : }
646 :
647 36880 : return out;
648 : }
649 :
650 : /**
651 : * @brief Internal struct to handle aggregation data in hash table.
652 : */
653 : typedef struct
654 : {
655 : GstAdapter *adapter;
656 : } gst_tensor_aggregation_data_s;
657 :
658 : #define AGGREGATION_DEFAULT_KEY 0xC0FFEEU
659 :
660 : /**
661 : * @brief Internal function to free aggregation data.
662 : */
663 : static void
664 868 : gst_tensor_aggregation_free_data (gpointer data)
665 : {
666 : gst_tensor_aggregation_data_s *aggr;
667 :
668 868 : aggr = (gst_tensor_aggregation_data_s *) data;
669 868 : if (aggr) {
670 868 : gst_adapter_clear (aggr->adapter);
671 868 : g_object_unref (aggr->adapter);
672 :
673 868 : g_free (aggr);
674 : }
675 868 : }
676 :
677 : /**
678 : * @brief Internal function to add new aggregation data.
679 : */
680 : static gst_tensor_aggregation_data_s *
681 925 : gst_tensor_aggregation_add_data (GHashTable * table, const guint32 key)
682 : {
683 : gst_tensor_aggregation_data_s *aggr;
684 : guint32 hashkey;
685 :
686 925 : g_return_val_if_fail (table != NULL, NULL);
687 925 : if (key == 0)
688 0 : hashkey = AGGREGATION_DEFAULT_KEY;
689 : else
690 925 : hashkey = key;
691 925 : aggr = g_new0 (gst_tensor_aggregation_data_s, 1);
692 925 : aggr->adapter = gst_adapter_new ();
693 :
694 925 : g_hash_table_insert (table, GINT_TO_POINTER (hashkey), aggr);
695 925 : return aggr;
696 : }
697 :
698 : /**
699 : * @brief Internal function to get aggregation data.
700 : */
701 : static gst_tensor_aggregation_data_s *
702 304 : gst_tensor_aggregation_get_data (GHashTable * table, const guint32 key)
703 : {
704 304 : g_return_val_if_fail (table != NULL, NULL);
705 :
706 316 : return (gst_tensor_aggregation_data_s *) g_hash_table_lookup (table,
707 12 : GINT_TO_POINTER (key == 0 ? AGGREGATION_DEFAULT_KEY : key));
708 : }
709 :
710 : /**
711 : * @brief Internal function to remove all buffers from aggregation data.
712 : */
713 : static void
714 3491 : gst_tensor_aggregation_clear_internal (gpointer key, gpointer value,
715 : gpointer user_data)
716 : {
717 : gst_tensor_aggregation_data_s *aggr;
718 :
719 : UNUSED (key);
720 : UNUSED (user_data);
721 :
722 3491 : aggr = (gst_tensor_aggregation_data_s *) value;
723 3491 : if (aggr) {
724 3491 : gst_adapter_clear (aggr->adapter);
725 : }
726 3491 : }
727 :
728 : /**
729 : * @brief Gets new hash table for tensor aggregation.
730 : * @return Newly allocated hash table, caller should release this using g_hash_table_destroy().
731 : */
732 : GHashTable *
733 922 : gst_tensor_aggregation_init (void)
734 : {
735 : GHashTable *table;
736 :
737 922 : table = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL,
738 : gst_tensor_aggregation_free_data);
739 :
740 : /**
741 : * Add default adapter (for the case if buffer has no specific id).
742 : * If gst-buffer has tensor-meta which includes client-id,
743 : * e.g., aggregation frames from multiple clients on query-server pipeline,
744 : * nnstreamer element should parse meta and request adapter with this id.
745 : * However, on normal pipeline, gst-buffer does not contain tensor-meta,
746 : * then the element may request adapter with null key string.
747 : */
748 922 : gst_tensor_aggregation_add_data (table, AGGREGATION_DEFAULT_KEY);
749 :
750 922 : return table;
751 : }
752 :
753 : /**
754 : * @brief Clears buffers from adapter.
755 : * @param table a hash table instance initialized with gst_tensor_aggregation_init()
756 : * @param key the key to look up (set null to get default adapter)
757 : */
758 : void
759 1 : gst_tensor_aggregation_clear (GHashTable * table, const guint32 key)
760 : {
761 : gst_tensor_aggregation_data_s *aggr;
762 :
763 1 : g_return_if_fail (table != NULL);
764 :
765 1 : aggr = gst_tensor_aggregation_get_data (table, key);
766 1 : gst_tensor_aggregation_clear_internal (NULL, aggr, NULL);
767 : }
768 :
769 : /**
770 : * @brief Clears buffers from all adapters in hash table.
771 : * @param table a hash table instance initialized with gst_tensor_aggregation_init()
772 : */
773 : void
774 3485 : gst_tensor_aggregation_clear_all (GHashTable * table)
775 : {
776 3485 : g_hash_table_foreach (table, gst_tensor_aggregation_clear_internal, NULL);
777 3485 : }
778 :
779 : /**
780 : * @brief Gets adapter from hash table.
781 : * @param table a hash table instance initialized with gst_tensor_aggregation_init()
782 : * @param key the key to look up (set null to get default adapter)
783 : * @return gst-adapter instance. DO NOT release this instance.
784 : */
785 : GstAdapter *
786 304 : gst_tensor_aggregation_get_adapter (GHashTable * table, const guint32 key)
787 : {
788 : gst_tensor_aggregation_data_s *aggr;
789 :
790 304 : g_return_val_if_fail (table != NULL, NULL);
791 :
792 303 : aggr = gst_tensor_aggregation_get_data (table, key);
793 303 : if (!aggr) {
794 : /*append new data */
795 3 : aggr = gst_tensor_aggregation_add_data (table, key);
796 : }
797 :
798 303 : return aggr->adapter;
799 : }
800 :
801 : /**
802 : * @brief Internal function to check tensor dimensions to append old caps for backward compatibility (rank 4).
803 : */
804 : static gboolean
805 10634 : _append_prev_caps (const GstTensorsConfig * config)
806 : {
807 : GstTensorsInfo *info;
808 : GstTensorInfo *_info;
809 : guint i, min_rank;
810 :
811 10634 : g_return_val_if_fail (config != NULL, FALSE);
812 :
813 10634 : info = (GstTensorsInfo *) (&config->info);
814 10634 : if (!gst_tensors_info_validate (info))
815 4727 : return FALSE;
816 :
817 12830 : for (i = 0; i < info->num_tensors; i++) {
818 6989 : _info = gst_tensors_info_get_nth_info (info, i);
819 :
820 6989 : min_rank = gst_tensor_dimension_get_min_rank (_info->dimension);
821 :
822 6989 : if (min_rank > NNS_TENSOR_RANK_LIMIT_PREV)
823 66 : return FALSE;
824 : }
825 :
826 5841 : return TRUE;
827 : }
828 :
829 : /**
830 : * @brief Internal function to get caps for single tensor from config.
831 : */
832 : static GstCaps *
833 5661 : _get_tensor_caps (const GstTensorsConfig * config)
834 : {
835 : GstCaps *caps, *prev1, *prev2;
836 : GstTensorsInfo *info;
837 : GstTensorInfo *_info;
838 : gboolean append_prev;
839 :
840 5661 : g_return_val_if_fail (config != NULL, NULL);
841 :
842 5661 : info = (GstTensorsInfo *) (&config->info);
843 5661 : if (info->num_tensors > 1)
844 103 : return NULL;
845 :
846 5558 : caps = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
847 5558 : _info = gst_tensors_info_get_nth_info (info, 0);
848 :
849 : /* caps for backward compatibility */
850 5558 : prev1 = prev2 = NULL;
851 5558 : append_prev = _append_prev_caps (config);
852 5558 : if (append_prev) {
853 3180 : prev1 = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
854 3180 : prev2 = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
855 : }
856 :
857 5558 : if (gst_tensor_dimension_is_valid (_info->dimension)) {
858 3351 : g_autofree gchar *dim_str =
859 3351 : gst_tensor_get_dimension_string (_info->dimension);
860 :
861 3351 : gst_caps_set_simple (caps, "dimension", G_TYPE_STRING, dim_str, NULL);
862 :
863 3351 : if (append_prev) {
864 3180 : g_autofree gchar *dim_prev1 =
865 3180 : gst_tensor_get_rank_dimension_string (_info->dimension,
866 : NNS_TENSOR_RANK_LIMIT_PREV, FALSE);
867 3180 : g_autofree gchar *dim_prev2 =
868 3180 : gst_tensor_get_rank_dimension_string (_info->dimension,
869 : NNS_TENSOR_RANK_LIMIT_PREV, TRUE);
870 :
871 3180 : gst_caps_set_simple (prev1, "dimension", G_TYPE_STRING, dim_prev1, NULL);
872 3180 : gst_caps_set_simple (prev2, "dimension", G_TYPE_STRING, dim_prev2, NULL);
873 : }
874 : }
875 :
876 5558 : if (_info->type != _NNS_END) {
877 3751 : const gchar *type_str = gst_tensor_get_type_string (_info->type);
878 :
879 3751 : gst_caps_set_simple (caps, "type", G_TYPE_STRING, type_str, NULL);
880 :
881 3751 : if (append_prev) {
882 3180 : gst_caps_set_simple (prev1, "type", G_TYPE_STRING, type_str, NULL);
883 3180 : gst_caps_set_simple (prev2, "type", G_TYPE_STRING, type_str, NULL);
884 : }
885 : }
886 :
887 5558 : if (config->rate_n >= 0 && config->rate_d > 0) {
888 2063 : gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION,
889 2063 : config->rate_n, config->rate_d, NULL);
890 :
891 2063 : if (append_prev) {
892 1998 : gst_caps_set_simple (prev1, "framerate", GST_TYPE_FRACTION,
893 1998 : config->rate_n, config->rate_d, NULL);
894 1998 : gst_caps_set_simple (prev2, "framerate", GST_TYPE_FRACTION,
895 1998 : config->rate_n, config->rate_d, NULL);
896 : }
897 : }
898 :
899 5558 : if (append_prev)
900 3180 : caps = gst_caps_merge (caps, gst_caps_merge (prev1, prev2));
901 :
902 5558 : return caps;
903 : }
904 :
905 : /**
906 : * @brief Internal function to get caps for multi tensors from config.
907 : */
908 : static GstCaps *
909 5076 : _get_tensors_caps (const GstTensorsConfig * config)
910 : {
911 : GstCaps *caps, *prev1, *prev2;
912 : gboolean append_prev;
913 :
914 5076 : g_return_val_if_fail (config != NULL, NULL);
915 :
916 5076 : caps = gst_caps_from_string (GST_TENSORS_CAP_DEFAULT);
917 :
918 : /* caps for backward compatibility */
919 5076 : prev1 = prev2 = NULL;
920 5076 : append_prev = _append_prev_caps (config);
921 5076 : if (append_prev) {
922 2661 : prev1 = gst_caps_from_string (GST_TENSORS_CAP_DEFAULT);
923 2661 : prev2 = gst_caps_from_string (GST_TENSORS_CAP_DEFAULT);
924 : }
925 :
926 5076 : if (config->info.num_tensors > 0) {
927 2847 : g_autofree gchar *type_str =
928 2847 : gst_tensors_info_get_types_string (&config->info);
929 2847 : g_autofree gchar *dim_str =
930 2847 : gst_tensors_info_get_dimensions_string (&config->info);
931 :
932 2847 : gst_caps_set_simple (caps, "num_tensors", G_TYPE_INT,
933 2847 : config->info.num_tensors, NULL);
934 2847 : gst_caps_set_simple (caps, "dimensions", G_TYPE_STRING, dim_str, NULL);
935 2847 : gst_caps_set_simple (caps, "types", G_TYPE_STRING, type_str, NULL);
936 :
937 2847 : if (append_prev) {
938 2661 : g_autofree gchar *dim_prev1 =
939 2661 : gst_tensors_info_get_rank_dimensions_string (&config->info,
940 : NNS_TENSOR_RANK_LIMIT_PREV, FALSE);
941 2661 : g_autofree gchar *dim_prev2 =
942 2661 : gst_tensors_info_get_rank_dimensions_string (&config->info,
943 : NNS_TENSOR_RANK_LIMIT_PREV, TRUE);
944 :
945 2661 : gst_caps_set_simple (prev1, "num_tensors", G_TYPE_INT,
946 2661 : config->info.num_tensors, NULL);
947 2661 : gst_caps_set_simple (prev1, "dimensions", G_TYPE_STRING, dim_prev1, NULL);
948 2661 : gst_caps_set_simple (prev1, "types", G_TYPE_STRING, type_str, NULL);
949 :
950 2661 : gst_caps_set_simple (prev2, "num_tensors", G_TYPE_INT,
951 2661 : config->info.num_tensors, NULL);
952 2661 : gst_caps_set_simple (prev2, "dimensions", G_TYPE_STRING, dim_prev2, NULL);
953 2661 : gst_caps_set_simple (prev2, "types", G_TYPE_STRING, type_str, NULL);
954 : }
955 : }
956 :
957 5076 : if (config->rate_n >= 0 && config->rate_d > 0) {
958 1495 : gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION,
959 1495 : config->rate_n, config->rate_d, NULL);
960 :
961 1495 : if (append_prev) {
962 1439 : gst_caps_set_simple (prev1, "framerate", GST_TYPE_FRACTION,
963 1439 : config->rate_n, config->rate_d, NULL);
964 1439 : gst_caps_set_simple (prev2, "framerate", GST_TYPE_FRACTION,
965 1439 : config->rate_n, config->rate_d, NULL);
966 : }
967 : }
968 :
969 5076 : if (append_prev)
970 2661 : caps = gst_caps_merge (caps, gst_caps_merge (prev1, prev2));
971 :
972 5076 : return caps;
973 : }
974 :
975 : /**
976 : * @brief Internal function to get caps for flexible tensor from config.
977 : */
978 : static GstCaps *
979 2863 : _get_flexible_caps (const GstTensorsConfig * config)
980 : {
981 : GstCaps *caps;
982 :
983 2863 : caps = gst_caps_from_string (GST_TENSORS_FLEX_CAP_DEFAULT);
984 :
985 2863 : if (config->rate_n >= 0 && config->rate_d > 0) {
986 862 : gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION,
987 862 : config->rate_n, config->rate_d, NULL);
988 : }
989 :
990 2863 : return caps;
991 : }
992 :
993 : /**
994 : * @brief Check given mimetype is tensor stream.
995 : * @param structure structure to be interpreted
996 : * @return TRUE if mimetype is tensor stream
997 : */
998 : gboolean
999 18326 : gst_structure_is_tensor_stream (const GstStructure * structure)
1000 : {
1001 : const gchar *name;
1002 :
1003 18326 : name = gst_structure_get_name (structure);
1004 18326 : g_return_val_if_fail (name != NULL, FALSE);
1005 :
1006 32484 : return (g_str_equal (name, NNS_MIMETYPE_TENSOR) ||
1007 14158 : g_str_equal (name, NNS_MIMETYPE_TENSORS));
1008 : }
1009 :
1010 : /**
1011 : * @brief Get media type from structure
1012 : * @param structure structure to be interpreted
1013 : * @return corresponding media type (returns _NNS_MEDIA_INVALID for unsupported type)
1014 : */
1015 : media_type
1016 20125 : gst_structure_get_media_type (const GstStructure * structure)
1017 : {
1018 : const gchar *name;
1019 :
1020 20125 : name = gst_structure_get_name (structure);
1021 :
1022 20125 : g_return_val_if_fail (name != NULL, _NNS_MEDIA_INVALID);
1023 :
1024 20125 : if (g_str_has_prefix (name, "video/")) {
1025 2762 : return _NNS_VIDEO;
1026 : }
1027 :
1028 17363 : if (g_str_has_prefix (name, "audio/")) {
1029 2179 : return _NNS_AUDIO;
1030 : }
1031 :
1032 15184 : if (g_str_has_prefix (name, "text/")) {
1033 2152 : return _NNS_TEXT;
1034 : }
1035 :
1036 13032 : if (g_str_equal (name, "application/octet-stream")) {
1037 4402 : return _NNS_OCTET;
1038 : }
1039 :
1040 8630 : if (gst_structure_is_tensor_stream (structure)) {
1041 2147 : return _NNS_TENSOR;
1042 : }
1043 :
1044 : /* unknown or unsupported type */
1045 6483 : return _NNS_MEDIA_INVALID;
1046 : }
1047 :
1048 : /**
1049 : * @brief Parse caps from peer pad and set tensors config.
1050 : * @param pad GstPad to get the capabilities
1051 : * @param config tensors config structure to be filled
1052 : * @param is_fixed flag to be updated when peer caps is fixed (not mandatory, do nothing when the param is null)
1053 : * @return TRUE if successfully configured from peer
1054 : */
1055 : gboolean
1056 4286 : gst_tensors_config_from_peer (GstPad * pad, GstTensorsConfig * config,
1057 : gboolean * is_fixed)
1058 : {
1059 : GstCaps *peer_caps;
1060 : GstStructure *structure;
1061 4286 : gboolean ret = FALSE;
1062 :
1063 4286 : g_return_val_if_fail (GST_IS_PAD (pad), FALSE);
1064 4286 : g_return_val_if_fail (config != NULL, FALSE);
1065 :
1066 4286 : gst_tensors_config_init (config);
1067 :
1068 4286 : if ((peer_caps = gst_pad_peer_query_caps (pad, NULL))) {
1069 4286 : if (gst_caps_get_size (peer_caps) > 0) {
1070 2181 : structure = gst_caps_get_structure (peer_caps, 0);
1071 2181 : ret = gst_tensors_config_from_structure (config, structure);
1072 : }
1073 :
1074 4286 : if (ret && is_fixed)
1075 30 : *is_fixed = gst_caps_is_fixed (peer_caps);
1076 :
1077 4286 : gst_caps_unref (peer_caps);
1078 : }
1079 :
1080 4286 : return ret;
1081 : }
1082 :
1083 : /**
1084 : * @brief Check whether two structures have the same dimension
1085 : */
1086 : static gboolean
1087 248 : _is_structure_dimension_same (GstStructure * st1, GstStructure * st2,
1088 : const gchar * fieldname)
1089 : {
1090 : const char *dim_str1;
1091 : const char *dim_str2;
1092 :
1093 248 : g_return_val_if_fail (gst_structure_has_field (st1, fieldname), FALSE);
1094 248 : g_return_val_if_fail (gst_structure_has_field (st2, fieldname), FALSE);
1095 :
1096 248 : dim_str1 = gst_structure_get_string (st1, fieldname);
1097 248 : dim_str2 = gst_structure_get_string (st2, fieldname);
1098 :
1099 248 : return gst_tensor_dimension_string_is_equal (dim_str1, dim_str2);
1100 : }
1101 :
1102 : /**
1103 : * @brief Update caps dimensions for negotiation
1104 : * @param caps caps to compare and update
1105 : * @param filter caps to compare
1106 : */
1107 : void
1108 1636 : gst_tensor_caps_update_dimension (GstCaps * caps, GstCaps * filter)
1109 : {
1110 : GstStructure *st_caps, *st_filter;
1111 : guint i, j;
1112 :
1113 1636 : g_return_if_fail (GST_IS_CAPS (caps));
1114 1636 : g_return_if_fail (GST_IS_CAPS (filter));
1115 :
1116 4982 : for (i = 0; i < gst_caps_get_size (caps); i++) {
1117 3346 : st_caps = gst_caps_get_structure (caps, i);
1118 :
1119 3346 : if (!gst_structure_is_tensor_stream (st_caps))
1120 0 : continue;
1121 :
1122 9655 : for (j = 0; j < gst_caps_get_size (filter); j++) {
1123 6309 : st_filter = gst_caps_get_structure (filter, j);
1124 :
1125 6309 : if (!gst_structure_is_tensor_stream (st_filter))
1126 0 : continue;
1127 :
1128 : /* other/tensor */
1129 6309 : if (gst_structure_has_field (st_caps, "dimension")
1130 3435 : && gst_structure_has_field (st_filter, "dimension")) {
1131 : /* update dimensions for negotiation */
1132 236 : if (_is_structure_dimension_same (st_caps, st_filter, "dimension")) {
1133 118 : gst_structure_set (st_caps, "dimension", G_TYPE_STRING,
1134 : gst_structure_get_string (st_filter, "dimension"), NULL);
1135 : }
1136 : }
1137 : /* other/tensors */
1138 6191 : else if (gst_structure_has_field (st_caps, "dimensions")
1139 1866 : && gst_structure_has_field (st_filter, "dimensions")) {
1140 : /* update dimensions for negotiation */
1141 127 : if (_is_structure_dimension_same (st_caps, st_filter, "dimensions")) {
1142 118 : gst_structure_set (st_caps, "dimensions", G_TYPE_STRING,
1143 : gst_structure_get_string (st_filter, "dimensions"), NULL);
1144 : }
1145 : }
1146 : }
1147 : }
1148 : }
1149 :
1150 : /**
1151 : * @brief Try intersecting @caps1 and @caps2 for tensor stream
1152 : * @param caps1 a GstCaps to intersect
1153 : * @param caps2 a GstCaps to intersect
1154 : * @return TRUE if intersection would be not empty.
1155 : */
1156 : gboolean
1157 14 : gst_tensor_caps_can_intersect (GstCaps * caps1, GstCaps * caps2)
1158 : {
1159 : GstStructure *structure1;
1160 : GstStructure *structure2;
1161 : GstStructure *structure_copy1;
1162 : GstStructure *structure_copy2;
1163 :
1164 : const gchar *name1;
1165 : const gchar *name2;
1166 :
1167 : gboolean intersectable;
1168 :
1169 14 : if (gst_caps_can_intersect (caps1, caps2))
1170 10 : return TRUE;
1171 :
1172 4 : structure1 = gst_caps_get_structure (caps1, 0);
1173 4 : structure2 = gst_caps_get_structure (caps2, 0);
1174 :
1175 4 : if (!gst_structure_is_tensor_stream (structure1)
1176 4 : || !gst_structure_is_tensor_stream (structure2))
1177 0 : return FALSE;
1178 :
1179 4 : name1 = gst_structure_get_name (structure1);
1180 4 : name2 = gst_structure_get_name (structure2);
1181 :
1182 4 : if (!g_str_equal (name1, name2))
1183 1 : return FALSE;
1184 :
1185 : /* other/tensor */
1186 3 : if (g_str_equal (name1, NNS_MIMETYPE_TENSOR)) {
1187 3 : if (gst_structure_has_field (structure1, "dimension")
1188 3 : && gst_structure_has_field (structure2, "dimension")) {
1189 3 : if (!_is_structure_dimension_same (structure1, structure2, "dimension"))
1190 1 : return FALSE;
1191 : }
1192 : }
1193 : /* other/tensors */
1194 0 : else if (gst_structure_has_field (structure1, "dimensions")
1195 0 : && gst_structure_has_field (structure2, "dimensions")) {
1196 0 : if (!_is_structure_dimension_same (structure1, structure2, "dimensions"))
1197 0 : return FALSE;
1198 : }
1199 :
1200 2 : structure_copy1 = gst_structure_copy (structure1);
1201 2 : structure_copy2 = gst_structure_copy (structure2);
1202 :
1203 2 : gst_structure_remove_field (structure_copy1, "dimension");
1204 2 : gst_structure_remove_field (structure_copy1, "dimensions");
1205 2 : gst_structure_remove_field (structure_copy2, "dimension");
1206 2 : gst_structure_remove_field (structure_copy2, "dimensions");
1207 :
1208 : intersectable =
1209 2 : gst_structure_can_intersect (structure_copy1, structure_copy2);
1210 :
1211 2 : gst_structure_free (structure_copy1);
1212 2 : gst_structure_free (structure_copy2);
1213 :
1214 2 : return intersectable;
1215 : }
1216 :
1217 : /**
1218 : * @brief Get pad caps from tensors config and caps of the peer connected to the pad.
1219 : * @param pad GstPad to get possible caps
1220 : * @param config tensors config structure
1221 : * @return caps for given config. Caller is responsible for unreffing the returned caps.
1222 : */
1223 : GstCaps *
1224 1210 : gst_tensor_pad_caps_from_config (GstPad * pad, const GstTensorsConfig * config)
1225 : {
1226 1210 : GstCaps *caps = NULL;
1227 : GstCaps *templ;
1228 : gboolean is_flexible, peer_is_flexible, peer_has_tensor_caps;
1229 : GstCaps *peer_caps;
1230 :
1231 1210 : g_return_val_if_fail (GST_IS_PAD (pad), NULL);
1232 1210 : g_return_val_if_fail (config != NULL, NULL);
1233 :
1234 1210 : templ = gst_pad_get_pad_template_caps (pad);
1235 :
1236 : /* check peer caps */
1237 1210 : peer_is_flexible = peer_has_tensor_caps = FALSE;
1238 :
1239 1210 : peer_caps = gst_pad_peer_query_caps (pad, NULL);
1240 1210 : if (peer_caps && gst_caps_get_size (peer_caps) > 0) {
1241 : GstCaps *tmp;
1242 : GstStructure *st;
1243 : GstTensorsConfig peer_config;
1244 :
1245 914 : tmp = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
1246 914 : peer_has_tensor_caps = gst_caps_can_intersect (tmp, peer_caps);
1247 914 : gst_caps_unref (tmp);
1248 :
1249 914 : st = gst_caps_get_structure (peer_caps, 0);
1250 914 : if (gst_tensors_config_from_structure (&peer_config, st))
1251 914 : peer_is_flexible = gst_tensors_config_is_flexible (&peer_config);
1252 914 : gst_tensors_config_free (&peer_config);
1253 : }
1254 :
1255 : /* other/tensors (flexible) */
1256 1210 : is_flexible = gst_tensors_config_is_flexible (config);
1257 :
1258 1210 : if (is_flexible || peer_is_flexible) {
1259 70 : caps = _get_flexible_caps (config);
1260 70 : goto intersectable;
1261 : }
1262 :
1263 : /* other/tensor */
1264 1140 : if (config->info.num_tensors == 1 && peer_has_tensor_caps) {
1265 729 : caps = _get_tensor_caps (config);
1266 729 : if (peer_caps)
1267 729 : gst_tensor_caps_update_dimension (caps, peer_caps);
1268 :
1269 729 : if (gst_caps_can_intersect (caps, templ))
1270 725 : goto done;
1271 :
1272 4 : gst_caps_unref (caps);
1273 : }
1274 :
1275 : /* other/tensors (static) */
1276 415 : caps = _get_tensors_caps (config);
1277 415 : if (peer_caps)
1278 415 : gst_tensor_caps_update_dimension (caps, peer_caps);
1279 :
1280 0 : intersectable:
1281 485 : if (!gst_caps_can_intersect (caps, templ)) {
1282 0 : gst_caps_unref (caps);
1283 0 : caps = NULL;
1284 : }
1285 :
1286 485 : done:
1287 1210 : gst_caps_unref (templ);
1288 1210 : if (peer_caps)
1289 1210 : gst_caps_unref (peer_caps);
1290 1210 : caps = gst_caps_truncate (caps);
1291 1210 : return caps;
1292 : }
1293 :
1294 : /**
1295 : * @brief Get all possible caps from tensors config. Unlike gst_tensor_pad_caps_from_config(), this function does not check peer caps.
1296 : * @param pad GstPad to get possible caps
1297 : * @param config tensors config structure
1298 : * @return caps for given config. Caller is responsible for unreffing the returned caps.
1299 : */
1300 : GstCaps *
1301 2072 : gst_tensor_pad_possible_caps_from_config (GstPad * pad,
1302 : const GstTensorsConfig * config)
1303 : {
1304 : GstCaps *caps, *tmp;
1305 : GstCaps *templ;
1306 :
1307 2072 : g_return_val_if_fail (GST_IS_PAD (pad), NULL);
1308 2072 : g_return_val_if_fail (config != NULL, NULL);
1309 :
1310 2072 : caps = gst_caps_new_empty ();
1311 2072 : templ = gst_pad_get_pad_template_caps (pad);
1312 :
1313 : /* append caps for static tensor */
1314 2072 : if (gst_tensors_config_is_static (config)) {
1315 : /* other/tensor */
1316 1952 : if ((tmp = _get_tensor_caps (config)) != NULL) {
1317 1849 : if (gst_caps_can_intersect (tmp, templ))
1318 1801 : gst_caps_append (caps, tmp);
1319 : else
1320 48 : gst_caps_unref (tmp);
1321 : }
1322 :
1323 : /* other/tensors */
1324 1952 : if ((tmp = _get_tensors_caps (config)) != NULL) {
1325 1952 : if (gst_caps_can_intersect (tmp, templ))
1326 1952 : gst_caps_append (caps, tmp);
1327 : else
1328 0 : gst_caps_unref (tmp);
1329 : }
1330 : }
1331 :
1332 : /* caps for flexible tensor */
1333 2072 : if ((tmp = _get_flexible_caps (config)) != NULL) {
1334 2072 : if (gst_caps_can_intersect (tmp, templ))
1335 1979 : gst_caps_append (caps, tmp);
1336 : else
1337 93 : gst_caps_unref (tmp);
1338 : }
1339 :
1340 : /* if no possible caps for given config, return null. */
1341 2072 : if (gst_caps_is_empty (caps)) {
1342 0 : gst_caps_unref (caps);
1343 0 : caps = NULL;
1344 : }
1345 :
1346 2072 : gst_caps_unref (templ);
1347 2072 : return caps;
1348 : }
1349 :
1350 : /**
1351 : * @brief Get tensor format of current pad caps.
1352 : * @param pad GstPad to check current caps.
1353 : * @return The tensor_format of current pad caps.
1354 : *
1355 : * If pad does not have tensor caps return _NNS_TENSOR_FORMAT_END
1356 : */
1357 : tensor_format
1358 128678 : gst_tensor_pad_get_format (GstPad * pad)
1359 : {
1360 : GstCaps *caps;
1361 128678 : tensor_format ret = _NNS_TENSOR_FORMAT_END;
1362 :
1363 128678 : g_return_val_if_fail (GST_IS_PAD (pad), _NNS_TENSOR_FORMAT_END);
1364 :
1365 128678 : caps = gst_pad_get_current_caps (pad);
1366 128678 : if (caps) {
1367 : GstTensorsConfig config;
1368 :
1369 128664 : if (gst_tensors_config_from_caps (&config, caps, TRUE)) {
1370 128553 : ret = config.info.format;
1371 : }
1372 128664 : gst_caps_unref (caps);
1373 128664 : gst_tensors_config_free (&config);
1374 : }
1375 :
1376 128678 : return ret;
1377 : }
1378 :
1379 : /**
1380 : * @brief Get caps from tensors config (for other/tensors)
1381 : * @param config tensors config info
1382 : * @return caps for given config
1383 : */
1384 : GstCaps *
1385 3431 : gst_tensors_caps_from_config (const GstTensorsConfig * config)
1386 : {
1387 : GstCaps *caps;
1388 :
1389 3431 : g_return_val_if_fail (config != NULL, NULL);
1390 :
1391 3430 : if (gst_tensors_config_is_flexible (config)) {
1392 721 : caps = _get_flexible_caps (config);
1393 : } else {
1394 2709 : caps = _get_tensors_caps (config);
1395 : }
1396 :
1397 3430 : caps = gst_caps_truncate (caps);
1398 :
1399 3430 : return caps;
1400 : }
1401 :
1402 : /**
1403 : * @brief Get tensor caps from tensors config
1404 : * @param config tensors config info
1405 : * @return caps for given config
1406 : */
1407 : GstCaps *
1408 2981 : gst_tensor_caps_from_config (const GstTensorsConfig * config)
1409 : {
1410 : GstCaps *caps;
1411 :
1412 2981 : g_return_val_if_fail (config != NULL, NULL);
1413 :
1414 2980 : caps = _get_tensor_caps (config);
1415 2980 : caps = gst_caps_truncate (caps);
1416 :
1417 2980 : return caps;
1418 : }
1419 :
1420 : /**
1421 : * @brief Parse structure and set tensors config (for other/tensors)
1422 : * @param config tensors config structure to be filled
1423 : * @param structure structure to be interpreted
1424 : * @return TRUE if no error
1425 : */
1426 : gboolean
1427 158047 : gst_tensors_config_from_structure (GstTensorsConfig * config,
1428 : const GstStructure * structure)
1429 : {
1430 : const gchar *name;
1431 158047 : tensor_format format = _NNS_TENSOR_FORMAT_STATIC;
1432 :
1433 158047 : g_return_val_if_fail (config != NULL, FALSE);
1434 158045 : gst_tensors_config_init (config);
1435 :
1436 158045 : g_return_val_if_fail (structure != NULL, FALSE);
1437 :
1438 158044 : name = gst_structure_get_name (structure);
1439 :
1440 158044 : if (g_str_equal (name, NNS_MIMETYPE_TENSOR)) {
1441 : /* other/tensor is always static */
1442 144720 : config->info.num_tensors = 1;
1443 :
1444 144720 : if (gst_structure_has_field (structure, "dimension")) {
1445 138846 : const gchar *dim_str = gst_structure_get_string (structure, "dimension");
1446 138846 : gst_tensor_parse_dimension (dim_str, config->info.info[0].dimension);
1447 : }
1448 :
1449 144720 : if (gst_structure_has_field (structure, "type")) {
1450 138989 : const gchar *type_str = gst_structure_get_string (structure, "type");
1451 138989 : config->info.info[0].type = gst_tensor_get_type (type_str);
1452 : }
1453 13324 : } else if (g_str_equal (name, NNS_MIMETYPE_TENSORS)) {
1454 13213 : if (gst_structure_has_field (structure, "format")) {
1455 : const gchar *format_str;
1456 :
1457 13211 : format_str = gst_structure_get_string (structure, "format");
1458 13211 : format = gst_tensor_get_format (format_str);
1459 :
1460 13211 : if (format == _NNS_TENSOR_FORMAT_END) {
1461 1486 : GST_INFO
1462 : ("Invalid format %s, it should be one of %s. Suppose tensor format is static.",
1463 : _STR_NULL (format_str), GST_TENSOR_FORMAT_ALL);
1464 : } else {
1465 11725 : config->info.format = format;
1466 : }
1467 : }
1468 :
1469 13213 : if (config->info.format == _NNS_TENSOR_FORMAT_STATIC) {
1470 11263 : gst_structure_get_int (structure, "num_tensors",
1471 11263 : (gint *) (&config->info.num_tensors));
1472 :
1473 : /* parse dimensions */
1474 11263 : if (gst_structure_has_field (structure, "dimensions")) {
1475 : const gchar *dims_str;
1476 : guint num_dims;
1477 :
1478 9018 : dims_str = gst_structure_get_string (structure, "dimensions");
1479 : num_dims =
1480 9018 : gst_tensors_info_parse_dimensions_string (&config->info, dims_str);
1481 :
1482 9018 : if (config->info.num_tensors != num_dims) {
1483 8 : nns_logw ("Invalid param, dimensions (%d) tensors (%d)\n",
1484 : num_dims, config->info.num_tensors);
1485 : }
1486 : }
1487 :
1488 : /* parse types */
1489 11263 : if (gst_structure_has_field (structure, "types")) {
1490 : const gchar *types_str;
1491 : guint num_types;
1492 :
1493 8970 : types_str = gst_structure_get_string (structure, "types");
1494 : num_types =
1495 8970 : gst_tensors_info_parse_types_string (&config->info, types_str);
1496 :
1497 8970 : if (config->info.num_tensors != num_types) {
1498 0 : nns_logw ("Invalid param, types (%d) tensors (%d)\n",
1499 : num_types, config->info.num_tensors);
1500 : }
1501 : }
1502 : }
1503 : } else {
1504 111 : nns_logw ("Unsupported type = %s\n", name ? name : "Unknown");
1505 111 : return FALSE;
1506 : }
1507 :
1508 157933 : if (gst_structure_has_field (structure, "framerate")) {
1509 157914 : gst_structure_get_fraction (structure, "framerate", &config->rate_n,
1510 157914 : &config->rate_d);
1511 : }
1512 :
1513 157933 : return TRUE;
1514 : }
1515 :
1516 : /**
1517 : * @brief Parse caps and set tensors config (for other/tensors)
1518 : * @param[out] config tensors config structure to be filled
1519 : * @param[in] caps incoming capability
1520 : * @param[in] validate TRUE to validate configuration
1521 : * @return TRUE/FALSE (if successfully configured, return TRUE)
1522 : */
1523 : gboolean
1524 149075 : gst_tensors_config_from_caps (GstTensorsConfig * config, const GstCaps * caps,
1525 : const gboolean validate)
1526 : {
1527 : GstStructure *structure;
1528 149075 : gboolean ret = FALSE;
1529 :
1530 149075 : gst_tensors_config_init (config);
1531 :
1532 149075 : if (validate && !gst_caps_is_fixed (caps)) {
1533 529 : nns_logw ("GstCaps is not fixed.");
1534 529 : return FALSE;
1535 : }
1536 :
1537 148546 : structure = gst_caps_get_structure (caps, 0);
1538 148546 : ret = gst_tensors_config_from_structure (config, structure);
1539 :
1540 148546 : if (ret && validate) {
1541 145697 : ret = gst_tensors_config_validate (config);
1542 : }
1543 :
1544 148546 : if (!ret) {
1545 114 : gst_tensors_config_free (config);
1546 : }
1547 :
1548 148546 : return ret;
1549 : }
1550 :
1551 : /**
1552 : * @brief Parse memory and fill the tensor meta.
1553 : * @param[out] meta tensor meta structure to be filled
1554 : * @param[in] mem pointer to GstMemory to be parsed
1555 : * @return TRUE if successfully set the meta
1556 : */
1557 : gboolean
1558 60773 : gst_tensor_meta_info_parse_memory (GstTensorMetaInfo * meta, GstMemory * mem)
1559 : {
1560 : GstMapInfo map;
1561 : gsize hsize, msize;
1562 : gboolean ret;
1563 :
1564 121545 : g_return_val_if_fail (mem != NULL, FALSE);
1565 60772 : g_return_val_if_fail (meta != NULL, FALSE);
1566 :
1567 60771 : gst_tensor_meta_info_init (meta);
1568 :
1569 : /* Check header size of tensor-meta. */
1570 60771 : hsize = gst_tensor_meta_info_get_header_size (meta);
1571 60771 : msize = gst_memory_get_sizes (mem, NULL, NULL);
1572 60771 : if (msize < hsize)
1573 8446 : return FALSE;
1574 :
1575 52325 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
1576 0 : nns_loge ("Failed to get the meta, cannot map the memory.");
1577 0 : return FALSE;
1578 : }
1579 :
1580 52325 : ret = gst_tensor_meta_info_parse_header (meta, map.data);
1581 :
1582 52325 : gst_memory_unmap (mem, &map);
1583 52325 : return ret;
1584 : }
1585 :
1586 : /**
1587 : * @brief Append header to memory.
1588 : * @param[in] meta tensor meta structure
1589 : * @param[in] mem pointer to GstMemory
1590 : * @return Newly allocated GstMemory (Caller should free returned memory using gst_memory_unref())
1591 : */
1592 : GstMemory *
1593 281 : gst_tensor_meta_info_append_header (GstTensorMetaInfo * meta, GstMemory * mem)
1594 : {
1595 281 : GstMemory *new_mem = NULL;
1596 : gsize msize, hsize;
1597 : GstMapInfo old_map, new_map;
1598 :
1599 561 : g_return_val_if_fail (mem != NULL, NULL);
1600 280 : g_return_val_if_fail (gst_tensor_meta_info_validate (meta), NULL);
1601 :
1602 279 : if (!gst_memory_map (mem, &old_map, GST_MAP_READ)) {
1603 0 : nns_loge ("Failed to append header, cannot map the old memory.");
1604 0 : return NULL;
1605 : }
1606 :
1607 : /* memory size (header + old memory) */
1608 279 : hsize = gst_tensor_meta_info_get_header_size (meta);
1609 279 : msize = hsize + old_map.size;
1610 :
1611 279 : new_mem = gst_allocator_alloc (NULL, msize, NULL);
1612 279 : if (!gst_memory_map (new_mem, &new_map, GST_MAP_WRITE)) {
1613 0 : nns_loge ("Failed to append header, cannot map the new memory.");
1614 0 : gst_memory_unmap (mem, &old_map);
1615 0 : gst_memory_unref (new_mem);
1616 0 : return NULL;
1617 : }
1618 :
1619 : /* set header and copy old data */
1620 279 : gst_tensor_meta_info_update_header (meta, new_map.data);
1621 279 : memcpy (new_map.data + hsize, old_map.data, old_map.size);
1622 :
1623 279 : gst_memory_unmap (mem, &old_map);
1624 279 : gst_memory_unmap (new_mem, &new_map);
1625 279 : return new_mem;
1626 : }
1627 :
1628 : /**
1629 : * @brief Get the nth GstMemory from given @a buffer.
1630 : * @param[in] buffer GstBuffer to be parsed.
1631 : * @param[in] index Index of GstMemory to be returned.
1632 : * @return GstMemory if found, otherwise NULL (Caller should free returned memory using gst_memory_unref()).
1633 : */
1634 : GstMemory *
1635 62959 : gst_tensor_buffer_get_nth_memory (GstBuffer * buffer, const guint index)
1636 : {
1637 : guint i, num_tensors;
1638 : gsize offset;
1639 62959 : GstMemory *extra_tensors_memory, *res_mem = NULL;
1640 : GstMapInfo extra_tensors_map;
1641 : GstTensorExtraInfo *extra_info;
1642 :
1643 62959 : if (!GST_IS_BUFFER (buffer)) {
1644 0 : nns_loge ("Failed to parse GstBuffer (invalid input buffer).");
1645 62959 : return NULL;
1646 : }
1647 :
1648 62959 : num_tensors = gst_tensor_buffer_get_count (buffer);
1649 62959 : if (index >= num_tensors) {
1650 0 : nns_loge ("Invalid index %u, the number of tensors in the buffer is %u.",
1651 : index, num_tensors);
1652 0 : return NULL;
1653 : }
1654 :
1655 : /* If num_tensors is less than or equal to NNS_TENSOR_MEMORY_MAX, it's trivial. */
1656 62959 : if (num_tensors <= NNS_TENSOR_MEMORY_MAX || index < NNS_TENSOR_MEMORY_MAX - 1) {
1657 62338 : return gst_buffer_get_memory (buffer, index);
1658 : }
1659 :
1660 : /* If num_tensors is greater than NNS_TENSOR_MEMORY_MAX, we need to parse extra info. */
1661 : extra_tensors_memory =
1662 621 : gst_buffer_peek_memory (buffer, NNS_TENSOR_MEMORY_MAX - 1);
1663 621 : if (!extra_tensors_memory) {
1664 0 : nns_loge ("Failed to get %d-th memory", NNS_TENSOR_MEMORY_MAX);
1665 0 : return NULL;
1666 : }
1667 :
1668 621 : if (!gst_memory_map (extra_tensors_memory, &extra_tensors_map, GST_MAP_READ)) {
1669 0 : nns_loge ("Failed to map %d-th memory", NNS_TENSOR_MEMORY_MAX);
1670 0 : return NULL;
1671 : }
1672 :
1673 : /* check header (extra info) of the memory */
1674 621 : if (!gst_memory_map_is_extra_tensor (&extra_tensors_map)) {
1675 0 : nns_loge ("Invalid extra header");
1676 0 : goto done;
1677 : }
1678 :
1679 : /* parse the memory */
1680 621 : extra_info = (GstTensorExtraInfo *) extra_tensors_map.data;
1681 621 : offset = sizeof (GstTensorExtraInfo);
1682 :
1683 : /* If index is NNS_TENSOR_MEMORY_MAX - 1 */
1684 621 : if (index == NNS_TENSOR_MEMORY_MAX - 1) {
1685 : res_mem =
1686 29 : gst_memory_share (extra_tensors_memory, offset, extra_info->reserved);
1687 29 : goto done;
1688 : }
1689 :
1690 592 : offset += extra_info->reserved;
1691 :
1692 31720 : for (i = 1; i <= index - NNS_TENSOR_MEMORY_MAX; ++i) {
1693 31128 : offset += gst_tensor_info_get_size (&extra_info->infos[i - 1]);
1694 : }
1695 :
1696 : /* wrap it as GstMemory */
1697 : res_mem =
1698 592 : gst_memory_share (extra_tensors_memory, offset,
1699 592 : gst_tensor_info_get_size (&extra_info->infos[index -
1700 : NNS_TENSOR_MEMORY_MAX]));
1701 :
1702 621 : done:
1703 621 : gst_memory_unmap (extra_tensors_memory, &extra_tensors_map);
1704 621 : return res_mem;
1705 : }
1706 :
1707 : /**
1708 : * @brief Append @a memory to given @a buffer.
1709 : * @param[in/out] buffer GstBuffer to be appended.
1710 : * @param[in] memory GstMemory to append. This function takes ownership of this, even if it returns failure.
1711 : * @param[in] info GstTensorInfo of given @a memory.
1712 : * @return TRUE if successfully appended, otherwise FALSE.
1713 : */
1714 : gboolean
1715 60748 : gst_tensor_buffer_append_memory (GstBuffer * buffer, GstMemory * memory,
1716 : const GstTensorInfo * info)
1717 : {
1718 : guint num_mems, new_mem_index;
1719 60748 : GstMemory *new_memory = NULL, *last_memory = NULL;
1720 : gsize offset, new_mem_size, last_mem_size;
1721 : GstMapInfo new_memory_map, last_memory_map, incoming_memory_map;
1722 : GstTensorExtraInfo *extra_info;
1723 : GstTensorMetaInfo meta;
1724 : gboolean is_extra, is_static;
1725 60748 : gboolean appended = FALSE;
1726 :
1727 60748 : if (!GST_IS_BUFFER (buffer)) {
1728 0 : nns_loge ("Failed to append memory, given buffer is invalid.");
1729 0 : goto failed;
1730 : }
1731 :
1732 60748 : if (!memory) {
1733 0 : nns_loge ("Failed to append memory, given memory is NULL.");
1734 0 : goto failed;
1735 : }
1736 :
1737 60748 : if (gst_tensor_meta_info_parse_memory (&meta, memory)) {
1738 555 : is_static = (meta.format == _NNS_TENSOR_FORMAT_STATIC);
1739 : } else {
1740 : /* Suppose given memory is static tensor. */
1741 60193 : is_static = TRUE;
1742 :
1743 : /* Error case if given tensor-info is invalid. */
1744 60193 : if (!gst_tensor_info_validate (info)) {
1745 0 : nns_loge ("Failed to get tensor info (invalid input info).");
1746 0 : goto failed;
1747 : }
1748 : }
1749 :
1750 60748 : num_mems = gst_buffer_n_memory (buffer);
1751 :
1752 : /* trivial call to gst_buffer_append_memory */
1753 60748 : if (num_mems < NNS_TENSOR_MEMORY_MAX) {
1754 60160 : gst_buffer_append_memory (buffer, memory);
1755 60748 : return TRUE;
1756 : }
1757 :
1758 : /* given buffer has NNS_TENSOR_MEMORY_MAX memory blocks */
1759 588 : last_memory = gst_buffer_peek_memory (buffer, num_mems - 1);
1760 588 : if (!last_memory) {
1761 0 : nns_loge ("Failed to get last memory");
1762 0 : goto failed;
1763 : }
1764 :
1765 588 : if (!gst_memory_map (last_memory, &last_memory_map, GST_MAP_READ)) {
1766 0 : nns_loge ("Failed to map last memory");
1767 0 : last_memory = NULL;
1768 0 : goto failed;
1769 : }
1770 :
1771 588 : new_mem_size = last_mem_size = gst_memory_get_sizes (last_memory, NULL, NULL);
1772 :
1773 : /* if the memory does not have proper header, append it */
1774 588 : is_extra = gst_memory_map_is_extra_tensor (&last_memory_map);
1775 588 : if (!is_extra) {
1776 28 : new_mem_size += sizeof (GstTensorExtraInfo);
1777 : }
1778 :
1779 588 : new_mem_size += gst_memory_get_sizes (memory, NULL, NULL);
1780 :
1781 588 : new_memory = gst_allocator_alloc (NULL, new_mem_size, NULL);
1782 588 : if (!new_memory) {
1783 0 : nns_loge ("Failed to allocate memory for extra tensors.");
1784 0 : goto failed;
1785 : }
1786 :
1787 588 : if (!gst_memory_map (new_memory, &new_memory_map, GST_MAP_WRITE)) {
1788 0 : nns_loge ("Failed to map extra memory");
1789 0 : gst_memory_unref (new_memory);
1790 0 : new_memory = NULL;
1791 0 : goto failed;
1792 : }
1793 :
1794 588 : if (!gst_memory_map (memory, &incoming_memory_map, GST_MAP_READ)) {
1795 0 : nns_loge ("Failed to map incoming memory");
1796 0 : goto failed;
1797 : }
1798 :
1799 588 : extra_info = (GstTensorExtraInfo *) new_memory_map.data;
1800 :
1801 : /* if the last_memory does not have proper header, append it */
1802 588 : if (!is_extra) {
1803 28 : gst_tensor_extra_info_init (extra_info, last_mem_size);
1804 28 : offset = sizeof (GstTensorExtraInfo);
1805 : } else {
1806 560 : offset = 0;
1807 : }
1808 :
1809 : /* copy last_memory into new_memory */
1810 588 : memcpy (new_memory_map.data + offset, last_memory_map.data,
1811 : last_memory_map.size);
1812 :
1813 : /* copy incoming_memory into new_memory */
1814 588 : new_mem_index = extra_info->num_extra_tensors;
1815 588 : extra_info->num_extra_tensors += 1;
1816 :
1817 : /* Copy tensor info into extra. */
1818 588 : if (is_static) {
1819 588 : gst_tensor_info_copy (&extra_info->infos[new_mem_index], info);
1820 :
1821 : /**
1822 : * Free the name string, cause it does not freed by gstreamer.
1823 : * @todo Make custom gst_allocator later?
1824 : */
1825 588 : g_clear_pointer (&extra_info->infos[new_mem_index].name, g_free);
1826 : } else {
1827 0 : gst_tensor_meta_info_convert (&meta, &extra_info->infos[new_mem_index]);
1828 : }
1829 :
1830 588 : memcpy (new_memory_map.data + offset + last_memory_map.size,
1831 588 : incoming_memory_map.data, incoming_memory_map.size);
1832 :
1833 588 : gst_memory_unmap (memory, &incoming_memory_map);
1834 588 : gst_memory_unmap (last_memory, &last_memory_map);
1835 588 : last_memory = NULL;
1836 :
1837 588 : gst_buffer_replace_memory (buffer, num_mems - 1, new_memory);
1838 588 : appended = TRUE;
1839 :
1840 588 : failed:
1841 588 : if (new_memory) {
1842 588 : gst_memory_unmap (new_memory, &new_memory_map);
1843 588 : if (!appended)
1844 0 : gst_memory_unref (new_memory);
1845 : }
1846 :
1847 588 : if (last_memory)
1848 0 : gst_memory_unmap (last_memory, &last_memory_map);
1849 :
1850 : /* Release incoming memory even if failed to append it into buffer. */
1851 588 : if (memory)
1852 588 : gst_memory_unref (memory);
1853 :
1854 588 : return appended;
1855 : }
1856 :
1857 : /**
1858 : * @brief Get the number of tensors in the buffer.
1859 : */
1860 : guint
1861 123230 : gst_tensor_buffer_get_count (GstBuffer * buffer)
1862 : {
1863 : guint num_mems;
1864 : GstMemory *mem;
1865 : GstMapInfo map;
1866 : GstTensorExtraInfo *extra_info;
1867 :
1868 246460 : g_return_val_if_fail (buffer != NULL, 0);
1869 :
1870 123230 : num_mems = gst_buffer_n_memory (buffer);
1871 123230 : if (num_mems < NNS_TENSOR_MEMORY_MAX) {
1872 121529 : return num_mems;
1873 : }
1874 :
1875 : /* num_mems == NNS_TENSOR_MEMORY_MAX */
1876 1701 : mem = gst_buffer_peek_memory (buffer, num_mems - 1);
1877 1701 : if (!mem) {
1878 0 : nns_loge ("Failed to get the last memory.");
1879 0 : return 0;
1880 : }
1881 :
1882 1701 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
1883 0 : nns_loge ("Failed to map the last memory.");
1884 0 : return 0;
1885 : }
1886 :
1887 1701 : if (gst_memory_map_is_extra_tensor (&map)) {
1888 1349 : extra_info = (GstTensorExtraInfo *) map.data;
1889 1349 : num_mems = extra_info->num_extra_tensors + NNS_TENSOR_MEMORY_MAX;
1890 : } else {
1891 352 : nns_logi ("The last memory does not have extra tensors header. "
1892 : "Assuming the number of tensors is %d.", num_mems);
1893 : }
1894 :
1895 1701 : gst_memory_unmap (mem, &map);
1896 :
1897 1701 : return num_mems;
1898 : }
1899 :
1900 : /**
1901 : * @brief Sets the value of a property based on the specified property value and GParamSpec.
1902 : *
1903 : * @param prop_value A pointer to the GValue where the property value will be set.
1904 : * @param param_spec A pointer to the GParamSpec that describes the property.
1905 : * @param property_value A string representing the value to be set for the property.
1906 : *
1907 : * @note This API is intended to be used by gst_tensor_parse_config_file ()
1908 : */
1909 : static void
1910 34 : set_property_value (GValue * prop_value, const GParamSpec * param_spec,
1911 : const gchar * property_value)
1912 : {
1913 34 : GType value_type = G_PARAM_SPEC_VALUE_TYPE (param_spec);
1914 :
1915 34 : g_value_init (prop_value, value_type);
1916 :
1917 34 : if (value_type == G_TYPE_BOOLEAN) {
1918 0 : gboolean value = g_ascii_strcasecmp (property_value, "true") == 0;
1919 0 : g_value_set_boolean (prop_value, value);
1920 34 : } else if (value_type == G_TYPE_INT) {
1921 0 : gint value = atoi (property_value);
1922 0 : g_value_set_int (prop_value, value);
1923 34 : } else if (value_type == G_TYPE_UINT) {
1924 0 : guint value = atoi (property_value);
1925 0 : g_value_set_uint (prop_value, value);
1926 34 : } else if (value_type == G_TYPE_FLOAT) {
1927 0 : gfloat value = atof (property_value);
1928 0 : g_value_set_float (prop_value, value);
1929 34 : } else if (value_type == G_TYPE_DOUBLE) {
1930 0 : gdouble value = atof (property_value);
1931 0 : g_value_set_double (prop_value, value);
1932 : } else {
1933 34 : g_value_set_string (prop_value, property_value); /** default is string */
1934 : }
1935 34 : }
1936 :
1937 : /**
1938 : * @brief Parses a configuration file and sets the corresponding properties on a GObject.
1939 : *
1940 : * This function reads the contents of the configuration file located at the given path
1941 : * and sets the properties of the specified GObject based on the configuration data.
1942 : *
1943 : * @param config_path The path to the configuration file.
1944 : * @param object The GObject on which to set the properties.
1945 : *
1946 : * @note The responsibility of managing the memory of the GObject passed as a parameter
1947 : * lies outside this function.
1948 : */
1949 : void
1950 9 : gst_tensor_parse_config_file (const gchar * config_path, const GObject * object)
1951 : {
1952 9 : g_autofree gchar *config_data = NULL;
1953 9 : g_auto (GStrv) lines = NULL;
1954 9 : GStrv line = NULL;
1955 9 : GError *error = NULL;
1956 9 : GObjectClass *g_object_class = G_OBJECT_GET_CLASS (object);
1957 :
1958 9 : if (!g_file_get_contents (config_path, &config_data, NULL, &error)) {
1959 0 : GST_DEBUG ("Failed to read config file: %s\n", error->message);
1960 0 : g_error_free (error);
1961 0 : return;
1962 : }
1963 :
1964 9 : lines = g_strsplit (config_data, "\n", -1);
1965 :
1966 : /** Iterate over each line */
1967 52 : for (line = lines; *line; ++line) {
1968 43 : g_auto (GStrv) parts = g_strsplit (*line, "=", 2);
1969 :
1970 43 : if (g_strv_length (parts) == 2) {
1971 68 : g_autofree gchar *property_name = g_strstrip (g_strdup (parts[0]));
1972 68 : g_autofree gchar *property_value = g_strstrip (g_strdup (parts[1]));
1973 :
1974 : GParamSpec *pdata =
1975 34 : g_object_class_find_property (g_object_class, property_name);
1976 :
1977 34 : if (pdata != NULL) {
1978 34 : GValue prop_value = G_VALUE_INIT;
1979 34 : set_property_value (&prop_value, pdata, property_value);
1980 34 : g_object_set_property (G_OBJECT (object), pdata->name, &prop_value);
1981 34 : g_value_unset (&prop_value);
1982 : }
1983 : }
1984 : }
1985 : }
|
