Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * NNStreamer Common Header's Contents (pipeline extension)
4 : * Copyright (C) 2020 MyungJoo Ham <myungjoo.ham@samsung.com>
5 : */
6 : /**
7 : * @file nnstreamer_plugin_api_impl.c
8 : * @date 14 Apr 2020
9 : * @brief Common data for NNStreamer, the GStreamer plugin for neural networks
10 : * @see https://github.com/nnstreamer/nnstreamer
11 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
12 : * @bug No known bugs except for NYI items
13 : *
14 : */
15 :
16 : #include <nnstreamer_util.h>
17 : #include <string.h>
18 : #include <tensor_common.h>
19 :
20 : static const gchar *gst_tensor_time_sync_mode_string[] = {
21 : [SYNC_NOSYNC] = "nosync",
22 : [SYNC_SLOWEST] = "slowest",
23 : [SYNC_BASEPAD] = "basepad",
24 : [SYNC_REFRESH] = "refresh",
25 : [SYNC_END] = NULL
26 : };
27 :
28 : /**
29 : * @brief The old rank of tensor.
30 : */
31 : #define NNS_TENSOR_RANK_LIMIT_PREV (4)
32 :
33 : #define NNS_TENSOR_EXTRA_MAGIC 0xf00dc0de
34 :
35 : /**
36 : * @brief Data structure to describe a "extra" tensor data.
37 : * This represents the information of the NNS_TENSOR_SIZE_LIMIT-th memory block for tensor stream.
38 : */
39 : typedef struct
40 : {
41 : uint32_t magic;
42 : uint32_t version;
43 : uint32_t num_extra_tensors;
44 : uint64_t reserved;
45 : GstTensorInfo infos[NNS_TENSOR_SIZE_EXTRA_LIMIT];
46 : } GstTensorExtraInfo;
47 :
48 : /**
49 : * @brief Check if given memory has extra tensors.
50 : * @param[in] map GstMapInfo of GstMemory to be checked.
51 : * @return TRUE if @map has extra tensors, otherwise FALSE.
52 : */
53 : static gboolean
54 2910 : gst_memory_map_is_extra_tensor (GstMapInfo * map)
55 : {
56 : GstTensorExtraInfo *extra_info;
57 : gboolean is_extra;
58 :
59 2910 : g_return_val_if_fail (map != NULL, FALSE);
60 :
61 2910 : if (map->size < sizeof (GstTensorExtraInfo))
62 380 : return FALSE;
63 :
64 2530 : extra_info = (GstTensorExtraInfo *) map->data;
65 :
66 : /* check magic in header (extra info) of the memory */
67 2530 : is_extra = (extra_info && extra_info->magic == NNS_TENSOR_EXTRA_MAGIC);
68 :
69 2530 : return is_extra;
70 : }
71 :
72 : /**
73 : * @brief Initialize GstTensorExtraInfo structure with given @a memory.
74 : * @param[in/out] extra GstTensorExtraInfo to be initialized.
75 : * @param[in] reserved_size The memory size of extra memory block.
76 : */
77 : static void
78 28 : gst_tensor_extra_info_init (GstTensorExtraInfo * extra, gsize reserved_size)
79 : {
80 : guint i;
81 :
82 28 : g_return_if_fail (extra != NULL);
83 :
84 28 : extra->magic = NNS_TENSOR_EXTRA_MAGIC;
85 28 : extra->version = 0;
86 28 : extra->num_extra_tensors = 0;
87 :
88 : /* set reserved size of NNS_TENSOR_SIZE_LIMIT-th memory */
89 28 : extra->reserved = reserved_size;
90 6748 : for (i = 0; i < NNS_TENSOR_SIZE_EXTRA_LIMIT; ++i) {
91 6720 : gst_tensor_info_init (&extra->infos[i]);
92 : }
93 : }
94 :
95 : /**
96 : * @brief Get the corresponding mode from the string value.
97 : * @param[in] str The string value for the mode.
98 : * @return Corresponding mode for the string. SYNC_END for errors.
99 : */
100 : tensor_time_sync_mode
101 66 : gst_tensor_time_sync_get_mode (const gchar * str)
102 : {
103 : gint index;
104 :
105 66 : index = find_key_strv (gst_tensor_time_sync_mode_string, str);
106 :
107 66 : return (index < 0) ? SYNC_END : index;
108 : }
109 :
110 : /**
111 : * @brief Get the time-sync mode string.
112 : * @return Corresponding mode string.
113 : */
114 : const gchar *
115 2 : gst_tensor_time_sync_get_mode_string (tensor_time_sync_mode mode)
116 : {
117 2 : return gst_tensor_time_sync_mode_string[mode];
118 : }
119 :
120 : /**
121 : * @brief Setup time sync option.
122 : * @param[in/out] filter "this" pointer. Sync mode & option MUST BE set already.
123 : * @return True if successfully set the option.
124 : */
125 : gboolean
126 95 : gst_tensor_time_sync_set_option_data (tensor_time_sync_data * sync)
127 : {
128 95 : g_return_val_if_fail (sync != NULL, FALSE);
129 :
130 95 : if (sync->mode == SYNC_END || sync->option == NULL)
131 66 : return FALSE;
132 :
133 29 : switch (sync->mode) {
134 0 : case SYNC_NOSYNC:
135 0 : break;
136 0 : case SYNC_SLOWEST:
137 0 : break;
138 29 : case SYNC_BASEPAD:
139 : {
140 29 : g_auto (GStrv) strv = g_strsplit (sync->option, ":", 2);
141 : guint sink_id;
142 : guint duration;
143 :
144 29 : if (strv[0] != NULL)
145 29 : sink_id = (guint) g_ascii_strtoull (strv[0], NULL, 10);
146 : else
147 0 : sink_id = 0;
148 :
149 29 : if (strv[1] != NULL)
150 27 : duration = (guint) g_ascii_strtoull (strv[1], NULL, 10);
151 : else
152 2 : duration = G_MAXINT;
153 :
154 29 : sync->data_basepad.sink_id = sink_id;
155 29 : sync->data_basepad.duration = duration;
156 29 : break;
157 : }
158 0 : default:
159 : /* unknown mode */
160 0 : GST_WARNING ("Unknown mode = %d", sync->mode);
161 0 : return FALSE;
162 : }
163 :
164 29 : return TRUE;
165 : }
166 :
167 : /**
168 : * @brief Internal function to detect EOS using the number of empty pads.
169 : * @param[in] collect Collect pad.
170 : * @param[in] sync Synchronization option.
171 : * @param[in] empty The number of empty pads (pad has no buffer).
172 : * @return True if EOS.
173 : */
174 : static gboolean
175 13386 : _gst_tensor_time_sync_is_eos (GstCollectPads * collect,
176 : tensor_time_sync_data * sync, guint empty)
177 : {
178 : guint total;
179 13386 : gboolean is_eos = FALSE;
180 :
181 13386 : total = g_slist_length (collect->data);
182 :
183 13386 : switch (sync->mode) {
184 11458 : case SYNC_REFRESH:
185 11458 : if (empty == total)
186 2 : is_eos = TRUE;
187 11458 : break;
188 1928 : default:
189 1928 : if (empty > 0)
190 214 : is_eos = TRUE;
191 1928 : break;
192 : }
193 :
194 13386 : return is_eos;
195 : }
196 :
197 : /**
198 : * @brief A function call to decide current timestamp among collected pads based on PTS.
199 : * It will decide current timestamp according to sync option.
200 : * GstMeta is also copied with same sync mode.
201 : */
202 : gboolean
203 6801 : gst_tensor_time_sync_get_current_time (GstCollectPads * collect,
204 : tensor_time_sync_data * sync, GstClockTime * current_time,
205 : GstBuffer * tensors_buf)
206 : {
207 6801 : GSList *walk = NULL;
208 : guint count, empty_pad;
209 :
210 6801 : g_return_val_if_fail (collect != NULL, FALSE);
211 6801 : g_return_val_if_fail (sync != NULL, FALSE);
212 6801 : g_return_val_if_fail (current_time != NULL, FALSE);
213 :
214 6801 : walk = collect->data;
215 6801 : count = empty_pad = 0;
216 :
217 21472 : while (walk) {
218 : GstCollectData *data;
219 : GstBuffer *buf;
220 14671 : gboolean need_update = FALSE;
221 :
222 14671 : data = (GstCollectData *) walk->data;
223 14671 : buf = gst_collect_pads_peek (collect, data);
224 14671 : walk = g_slist_next (walk);
225 :
226 14671 : if (buf) {
227 8400 : switch (sync->mode) {
228 7742 : case SYNC_NOSYNC:
229 : /* fall-through */
230 : case SYNC_SLOWEST:
231 : case SYNC_REFRESH:
232 7742 : if (*current_time < GST_BUFFER_PTS (buf))
233 6284 : need_update = TRUE;
234 7742 : break;
235 658 : case SYNC_BASEPAD:
236 658 : if (count == sync->data_basepad.sink_id)
237 296 : need_update = TRUE;
238 658 : break;
239 0 : default:
240 0 : break;
241 : }
242 8400 : if (need_update) {
243 6580 : *current_time = GST_BUFFER_PTS (buf);
244 6580 : gst_buffer_copy_into (tensors_buf, buf, GST_BUFFER_COPY_METADATA,
245 : 0, -1);
246 : }
247 8400 : gst_buffer_unref (buf);
248 : } else {
249 6271 : empty_pad++;
250 : }
251 :
252 14671 : count++;
253 : }
254 :
255 6801 : return _gst_tensor_time_sync_is_eos (collect, sync, empty_pad);
256 : }
257 :
258 : /**
259 : * @brief A function to be called while processing a flushing event.
260 : * It should clear old buffer and reset pad data.
261 : */
262 : void
263 152 : gst_tensor_time_sync_flush (GstCollectPads * collect)
264 : {
265 : GSList *walk;
266 : GstTensorCollectPadData *pad;
267 :
268 152 : g_return_if_fail (collect != NULL);
269 :
270 152 : walk = collect->data;
271 633 : while (walk) {
272 481 : pad = (GstTensorCollectPadData *) walk->data;
273 :
274 481 : if (pad->buffer) {
275 418 : gst_buffer_unref (pad->buffer);
276 418 : pad->buffer = NULL;
277 : }
278 :
279 481 : walk = g_slist_next (walk);
280 : }
281 : }
282 :
283 : /**
284 : * @brief Internal function to update buffer in pad data based on the sync mode.
285 : */
286 : static gboolean
287 3778 : _gst_tensor_time_sync_buffer_update (GstCollectPads * collect,
288 : GstCollectData * data, GstClockTime current, GstClockTime base,
289 : tensor_time_sync_data * sync)
290 : {
291 : GstTensorCollectPadData *pad;
292 : GstBuffer *buf;
293 :
294 3778 : pad = (GstTensorCollectPadData *) data;
295 :
296 3778 : buf = gst_collect_pads_peek (collect, data);
297 3778 : if (buf != NULL) {
298 3645 : if (GST_BUFFER_PTS (buf) < current) {
299 671 : gst_buffer_unref (buf);
300 671 : if (pad->buffer != NULL)
301 669 : gst_buffer_unref (pad->buffer);
302 671 : pad->buffer = gst_collect_pads_pop (collect, data);
303 671 : return FALSE;
304 : }
305 :
306 4399 : if ((sync->mode == SYNC_SLOWEST && pad->buffer != NULL &&
307 1425 : (ABS (GST_CLOCK_DIFF (current, GST_BUFFER_PTS (pad->buffer))) <
308 1425 : ABS (GST_CLOCK_DIFF (current, GST_BUFFER_PTS (buf))))) ||
309 3943 : (sync->mode == SYNC_BASEPAD && pad->buffer != NULL &&
310 1133 : (((GstClockTime) ABS (GST_CLOCK_DIFF (current,
311 : GST_BUFFER_PTS (buf)))) > base))) {
312 : /* keep last buffer */
313 : } else {
314 : /* update last buffer */
315 2240 : if (pad->buffer != NULL)
316 1824 : gst_buffer_unref (pad->buffer);
317 2240 : pad->buffer = gst_collect_pads_pop (collect, data);
318 : }
319 :
320 2974 : gst_buffer_unref (buf);
321 : }
322 :
323 3107 : return TRUE;
324 : }
325 :
326 : /**
327 : * @brief A function call to make tensors from collected pads.
328 : * It decide which buffer is going to be used according to sync option.
329 : * @return True to push buffer.
330 : */
331 : gboolean
332 7257 : gst_tensor_time_sync_buffer_from_collectpad (GstCollectPads * collect,
333 : tensor_time_sync_data * sync, GstClockTime current_time,
334 : GstBuffer * tensors_buf, GstTensorsConfig * configs, gboolean * is_eos)
335 : {
336 7257 : GSList *walk = NULL;
337 : GstCollectData *data;
338 : GstTensorCollectPadData *pad;
339 7257 : GstBuffer *buf = NULL;
340 : GstMemory *mem;
341 7257 : gint old_numerator = G_MAXINT;
342 7257 : gint old_denominator = G_MAXINT;
343 : guint counting, empty_pad;
344 : GstTensorsConfig in_configs;
345 7257 : GstClockTime base_time = 0;
346 : GstTensorInfo *_info;
347 : guint i, j;
348 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT];
349 : tensor_format in_formats[NNS_TENSOR_SIZE_LIMIT];
350 :
351 14514 : g_return_val_if_fail (collect != NULL, FALSE);
352 7257 : g_return_val_if_fail (sync != NULL, FALSE);
353 7257 : g_return_val_if_fail (tensors_buf != NULL, FALSE);
354 7257 : g_return_val_if_fail (configs != NULL, FALSE);
355 7257 : g_return_val_if_fail (is_eos != NULL, FALSE);
356 :
357 7257 : walk = collect->data;
358 7257 : counting = empty_pad = 0;
359 :
360 7257 : if (sync->mode == SYNC_BASEPAD) {
361 711 : walk = g_slist_nth (walk, sync->data_basepad.sink_id);
362 711 : if (walk == NULL) {
363 0 : GST_ERROR_OBJECT (collect, "Cannot get GstCollectData from GSList");
364 0 : return FALSE;
365 : }
366 :
367 711 : data = (GstCollectData *) walk->data;
368 711 : pad = (GstTensorCollectPadData *) data;
369 :
370 711 : buf = gst_collect_pads_peek (collect, data);
371 711 : if (buf != NULL) {
372 649 : if (pad->buffer != NULL)
373 624 : base_time =
374 624 : MIN ((GstClockTimeDiff) sync->data_basepad.duration,
375 : ABS (GST_CLOCK_DIFF (GST_BUFFER_PTS (buf),
376 : GST_BUFFER_PTS (pad->buffer))) - 1);
377 649 : gst_buffer_unref (buf);
378 : }
379 : }
380 :
381 7257 : walk = collect->data;
382 :
383 7257 : gst_tensors_config_init (&in_configs);
384 :
385 22077 : while (walk) {
386 15492 : gboolean configured = FALSE;
387 15492 : gboolean is_empty = FALSE;
388 :
389 15492 : data = (GstCollectData *) walk->data;
390 15492 : pad = (GstTensorCollectPadData *) data;
391 :
392 15492 : if (gst_pad_has_current_caps (data->pad)) {
393 15491 : GstCaps *caps = gst_pad_get_current_caps (data->pad);
394 :
395 15491 : gst_tensors_config_free (&in_configs);
396 15491 : configured = gst_tensors_config_from_caps (&in_configs, caps, TRUE);
397 :
398 15491 : gst_caps_unref (caps);
399 : }
400 :
401 : /**
402 : * This would be an internal logic error.
403 : * in_configs should be already confirmed valid at the negotiation phase
404 : * and this function should be called in a running pipeline.
405 : * If new sync mode is enabled (e.g., handle output when a pad gets new buffer),
406 : * this may cause unexpected exception.
407 : */
408 15492 : if (!configured) {
409 1 : return FALSE;
410 : }
411 :
412 15491 : if (in_configs.rate_d < old_denominator)
413 7256 : old_denominator = in_configs.rate_d;
414 15491 : if (in_configs.rate_n < old_numerator)
415 7464 : old_numerator = in_configs.rate_n;
416 :
417 15491 : walk = g_slist_next (walk);
418 :
419 15491 : switch (sync->mode) {
420 3778 : case SYNC_SLOWEST:
421 : /* fall-through */
422 : case SYNC_BASEPAD:
423 3778 : if (!_gst_tensor_time_sync_buffer_update (collect, data,
424 : current_time, base_time, sync))
425 671 : return FALSE;
426 3107 : buf = gst_buffer_ref (pad->buffer);
427 3107 : is_empty = (buf == NULL);
428 3107 : break;
429 257 : case SYNC_NOSYNC:
430 257 : buf = gst_collect_pads_pop (collect, data);
431 257 : is_empty = (buf == NULL);
432 257 : break;
433 11456 : case SYNC_REFRESH:
434 11456 : buf = gst_collect_pads_pop (collect, data);
435 11456 : if (buf != NULL) {
436 5729 : if (pad->buffer != NULL) {
437 5727 : gst_buffer_unref (pad->buffer);
438 : }
439 5729 : pad->buffer = gst_buffer_ref (buf);
440 : } else {
441 5727 : if (pad->buffer == NULL) {
442 0 : *is_eos = FALSE;
443 0 : ml_logd ("Not the all buffers are arrived yet.");
444 0 : return FALSE;
445 : }
446 5727 : is_empty = TRUE;
447 5727 : buf = gst_buffer_ref (pad->buffer);
448 : }
449 11456 : break;
450 0 : default:
451 0 : break;
452 : }
453 :
454 14820 : if (GST_IS_BUFFER (buf)) {
455 14820 : guint32 n_tensor = gst_tensor_buffer_get_count (buf);
456 14820 : buf = gst_tensor_buffer_from_config (buf, &in_configs);
457 :
458 : /** These are internal logic error. If given inputs are incorrect,
459 : the negotiation should have been failed before this stage. */
460 14820 : if (gst_tensors_config_is_static (&in_configs))
461 14794 : g_assert (n_tensor == in_configs.info.num_tensors);
462 14820 : g_assert ((counting + n_tensor) <= NNS_TENSOR_SIZE_LIMIT);
463 :
464 14820 : if (gst_tensors_config_is_flexible (&in_configs))
465 26 : configs->info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
466 :
467 30020 : for (i = 0; i < n_tensor; ++i) {
468 15200 : in_mem[counting] = gst_tensor_buffer_get_nth_memory (buf, i);
469 :
470 : /* set info */
471 15200 : gst_tensor_info_copy (gst_tensors_info_get_nth_info (&configs->info,
472 15200 : counting), gst_tensors_info_get_nth_info (&in_configs.info, i));
473 15200 : in_formats[counting] = in_configs.info.format;
474 15200 : counting++;
475 : }
476 :
477 14820 : gst_buffer_unref (buf);
478 : }
479 14820 : if (is_empty)
480 5727 : empty_pad++;
481 : }
482 :
483 : /* append memories to output buffer */
484 20990 : for (i = 0; i < counting; i++) {
485 14405 : _info = gst_tensors_info_get_nth_info (&configs->info, i);
486 14405 : mem = in_mem[i];
487 :
488 14405 : if (gst_tensors_config_is_flexible (configs)) {
489 : /* append header if input tensor is not flexible */
490 46 : if (in_formats[i] != _NNS_TENSOR_FORMAT_FLEXIBLE) {
491 : GstTensorMetaInfo meta;
492 :
493 20 : gst_tensor_info_convert_to_meta (_info, &meta);
494 20 : mem = gst_tensor_meta_info_append_header (&meta, in_mem[i]);
495 20 : gst_memory_unref (in_mem[i]);
496 : }
497 : }
498 :
499 14405 : if (!gst_tensor_buffer_append_memory (tensors_buf, mem, _info)) {
500 0 : for (j = i + 1; j < counting; j++)
501 0 : gst_memory_unref (in_mem[j]);
502 :
503 0 : nns_loge ("Failed to append memory to buffer.");
504 0 : return FALSE;
505 : }
506 : }
507 :
508 6585 : configs->info.num_tensors = counting;
509 6585 : configs->rate_d = old_denominator;
510 6585 : configs->rate_n = old_numerator;
511 :
512 6585 : GST_BUFFER_PTS (tensors_buf) = current_time;
513 :
514 6585 : gst_tensors_config_free (&in_configs);
515 :
516 : /* check eos */
517 6585 : *is_eos = _gst_tensor_time_sync_is_eos (collect, sync, empty_pad);
518 6585 : return !(*is_eos);
519 : }
520 :
521 : /**
522 : * @brief Configure gst-buffer with tensors information.
523 : * NNStreamer handles single memory chunk as single tensor.
524 : * If incoming buffer has invalid memories, separate it and generate new gst-buffer using tensors information.
525 : * Note that this function always takes the ownership of input buffer.
526 : * @param in input buffer
527 : * @param config tensors config structure
528 : * @return Newly allocated buffer. Null if failed. Caller should unref the buffer using gst_buffer_unref().
529 : */
530 : GstBuffer *
531 37209 : gst_tensor_buffer_from_config (GstBuffer * in, GstTensorsConfig * config)
532 : {
533 37209 : GstBuffer *out = NULL;
534 37209 : GstMemory *all = NULL;
535 : GstMapInfo map;
536 : guint i, num;
537 : gsize total, offset;
538 : gsize mem_size[NNS_TENSOR_MEMORY_MAX];
539 37209 : gboolean configured = FALSE;
540 37209 : gboolean is_extra = FALSE;
541 :
542 37209 : if (!GST_IS_BUFFER (in)) {
543 1 : nns_loge ("Failed to get tensor buffer, invalid input buffer.");
544 37209 : return NULL;
545 : }
546 :
547 37208 : if (!gst_tensors_config_validate (config)) {
548 2 : nns_loge ("Failed to get tensor buffer, invalid tensor configuration.");
549 2 : goto error;
550 : }
551 :
552 37206 : num = gst_buffer_n_memory (in);
553 37206 : total = gst_buffer_get_size (in);
554 :
555 : /* get memory size */
556 37206 : if (gst_tensors_config_is_static (config)) {
557 37129 : if (num == config->info.num_tensors) {
558 : /* Do nothing, pass input buffer. */
559 37122 : out = gst_buffer_ref (in);
560 37122 : goto done;
561 : }
562 :
563 7 : num = config->info.num_tensors;
564 7 : if ((is_extra = (num > NNS_TENSOR_MEMORY_MAX)))
565 5 : num = NNS_TENSOR_MEMORY_MAX;
566 93 : for (i = 0; i < num; i++)
567 86 : mem_size[i] = gst_tensors_info_get_size (&config->info, i);
568 7 : if (is_extra) {
569 5 : mem_size[num - 1] += sizeof (GstTensorExtraInfo);
570 25 : for (; i < config->info.num_tensors; i++)
571 20 : mem_size[num - 1] += gst_tensors_info_get_size (&config->info, i);
572 : }
573 : } else {
574 77 : if (num > 1) {
575 : /* Suppose it is already configured. */
576 20 : out = gst_buffer_ref (in);
577 20 : goto done;
578 : }
579 :
580 57 : if (!gst_buffer_map (in, &map, GST_MAP_READ)) {
581 0 : nns_loge ("Failed to get tensor buffer, cannot get the memory info.");
582 0 : goto error;
583 : }
584 :
585 57 : num = 0;
586 57 : offset = 0;
587 116 : while (offset < total) {
588 : GstTensorMetaInfo meta;
589 59 : gpointer h = map.data + offset;
590 :
591 59 : if (num >= NNS_TENSOR_MEMORY_MAX - 1) {
592 : /* Suppose remained memory may include extra tensors. */
593 0 : mem_size[num++] = total - offset;
594 0 : break;
595 : }
596 :
597 59 : gst_tensor_meta_info_parse_header (&meta, h);
598 59 : mem_size[num] = gst_tensor_meta_info_get_header_size (&meta);
599 59 : mem_size[num] += gst_tensor_meta_info_get_data_size (&meta);
600 :
601 59 : offset += mem_size[num];
602 59 : num++;
603 : }
604 :
605 57 : gst_buffer_unmap (in, &map);
606 :
607 57 : if (num == 1) {
608 : /* Do nothing, pass input buffer. */
609 56 : out = gst_buffer_ref (in);
610 56 : goto done;
611 : }
612 : }
613 :
614 : /* configure output buffer */
615 8 : out = gst_buffer_new ();
616 8 : all = gst_buffer_get_all_memory (in);
617 8 : offset = 0;
618 :
619 95 : for (i = 0; i < num; i++) {
620 : /* invalid memory size */
621 88 : if (offset + mem_size[i] > total) {
622 1 : nns_loge ("Failed to get tensor buffer, data size is mismatched.");
623 1 : goto error;
624 : }
625 :
626 87 : gst_buffer_append_memory (out, gst_memory_share (all, offset, mem_size[i]));
627 87 : offset += mem_size[i];
628 : }
629 :
630 7 : gst_buffer_copy_into (out, in, GST_BUFFER_COPY_METADATA, 0, -1);
631 :
632 37205 : done:
633 37205 : configured = TRUE;
634 37208 : error:
635 37208 : gst_buffer_unref (in);
636 :
637 37208 : if (all)
638 8 : gst_memory_unref (all);
639 :
640 37208 : if (!configured) {
641 3 : if (out) {
642 1 : gst_buffer_unref (out);
643 1 : out = NULL;
644 : }
645 : }
646 :
647 37208 : return out;
648 : }
649 :
650 : /**
651 : * @brief Internal struct to handle aggregation data in hash table.
652 : */
653 : typedef struct
654 : {
655 : GstAdapter *adapter;
656 : } gst_tensor_aggregation_data_s;
657 :
658 : /**
659 : * @brief Internal function to free aggregation data.
660 : */
661 : static void
662 870 : gst_tensor_aggregation_free_data (gpointer data)
663 : {
664 : gst_tensor_aggregation_data_s *aggr;
665 :
666 870 : aggr = (gst_tensor_aggregation_data_s *) data;
667 870 : if (aggr) {
668 870 : gst_adapter_clear (aggr->adapter);
669 870 : g_object_unref (aggr->adapter);
670 :
671 870 : g_free (aggr);
672 : }
673 870 : }
674 :
675 : /**
676 : * @brief Internal function to add new aggregation data.
677 : */
678 : static gst_tensor_aggregation_data_s *
679 927 : gst_tensor_aggregation_add_data (GHashTable * table, const gint64 key)
680 : {
681 : gst_tensor_aggregation_data_s *aggr;
682 : gint64 *hashkey;
683 :
684 927 : g_return_val_if_fail (table != NULL, NULL);
685 :
686 927 : hashkey = g_new (gint64, 1);
687 927 : *hashkey = key;
688 :
689 927 : aggr = g_new0 (gst_tensor_aggregation_data_s, 1);
690 927 : aggr->adapter = gst_adapter_new ();
691 :
692 927 : g_hash_table_insert (table, hashkey, aggr);
693 927 : return aggr;
694 : }
695 :
696 : /**
697 : * @brief Internal function to get aggregation data.
698 : */
699 : static gst_tensor_aggregation_data_s *
700 304 : gst_tensor_aggregation_get_data (GHashTable * table, const gint64 key)
701 : {
702 304 : g_return_val_if_fail (table != NULL, NULL);
703 :
704 304 : return (gst_tensor_aggregation_data_s *) g_hash_table_lookup (table, &key);
705 : }
706 :
707 : /**
708 : * @brief Internal function to remove all buffers from aggregation data.
709 : */
710 : static void
711 3499 : gst_tensor_aggregation_clear_internal (gpointer key, gpointer value,
712 : gpointer user_data)
713 : {
714 : gst_tensor_aggregation_data_s *aggr;
715 :
716 : UNUSED (key);
717 : UNUSED (user_data);
718 :
719 3499 : aggr = (gst_tensor_aggregation_data_s *) value;
720 3499 : if (aggr) {
721 3499 : gst_adapter_clear (aggr->adapter);
722 : }
723 3499 : }
724 :
725 : /**
726 : * @brief Gets new hash table for tensor aggregation.
727 : * @return Newly allocated hash table, caller should release this using g_hash_table_destroy().
728 : */
729 : GHashTable *
730 924 : gst_tensor_aggregation_init (void)
731 : {
732 : GHashTable *table;
733 :
734 924 : table = g_hash_table_new_full (g_int64_hash, g_int64_equal, g_free,
735 : gst_tensor_aggregation_free_data);
736 :
737 : /**
738 : * Add default adapter (for the case if buffer has no specific id).
739 : * If gst-buffer has tensor-meta which includes client-id,
740 : * e.g., aggregation frames from multiple clients on query-server pipeline,
741 : * nnstreamer element should parse meta and request adapter with this id.
742 : * However, on normal pipeline, gst-buffer does not contain tensor-meta,
743 : * then the element may request adapter with null key string.
744 : */
745 924 : gst_tensor_aggregation_add_data (table, 0);
746 :
747 924 : return table;
748 : }
749 :
750 : /**
751 : * @brief Clears buffers from adapter.
752 : * @param table a hash table instance initialized with gst_tensor_aggregation_init()
753 : * @param key the key to look up (set 0 to get default adapter)
754 : */
755 : void
756 1 : gst_tensor_aggregation_clear (GHashTable * table, const gint64 key)
757 : {
758 : gst_tensor_aggregation_data_s *aggr;
759 :
760 1 : g_return_if_fail (table != NULL);
761 1 : g_return_if_fail (key >= 0);
762 :
763 1 : aggr = gst_tensor_aggregation_get_data (table, key);
764 1 : gst_tensor_aggregation_clear_internal (NULL, aggr, NULL);
765 : }
766 :
767 : /**
768 : * @brief Clears buffers from all adapters in hash table.
769 : * @param table a hash table instance initialized with gst_tensor_aggregation_init()
770 : */
771 : void
772 3493 : gst_tensor_aggregation_clear_all (GHashTable * table)
773 : {
774 3493 : g_hash_table_foreach (table, gst_tensor_aggregation_clear_internal, NULL);
775 3493 : }
776 :
777 : /**
778 : * @brief Gets adapter from hash table.
779 : * @param table a hash table instance initialized with gst_tensor_aggregation_init()
780 : * @param key the key to look up (set 0 to get default adapter)
781 : * @return gst-adapter instance. DO NOT release this instance.
782 : */
783 : GstAdapter *
784 304 : gst_tensor_aggregation_get_adapter (GHashTable * table, const gint64 key)
785 : {
786 : gst_tensor_aggregation_data_s *aggr;
787 :
788 304 : g_return_val_if_fail (table != NULL, NULL);
789 303 : g_return_val_if_fail (key >= 0, NULL);
790 :
791 303 : aggr = gst_tensor_aggregation_get_data (table, key);
792 303 : if (!aggr) {
793 : /*append new data */
794 3 : aggr = gst_tensor_aggregation_add_data (table, key);
795 : }
796 :
797 303 : return aggr->adapter;
798 : }
799 :
800 : /**
801 : * @brief Internal function to check tensor dimensions to append old caps for backward compatibility (rank 4).
802 : */
803 : static gboolean
804 10630 : _append_prev_caps (const GstTensorsConfig * config)
805 : {
806 : GstTensorsInfo *info;
807 : GstTensorInfo *_info;
808 : guint i, min_rank;
809 :
810 10630 : g_return_val_if_fail (config != NULL, FALSE);
811 :
812 10630 : info = (GstTensorsInfo *) (&config->info);
813 10630 : if (!gst_tensors_info_validate (info))
814 4721 : return FALSE;
815 :
816 12834 : for (i = 0; i < info->num_tensors; i++) {
817 6991 : _info = gst_tensors_info_get_nth_info (info, i);
818 :
819 6991 : min_rank = gst_tensor_dimension_get_min_rank (_info->dimension);
820 :
821 6991 : if (min_rank > NNS_TENSOR_RANK_LIMIT_PREV)
822 66 : return FALSE;
823 : }
824 :
825 5843 : return TRUE;
826 : }
827 :
828 : /**
829 : * @brief Internal function to get caps for single tensor from config.
830 : */
831 : static GstCaps *
832 5660 : _get_tensor_caps (const GstTensorsConfig * config)
833 : {
834 : GstCaps *caps, *prev1, *prev2;
835 : GstTensorsInfo *info;
836 : GstTensorInfo *_info;
837 : gboolean append_prev;
838 :
839 5660 : g_return_val_if_fail (config != NULL, NULL);
840 :
841 5660 : info = (GstTensorsInfo *) (&config->info);
842 5660 : if (info->num_tensors > 1)
843 103 : return NULL;
844 :
845 5557 : caps = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
846 5557 : _info = gst_tensors_info_get_nth_info (info, 0);
847 :
848 : /* caps for backward compatibility */
849 5557 : prev1 = prev2 = NULL;
850 5557 : append_prev = _append_prev_caps (config);
851 5557 : if (append_prev) {
852 3182 : prev1 = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
853 3182 : prev2 = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
854 : }
855 :
856 5557 : if (gst_tensor_dimension_is_valid (_info->dimension)) {
857 3353 : g_autofree gchar *dim_str =
858 3353 : gst_tensor_get_dimension_string (_info->dimension);
859 :
860 3353 : gst_caps_set_simple (caps, "dimension", G_TYPE_STRING, dim_str, NULL);
861 :
862 3353 : if (append_prev) {
863 3182 : g_autofree gchar *dim_prev1 =
864 3182 : gst_tensor_get_rank_dimension_string (_info->dimension,
865 : NNS_TENSOR_RANK_LIMIT_PREV, FALSE);
866 3182 : g_autofree gchar *dim_prev2 =
867 3182 : gst_tensor_get_rank_dimension_string (_info->dimension,
868 : NNS_TENSOR_RANK_LIMIT_PREV, TRUE);
869 :
870 3182 : gst_caps_set_simple (prev1, "dimension", G_TYPE_STRING, dim_prev1, NULL);
871 3182 : gst_caps_set_simple (prev2, "dimension", G_TYPE_STRING, dim_prev2, NULL);
872 : }
873 : }
874 :
875 5557 : if (_info->type != _NNS_END) {
876 3753 : const gchar *type_str = gst_tensor_get_type_string (_info->type);
877 :
878 3753 : gst_caps_set_simple (caps, "type", G_TYPE_STRING, type_str, NULL);
879 :
880 3753 : if (append_prev) {
881 3182 : gst_caps_set_simple (prev1, "type", G_TYPE_STRING, type_str, NULL);
882 3182 : gst_caps_set_simple (prev2, "type", G_TYPE_STRING, type_str, NULL);
883 : }
884 : }
885 :
886 5557 : if (config->rate_n >= 0 && config->rate_d > 0) {
887 2062 : gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION,
888 2062 : config->rate_n, config->rate_d, NULL);
889 :
890 2062 : if (append_prev) {
891 2000 : gst_caps_set_simple (prev1, "framerate", GST_TYPE_FRACTION,
892 2000 : config->rate_n, config->rate_d, NULL);
893 2000 : gst_caps_set_simple (prev2, "framerate", GST_TYPE_FRACTION,
894 2000 : config->rate_n, config->rate_d, NULL);
895 : }
896 : }
897 :
898 5557 : if (append_prev)
899 3182 : caps = gst_caps_merge (caps, gst_caps_merge (prev1, prev2));
900 :
901 5557 : return caps;
902 : }
903 :
904 : /**
905 : * @brief Internal function to get caps for multi tensors from config.
906 : */
907 : static GstCaps *
908 5073 : _get_tensors_caps (const GstTensorsConfig * config)
909 : {
910 : GstCaps *caps, *prev1, *prev2;
911 : gboolean append_prev;
912 :
913 5073 : g_return_val_if_fail (config != NULL, NULL);
914 :
915 5073 : caps = gst_caps_from_string (GST_TENSORS_CAP_DEFAULT);
916 :
917 : /* caps for backward compatibility */
918 5073 : prev1 = prev2 = NULL;
919 5073 : append_prev = _append_prev_caps (config);
920 5073 : if (append_prev) {
921 2661 : prev1 = gst_caps_from_string (GST_TENSORS_CAP_DEFAULT);
922 2661 : prev2 = gst_caps_from_string (GST_TENSORS_CAP_DEFAULT);
923 : }
924 :
925 5073 : if (config->info.num_tensors > 0) {
926 2847 : g_autofree gchar *type_str =
927 2847 : gst_tensors_info_get_types_string (&config->info);
928 2847 : g_autofree gchar *dim_str =
929 2847 : gst_tensors_info_get_dimensions_string (&config->info);
930 :
931 2847 : gst_caps_set_simple (caps, "num_tensors", G_TYPE_INT,
932 2847 : config->info.num_tensors, NULL);
933 2847 : gst_caps_set_simple (caps, "dimensions", G_TYPE_STRING, dim_str, NULL);
934 2847 : gst_caps_set_simple (caps, "types", G_TYPE_STRING, type_str, NULL);
935 :
936 2847 : if (append_prev) {
937 2661 : g_autofree gchar *dim_prev1 =
938 2661 : gst_tensors_info_get_rank_dimensions_string (&config->info,
939 : NNS_TENSOR_RANK_LIMIT_PREV, FALSE);
940 2661 : g_autofree gchar *dim_prev2 =
941 2661 : gst_tensors_info_get_rank_dimensions_string (&config->info,
942 : NNS_TENSOR_RANK_LIMIT_PREV, TRUE);
943 :
944 2661 : gst_caps_set_simple (prev1, "num_tensors", G_TYPE_INT,
945 2661 : config->info.num_tensors, NULL);
946 2661 : gst_caps_set_simple (prev1, "dimensions", G_TYPE_STRING, dim_prev1, NULL);
947 2661 : gst_caps_set_simple (prev1, "types", G_TYPE_STRING, type_str, NULL);
948 :
949 2661 : gst_caps_set_simple (prev2, "num_tensors", G_TYPE_INT,
950 2661 : config->info.num_tensors, NULL);
951 2661 : gst_caps_set_simple (prev2, "dimensions", G_TYPE_STRING, dim_prev2, NULL);
952 2661 : gst_caps_set_simple (prev2, "types", G_TYPE_STRING, type_str, NULL);
953 : }
954 : }
955 :
956 5073 : if (config->rate_n >= 0 && config->rate_d > 0) {
957 1492 : gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION,
958 1492 : config->rate_n, config->rate_d, NULL);
959 :
960 1492 : if (append_prev) {
961 1439 : gst_caps_set_simple (prev1, "framerate", GST_TYPE_FRACTION,
962 1439 : config->rate_n, config->rate_d, NULL);
963 1439 : gst_caps_set_simple (prev2, "framerate", GST_TYPE_FRACTION,
964 1439 : config->rate_n, config->rate_d, NULL);
965 : }
966 : }
967 :
968 5073 : if (append_prev)
969 2661 : caps = gst_caps_merge (caps, gst_caps_merge (prev1, prev2));
970 :
971 5073 : return caps;
972 : }
973 :
974 : /**
975 : * @brief Internal function to get caps for flexible tensor from config.
976 : */
977 : static GstCaps *
978 2859 : _get_flexible_caps (const GstTensorsConfig * config)
979 : {
980 : GstCaps *caps;
981 :
982 2859 : caps = gst_caps_from_string (GST_TENSORS_FLEX_CAP_DEFAULT);
983 :
984 2859 : if (config->rate_n >= 0 && config->rate_d > 0) {
985 858 : gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION,
986 858 : config->rate_n, config->rate_d, NULL);
987 : }
988 :
989 2859 : return caps;
990 : }
991 :
992 : /**
993 : * @brief Check given mimetype is tensor stream.
994 : * @param structure structure to be interpreted
995 : * @return TRUE if mimetype is tensor stream
996 : */
997 : gboolean
998 18320 : gst_structure_is_tensor_stream (const GstStructure * structure)
999 : {
1000 : const gchar *name;
1001 :
1002 18320 : name = gst_structure_get_name (structure);
1003 18320 : g_return_val_if_fail (name != NULL, FALSE);
1004 :
1005 32471 : return (g_str_equal (name, NNS_MIMETYPE_TENSOR) ||
1006 14151 : g_str_equal (name, NNS_MIMETYPE_TENSORS));
1007 : }
1008 :
1009 : /**
1010 : * @brief Get media type from structure
1011 : * @param structure structure to be interpreted
1012 : * @return corresponding media type (returns _NNS_MEDIA_INVALID for unsupported type)
1013 : */
1014 : media_type
1015 20145 : gst_structure_get_media_type (const GstStructure * structure)
1016 : {
1017 : const gchar *name;
1018 :
1019 20145 : name = gst_structure_get_name (structure);
1020 :
1021 20145 : g_return_val_if_fail (name != NULL, _NNS_MEDIA_INVALID);
1022 :
1023 20145 : if (g_str_has_prefix (name, "video/")) {
1024 2766 : return _NNS_VIDEO;
1025 : }
1026 :
1027 17379 : if (g_str_has_prefix (name, "audio/")) {
1028 2181 : return _NNS_AUDIO;
1029 : }
1030 :
1031 15198 : if (g_str_has_prefix (name, "text/")) {
1032 2154 : return _NNS_TEXT;
1033 : }
1034 :
1035 13044 : if (g_str_equal (name, "application/octet-stream")) {
1036 4406 : return _NNS_OCTET;
1037 : }
1038 :
1039 8638 : if (gst_structure_is_tensor_stream (structure)) {
1040 2149 : return _NNS_TENSOR;
1041 : }
1042 :
1043 : /* unknown or unsupported type */
1044 6489 : return _NNS_MEDIA_INVALID;
1045 : }
1046 :
1047 : /**
1048 : * @brief Parse caps from peer pad and set tensors config.
1049 : * @param pad GstPad to get the capabilities
1050 : * @param config tensors config structure to be filled
1051 : * @param is_fixed flag to be updated when peer caps is fixed (not mandatory, do nothing when the param is null)
1052 : * @return TRUE if successfully configured from peer
1053 : */
1054 : gboolean
1055 4292 : gst_tensors_config_from_peer (GstPad * pad, GstTensorsConfig * config,
1056 : gboolean * is_fixed)
1057 : {
1058 : GstCaps *peer_caps;
1059 : GstStructure *structure;
1060 4292 : gboolean ret = FALSE;
1061 :
1062 4292 : g_return_val_if_fail (GST_IS_PAD (pad), FALSE);
1063 4292 : g_return_val_if_fail (config != NULL, FALSE);
1064 :
1065 4292 : gst_tensors_config_init (config);
1066 :
1067 4292 : if ((peer_caps = gst_pad_peer_query_caps (pad, NULL))) {
1068 4292 : if (gst_caps_get_size (peer_caps) > 0) {
1069 2183 : structure = gst_caps_get_structure (peer_caps, 0);
1070 2183 : ret = gst_tensors_config_from_structure (config, structure);
1071 : }
1072 :
1073 4292 : if (ret && is_fixed)
1074 30 : *is_fixed = gst_caps_is_fixed (peer_caps);
1075 :
1076 4292 : gst_caps_unref (peer_caps);
1077 : }
1078 :
1079 4292 : return ret;
1080 : }
1081 :
1082 : /**
1083 : * @brief Check whether two structures have the same dimension
1084 : */
1085 : static gboolean
1086 248 : _is_structure_dimension_same (GstStructure * st1, GstStructure * st2,
1087 : const gchar * fieldname)
1088 : {
1089 : const char *dim_str1;
1090 : const char *dim_str2;
1091 :
1092 248 : g_return_val_if_fail (gst_structure_has_field (st1, fieldname), FALSE);
1093 248 : g_return_val_if_fail (gst_structure_has_field (st2, fieldname), FALSE);
1094 :
1095 248 : dim_str1 = gst_structure_get_string (st1, fieldname);
1096 248 : dim_str2 = gst_structure_get_string (st2, fieldname);
1097 :
1098 248 : return gst_tensor_dimension_string_is_equal (dim_str1, dim_str2);
1099 : }
1100 :
1101 : /**
1102 : * @brief Update caps dimensions for negotiation
1103 : * @param caps caps to compare and update
1104 : * @param filter caps to compare
1105 : */
1106 : void
1107 1634 : gst_tensor_caps_update_dimension (GstCaps * caps, GstCaps * filter)
1108 : {
1109 : GstStructure *st_caps, *st_filter;
1110 : guint i, j;
1111 :
1112 1634 : g_return_if_fail (GST_IS_CAPS (caps));
1113 1634 : g_return_if_fail (GST_IS_CAPS (filter));
1114 :
1115 4972 : for (i = 0; i < gst_caps_get_size (caps); i++) {
1116 3338 : st_caps = gst_caps_get_structure (caps, i);
1117 :
1118 3338 : if (!gst_structure_is_tensor_stream (st_caps))
1119 0 : continue;
1120 :
1121 9641 : for (j = 0; j < gst_caps_get_size (filter); j++) {
1122 6303 : st_filter = gst_caps_get_structure (filter, j);
1123 :
1124 6303 : if (!gst_structure_is_tensor_stream (st_filter))
1125 0 : continue;
1126 :
1127 : /* other/tensor */
1128 6303 : if (gst_structure_has_field (st_caps, "dimension")
1129 3439 : && gst_structure_has_field (st_filter, "dimension")) {
1130 : /* update dimensions for negotiation */
1131 236 : if (_is_structure_dimension_same (st_caps, st_filter, "dimension")) {
1132 118 : gst_structure_set (st_caps, "dimension", G_TYPE_STRING,
1133 : gst_structure_get_string (st_filter, "dimension"), NULL);
1134 : }
1135 : }
1136 : /* other/tensors */
1137 6185 : else if (gst_structure_has_field (st_caps, "dimensions")
1138 1866 : && gst_structure_has_field (st_filter, "dimensions")) {
1139 : /* update dimensions for negotiation */
1140 127 : if (_is_structure_dimension_same (st_caps, st_filter, "dimensions")) {
1141 118 : gst_structure_set (st_caps, "dimensions", G_TYPE_STRING,
1142 : gst_structure_get_string (st_filter, "dimensions"), NULL);
1143 : }
1144 : }
1145 : }
1146 : }
1147 : }
1148 :
1149 : /**
1150 : * @brief Try intersecting @caps1 and @caps2 for tensor stream
1151 : * @param caps1 a GstCaps to intersect
1152 : * @param caps2 a GstCaps to intersect
1153 : * @return TRUE if intersection would be not empty.
1154 : */
1155 : gboolean
1156 14 : gst_tensor_caps_can_intersect (GstCaps * caps1, GstCaps * caps2)
1157 : {
1158 : GstStructure *structure1;
1159 : GstStructure *structure2;
1160 : GstStructure *structure_copy1;
1161 : GstStructure *structure_copy2;
1162 :
1163 : const gchar *name1;
1164 : const gchar *name2;
1165 :
1166 : gboolean intersectable;
1167 :
1168 14 : if (gst_caps_can_intersect (caps1, caps2))
1169 10 : return TRUE;
1170 :
1171 4 : structure1 = gst_caps_get_structure (caps1, 0);
1172 4 : structure2 = gst_caps_get_structure (caps2, 0);
1173 :
1174 4 : if (!gst_structure_is_tensor_stream (structure1)
1175 4 : || !gst_structure_is_tensor_stream (structure2))
1176 0 : return FALSE;
1177 :
1178 4 : name1 = gst_structure_get_name (structure1);
1179 4 : name2 = gst_structure_get_name (structure2);
1180 :
1181 4 : if (!g_str_equal (name1, name2))
1182 1 : return FALSE;
1183 :
1184 : /* other/tensor */
1185 3 : if (g_str_equal (name1, NNS_MIMETYPE_TENSOR)) {
1186 3 : if (gst_structure_has_field (structure1, "dimension")
1187 3 : && gst_structure_has_field (structure2, "dimension")) {
1188 3 : if (!_is_structure_dimension_same (structure1, structure2, "dimension"))
1189 1 : return FALSE;
1190 : }
1191 : }
1192 : /* other/tensors */
1193 0 : else if (gst_structure_has_field (structure1, "dimensions")
1194 0 : && gst_structure_has_field (structure2, "dimensions")) {
1195 0 : if (!_is_structure_dimension_same (structure1, structure2, "dimensions"))
1196 0 : return FALSE;
1197 : }
1198 :
1199 2 : structure_copy1 = gst_structure_copy (structure1);
1200 2 : structure_copy2 = gst_structure_copy (structure2);
1201 :
1202 2 : gst_structure_remove_field (structure_copy1, "dimension");
1203 2 : gst_structure_remove_field (structure_copy1, "dimensions");
1204 2 : gst_structure_remove_field (structure_copy2, "dimension");
1205 2 : gst_structure_remove_field (structure_copy2, "dimensions");
1206 :
1207 : intersectable =
1208 2 : gst_structure_can_intersect (structure_copy1, structure_copy2);
1209 :
1210 2 : gst_structure_free (structure_copy1);
1211 2 : gst_structure_free (structure_copy2);
1212 :
1213 2 : return intersectable;
1214 : }
1215 :
1216 : /**
1217 : * @brief Get pad caps from tensors config and caps of the peer connected to the pad.
1218 : * @param pad GstPad to get possible caps
1219 : * @param config tensors config structure
1220 : * @return caps for given config. Caller is responsible for unreffing the returned caps.
1221 : */
1222 : GstCaps *
1223 1212 : gst_tensor_pad_caps_from_config (GstPad * pad, const GstTensorsConfig * config)
1224 : {
1225 1212 : GstCaps *caps = NULL;
1226 : GstCaps *templ;
1227 : gboolean is_flexible, peer_is_flexible, peer_has_tensor_caps;
1228 : GstCaps *peer_caps;
1229 :
1230 1212 : g_return_val_if_fail (GST_IS_PAD (pad), NULL);
1231 1212 : g_return_val_if_fail (config != NULL, NULL);
1232 :
1233 1212 : templ = gst_pad_get_pad_template_caps (pad);
1234 :
1235 : /* check peer caps */
1236 1212 : peer_is_flexible = peer_has_tensor_caps = FALSE;
1237 :
1238 1212 : peer_caps = gst_pad_peer_query_caps (pad, NULL);
1239 1212 : if (peer_caps && gst_caps_get_size (peer_caps) > 0) {
1240 : GstCaps *tmp;
1241 : GstStructure *st;
1242 : GstTensorsConfig peer_config;
1243 :
1244 916 : tmp = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
1245 916 : peer_has_tensor_caps = gst_caps_can_intersect (tmp, peer_caps);
1246 916 : gst_caps_unref (tmp);
1247 :
1248 916 : st = gst_caps_get_structure (peer_caps, 0);
1249 916 : if (gst_tensors_config_from_structure (&peer_config, st))
1250 916 : peer_is_flexible = gst_tensors_config_is_flexible (&peer_config);
1251 916 : gst_tensors_config_free (&peer_config);
1252 : }
1253 :
1254 : /* other/tensors (flexible) */
1255 1212 : is_flexible = gst_tensors_config_is_flexible (config);
1256 :
1257 1212 : if (is_flexible || peer_is_flexible) {
1258 70 : caps = _get_flexible_caps (config);
1259 70 : goto intersectable;
1260 : }
1261 :
1262 : /* other/tensor */
1263 1142 : if (config->info.num_tensors == 1 && peer_has_tensor_caps) {
1264 731 : caps = _get_tensor_caps (config);
1265 731 : if (peer_caps)
1266 731 : gst_tensor_caps_update_dimension (caps, peer_caps);
1267 :
1268 731 : if (gst_caps_can_intersect (caps, templ))
1269 727 : goto done;
1270 :
1271 4 : gst_caps_unref (caps);
1272 : }
1273 :
1274 : /* other/tensors (static) */
1275 415 : caps = _get_tensors_caps (config);
1276 415 : if (peer_caps)
1277 415 : gst_tensor_caps_update_dimension (caps, peer_caps);
1278 :
1279 0 : intersectable:
1280 485 : if (!gst_caps_can_intersect (caps, templ)) {
1281 0 : gst_caps_unref (caps);
1282 0 : caps = NULL;
1283 : }
1284 :
1285 485 : done:
1286 1212 : gst_caps_unref (templ);
1287 1212 : if (peer_caps)
1288 1212 : gst_caps_unref (peer_caps);
1289 1212 : caps = gst_caps_truncate (caps);
1290 1212 : return caps;
1291 : }
1292 :
1293 : /**
1294 : * @brief Get all possible caps from tensors config. Unlike gst_tensor_pad_caps_from_config(), this function does not check peer caps.
1295 : * @param pad GstPad to get possible caps
1296 : * @param config tensors config structure
1297 : * @return caps for given config. Caller is responsible for unreffing the returned caps.
1298 : */
1299 : GstCaps *
1300 2068 : gst_tensor_pad_possible_caps_from_config (GstPad * pad,
1301 : const GstTensorsConfig * config)
1302 : {
1303 : GstCaps *caps, *tmp;
1304 : GstCaps *templ;
1305 :
1306 2068 : g_return_val_if_fail (GST_IS_PAD (pad), NULL);
1307 2068 : g_return_val_if_fail (config != NULL, NULL);
1308 :
1309 2068 : caps = gst_caps_new_empty ();
1310 2068 : templ = gst_pad_get_pad_template_caps (pad);
1311 :
1312 : /* append caps for static tensor */
1313 2068 : if (gst_tensors_config_is_static (config)) {
1314 : /* other/tensor */
1315 1949 : if ((tmp = _get_tensor_caps (config)) != NULL) {
1316 1846 : if (gst_caps_can_intersect (tmp, templ))
1317 1798 : gst_caps_append (caps, tmp);
1318 : else
1319 48 : gst_caps_unref (tmp);
1320 : }
1321 :
1322 : /* other/tensors */
1323 1949 : if ((tmp = _get_tensors_caps (config)) != NULL) {
1324 1949 : if (gst_caps_can_intersect (tmp, templ))
1325 1949 : gst_caps_append (caps, tmp);
1326 : else
1327 0 : gst_caps_unref (tmp);
1328 : }
1329 : }
1330 :
1331 : /* caps for flexible tensor */
1332 2068 : if ((tmp = _get_flexible_caps (config)) != NULL) {
1333 2068 : if (gst_caps_can_intersect (tmp, templ))
1334 1975 : gst_caps_append (caps, tmp);
1335 : else
1336 93 : gst_caps_unref (tmp);
1337 : }
1338 :
1339 : /* if no possible caps for given config, return null. */
1340 2068 : if (gst_caps_is_empty (caps)) {
1341 0 : gst_caps_unref (caps);
1342 0 : caps = NULL;
1343 : }
1344 :
1345 2068 : gst_caps_unref (templ);
1346 2068 : return caps;
1347 : }
1348 :
1349 : /**
1350 : * @brief Get tensor format of current pad caps.
1351 : * @param pad GstPad to check current caps.
1352 : * @return The tensor_format of current pad caps.
1353 : *
1354 : * If pad does not have tensor caps return _NNS_TENSOR_FORMAT_END
1355 : */
1356 : tensor_format
1357 129808 : gst_tensor_pad_get_format (GstPad * pad)
1358 : {
1359 : GstCaps *caps;
1360 129808 : tensor_format ret = _NNS_TENSOR_FORMAT_END;
1361 :
1362 129808 : g_return_val_if_fail (GST_IS_PAD (pad), _NNS_TENSOR_FORMAT_END);
1363 :
1364 129808 : caps = gst_pad_get_current_caps (pad);
1365 129808 : if (caps) {
1366 : GstTensorsConfig config;
1367 :
1368 129795 : if (gst_tensors_config_from_caps (&config, caps, TRUE)) {
1369 129684 : ret = config.info.format;
1370 : }
1371 129795 : gst_caps_unref (caps);
1372 129795 : gst_tensors_config_free (&config);
1373 : }
1374 :
1375 129808 : return ret;
1376 : }
1377 :
1378 : /**
1379 : * @brief Get caps from tensors config (for other/tensors)
1380 : * @param config tensors config info
1381 : * @return caps for given config
1382 : */
1383 : GstCaps *
1384 3431 : gst_tensors_caps_from_config (const GstTensorsConfig * config)
1385 : {
1386 : GstCaps *caps;
1387 :
1388 3431 : g_return_val_if_fail (config != NULL, NULL);
1389 :
1390 3430 : if (gst_tensors_config_is_flexible (config)) {
1391 721 : caps = _get_flexible_caps (config);
1392 : } else {
1393 2709 : caps = _get_tensors_caps (config);
1394 : }
1395 :
1396 3430 : caps = gst_caps_truncate (caps);
1397 :
1398 3430 : return caps;
1399 : }
1400 :
1401 : /**
1402 : * @brief Get tensor caps from tensors config
1403 : * @param config tensors config info
1404 : * @return caps for given config
1405 : */
1406 : GstCaps *
1407 2981 : gst_tensor_caps_from_config (const GstTensorsConfig * config)
1408 : {
1409 : GstCaps *caps;
1410 :
1411 2981 : g_return_val_if_fail (config != NULL, NULL);
1412 :
1413 2980 : caps = _get_tensor_caps (config);
1414 2980 : caps = gst_caps_truncate (caps);
1415 :
1416 2980 : return caps;
1417 : }
1418 :
1419 : /**
1420 : * @brief Parse structure and set tensors config (for other/tensors)
1421 : * @param config tensors config structure to be filled
1422 : * @param structure structure to be interpreted
1423 : * @return TRUE if no error
1424 : */
1425 : gboolean
1426 159310 : gst_tensors_config_from_structure (GstTensorsConfig * config,
1427 : const GstStructure * structure)
1428 : {
1429 : const gchar *name;
1430 159310 : tensor_format format = _NNS_TENSOR_FORMAT_STATIC;
1431 :
1432 159310 : g_return_val_if_fail (config != NULL, FALSE);
1433 159308 : gst_tensors_config_init (config);
1434 :
1435 159308 : g_return_val_if_fail (structure != NULL, FALSE);
1436 :
1437 159307 : name = gst_structure_get_name (structure);
1438 :
1439 159307 : if (g_str_equal (name, NNS_MIMETYPE_TENSOR)) {
1440 : /* other/tensor is always static */
1441 145921 : config->info.num_tensors = 1;
1442 :
1443 145921 : if (gst_structure_has_field (structure, "dimension")) {
1444 140044 : const gchar *dim_str = gst_structure_get_string (structure, "dimension");
1445 140044 : gst_tensor_parse_dimension (dim_str, config->info.info[0].dimension);
1446 : }
1447 :
1448 145921 : if (gst_structure_has_field (structure, "type")) {
1449 140187 : const gchar *type_str = gst_structure_get_string (structure, "type");
1450 140187 : config->info.info[0].type = gst_tensor_get_type (type_str);
1451 : }
1452 13386 : } else if (g_str_equal (name, NNS_MIMETYPE_TENSORS)) {
1453 13275 : if (gst_structure_has_field (structure, "format")) {
1454 : const gchar *format_str;
1455 :
1456 13273 : format_str = gst_structure_get_string (structure, "format");
1457 13273 : format = gst_tensor_get_format (format_str);
1458 :
1459 13273 : if (format == _NNS_TENSOR_FORMAT_END) {
1460 1486 : GST_INFO
1461 : ("Invalid format %s, it should be one of %s. Suppose tensor format is static.",
1462 : _STR_NULL (format_str), GST_TENSOR_FORMAT_ALL);
1463 : } else {
1464 11787 : config->info.format = format;
1465 : }
1466 : }
1467 :
1468 13275 : if (config->info.format == _NNS_TENSOR_FORMAT_STATIC) {
1469 11328 : gst_structure_get_int (structure, "num_tensors",
1470 11328 : (gint *) (&config->info.num_tensors));
1471 :
1472 : /* parse dimensions */
1473 11328 : if (gst_structure_has_field (structure, "dimensions")) {
1474 : const gchar *dims_str;
1475 : guint num_dims;
1476 :
1477 9083 : dims_str = gst_structure_get_string (structure, "dimensions");
1478 : num_dims =
1479 9083 : gst_tensors_info_parse_dimensions_string (&config->info, dims_str);
1480 :
1481 9083 : if (config->info.num_tensors != num_dims) {
1482 8 : nns_logw ("Invalid param, dimensions (%d) tensors (%d)\n",
1483 : num_dims, config->info.num_tensors);
1484 : }
1485 : }
1486 :
1487 : /* parse types */
1488 11328 : if (gst_structure_has_field (structure, "types")) {
1489 : const gchar *types_str;
1490 : guint num_types;
1491 :
1492 9035 : types_str = gst_structure_get_string (structure, "types");
1493 : num_types =
1494 9035 : gst_tensors_info_parse_types_string (&config->info, types_str);
1495 :
1496 9035 : if (config->info.num_tensors != num_types) {
1497 0 : nns_logw ("Invalid param, types (%d) tensors (%d)\n",
1498 : num_types, config->info.num_tensors);
1499 : }
1500 : }
1501 : }
1502 : } else {
1503 111 : nns_logw ("Unsupported type = %s\n", name ? name : "Unknown");
1504 111 : return FALSE;
1505 : }
1506 :
1507 159196 : if (gst_structure_has_field (structure, "framerate")) {
1508 159177 : gst_structure_get_fraction (structure, "framerate", &config->rate_n,
1509 159177 : &config->rate_d);
1510 : }
1511 :
1512 159196 : return TRUE;
1513 : }
1514 :
1515 : /**
1516 : * @brief Parse caps and set tensors config (for other/tensors)
1517 : * @param[out] config tensors config structure to be filled
1518 : * @param[in] caps incoming capability
1519 : * @param[in] validate TRUE to validate configuration
1520 : * @return TRUE/FALSE (if successfully configured, return TRUE)
1521 : */
1522 : gboolean
1523 150334 : gst_tensors_config_from_caps (GstTensorsConfig * config, const GstCaps * caps,
1524 : const gboolean validate)
1525 : {
1526 : GstStructure *structure;
1527 150334 : gboolean ret = FALSE;
1528 :
1529 150334 : gst_tensors_config_init (config);
1530 :
1531 150334 : if (validate && !gst_caps_is_fixed (caps)) {
1532 529 : nns_logw ("GstCaps is not fixed.");
1533 529 : return FALSE;
1534 : }
1535 :
1536 149805 : structure = gst_caps_get_structure (caps, 0);
1537 149805 : ret = gst_tensors_config_from_structure (config, structure);
1538 :
1539 149805 : if (ret && validate) {
1540 146963 : ret = gst_tensors_config_validate (config);
1541 : }
1542 :
1543 149805 : if (!ret) {
1544 114 : gst_tensors_config_free (config);
1545 : }
1546 :
1547 149805 : return ret;
1548 : }
1549 :
1550 : /**
1551 : * @brief Parse memory and fill the tensor meta.
1552 : * @param[out] meta tensor meta structure to be filled
1553 : * @param[in] mem pointer to GstMemory to be parsed
1554 : * @return TRUE if successfully set the meta
1555 : */
1556 : gboolean
1557 61309 : gst_tensor_meta_info_parse_memory (GstTensorMetaInfo * meta, GstMemory * mem)
1558 : {
1559 : GstMapInfo map;
1560 : gsize hsize, msize;
1561 : gboolean ret;
1562 :
1563 122617 : g_return_val_if_fail (mem != NULL, FALSE);
1564 61308 : g_return_val_if_fail (meta != NULL, FALSE);
1565 :
1566 61307 : gst_tensor_meta_info_init (meta);
1567 :
1568 : /* Check header size of tensor-meta. */
1569 61307 : hsize = gst_tensor_meta_info_get_header_size (meta);
1570 61307 : msize = gst_memory_get_sizes (mem, NULL, NULL);
1571 61307 : if (msize < hsize)
1572 8521 : return FALSE;
1573 :
1574 52786 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
1575 0 : nns_loge ("Failed to get the meta, cannot map the memory.");
1576 0 : return FALSE;
1577 : }
1578 :
1579 52786 : ret = gst_tensor_meta_info_parse_header (meta, map.data);
1580 :
1581 52786 : gst_memory_unmap (mem, &map);
1582 52786 : return ret;
1583 : }
1584 :
1585 : /**
1586 : * @brief Append header to memory.
1587 : * @param[in] meta tensor meta structure
1588 : * @param[in] mem pointer to GstMemory
1589 : * @return Newly allocated GstMemory (Caller should free returned memory using gst_memory_unref())
1590 : */
1591 : GstMemory *
1592 282 : gst_tensor_meta_info_append_header (GstTensorMetaInfo * meta, GstMemory * mem)
1593 : {
1594 282 : GstMemory *new_mem = NULL;
1595 : gsize msize, hsize;
1596 : GstMapInfo old_map, new_map;
1597 :
1598 563 : g_return_val_if_fail (mem != NULL, NULL);
1599 281 : g_return_val_if_fail (gst_tensor_meta_info_validate (meta), NULL);
1600 :
1601 280 : if (!gst_memory_map (mem, &old_map, GST_MAP_READ)) {
1602 0 : nns_loge ("Failed to append header, cannot map the old memory.");
1603 0 : return NULL;
1604 : }
1605 :
1606 : /* memory size (header + old memory) */
1607 280 : hsize = gst_tensor_meta_info_get_header_size (meta);
1608 280 : msize = hsize + old_map.size;
1609 :
1610 280 : new_mem = gst_allocator_alloc (NULL, msize, NULL);
1611 280 : if (!gst_memory_map (new_mem, &new_map, GST_MAP_WRITE)) {
1612 0 : nns_loge ("Failed to append header, cannot map the new memory.");
1613 0 : gst_memory_unmap (mem, &old_map);
1614 0 : gst_memory_unref (new_mem);
1615 0 : return NULL;
1616 : }
1617 :
1618 : /* set header and copy old data */
1619 280 : gst_tensor_meta_info_update_header (meta, new_map.data);
1620 280 : memcpy (new_map.data + hsize, old_map.data, old_map.size);
1621 :
1622 280 : gst_memory_unmap (mem, &old_map);
1623 280 : gst_memory_unmap (new_mem, &new_map);
1624 280 : return new_mem;
1625 : }
1626 :
1627 : /**
1628 : * @brief Get the nth GstMemory from given @a buffer.
1629 : * @param[in] buffer GstBuffer to be parsed.
1630 : * @param[in] index Index of GstMemory to be returned.
1631 : * @return GstMemory if found, otherwise NULL (Caller should free returned memory using gst_memory_unref()).
1632 : */
1633 : GstMemory *
1634 63514 : gst_tensor_buffer_get_nth_memory (GstBuffer * buffer, const guint index)
1635 : {
1636 : guint i, num_tensors;
1637 : gsize offset;
1638 63514 : GstMemory *extra_tensors_memory, *res_mem = NULL;
1639 : GstMapInfo extra_tensors_map;
1640 : GstTensorExtraInfo *extra_info;
1641 :
1642 63514 : if (!GST_IS_BUFFER (buffer)) {
1643 0 : nns_loge ("Failed to parse GstBuffer (invalid input buffer).");
1644 63514 : return NULL;
1645 : }
1646 :
1647 63514 : num_tensors = gst_tensor_buffer_get_count (buffer);
1648 63514 : if (index >= num_tensors) {
1649 0 : nns_loge ("Invalid index %u, the number of tensors in the buffer is %u.",
1650 : index, num_tensors);
1651 0 : return NULL;
1652 : }
1653 :
1654 : /* If num_tensors is less than or equal to NNS_TENSOR_MEMORY_MAX, it's trivial. */
1655 63514 : if (num_tensors <= NNS_TENSOR_MEMORY_MAX || index < NNS_TENSOR_MEMORY_MAX - 1) {
1656 62893 : return gst_buffer_get_memory (buffer, index);
1657 : }
1658 :
1659 : /* If num_tensors is greater than NNS_TENSOR_MEMORY_MAX, we need to parse extra info. */
1660 : extra_tensors_memory =
1661 621 : gst_buffer_peek_memory (buffer, NNS_TENSOR_MEMORY_MAX - 1);
1662 621 : if (!extra_tensors_memory) {
1663 0 : nns_loge ("Failed to get %d-th memory", NNS_TENSOR_MEMORY_MAX);
1664 0 : return NULL;
1665 : }
1666 :
1667 621 : if (!gst_memory_map (extra_tensors_memory, &extra_tensors_map, GST_MAP_READ)) {
1668 0 : nns_loge ("Failed to map %d-th memory", NNS_TENSOR_MEMORY_MAX);
1669 0 : return NULL;
1670 : }
1671 :
1672 : /* check header (extra info) of the memory */
1673 621 : if (!gst_memory_map_is_extra_tensor (&extra_tensors_map)) {
1674 0 : nns_loge ("Invalid extra header");
1675 0 : goto done;
1676 : }
1677 :
1678 : /* parse the memory */
1679 621 : extra_info = (GstTensorExtraInfo *) extra_tensors_map.data;
1680 621 : offset = sizeof (GstTensorExtraInfo);
1681 :
1682 : /* If index is NNS_TENSOR_MEMORY_MAX - 1 */
1683 621 : if (index == NNS_TENSOR_MEMORY_MAX - 1) {
1684 : res_mem =
1685 29 : gst_memory_share (extra_tensors_memory, offset, extra_info->reserved);
1686 29 : goto done;
1687 : }
1688 :
1689 592 : offset += extra_info->reserved;
1690 :
1691 31720 : for (i = 1; i <= index - NNS_TENSOR_MEMORY_MAX; ++i) {
1692 31128 : offset += gst_tensor_info_get_size (&extra_info->infos[i - 1]);
1693 : }
1694 :
1695 : /* wrap it as GstMemory */
1696 : res_mem =
1697 592 : gst_memory_share (extra_tensors_memory, offset,
1698 592 : gst_tensor_info_get_size (&extra_info->infos[index -
1699 : NNS_TENSOR_MEMORY_MAX]));
1700 :
1701 621 : done:
1702 621 : gst_memory_unmap (extra_tensors_memory, &extra_tensors_map);
1703 621 : return res_mem;
1704 : }
1705 :
1706 : /**
1707 : * @brief Append @a memory to given @a buffer.
1708 : * @param[in/out] buffer GstBuffer to be appended.
1709 : * @param[in] memory GstMemory to append. This function takes ownership of this, even if it returns failure.
1710 : * @param[in] info GstTensorInfo of given @a memory.
1711 : * @return TRUE if successfully appended, otherwise FALSE.
1712 : */
1713 : gboolean
1714 61284 : gst_tensor_buffer_append_memory (GstBuffer * buffer, GstMemory * memory,
1715 : const GstTensorInfo * info)
1716 : {
1717 : guint num_mems, new_mem_index;
1718 61284 : GstMemory *new_memory = NULL, *last_memory = NULL;
1719 : gsize offset, new_mem_size, last_mem_size;
1720 : GstMapInfo new_memory_map, last_memory_map, incoming_memory_map;
1721 : GstTensorExtraInfo *extra_info;
1722 : GstTensorMetaInfo meta;
1723 : gboolean is_extra, is_static;
1724 61284 : gboolean appended = FALSE;
1725 :
1726 61284 : if (!GST_IS_BUFFER (buffer)) {
1727 0 : nns_loge ("Failed to append memory, given buffer is invalid.");
1728 0 : goto failed;
1729 : }
1730 :
1731 61284 : if (!memory) {
1732 0 : nns_loge ("Failed to append memory, given memory is NULL.");
1733 0 : goto failed;
1734 : }
1735 :
1736 61284 : if (gst_tensor_meta_info_parse_memory (&meta, memory)) {
1737 558 : is_static = (meta.format == _NNS_TENSOR_FORMAT_STATIC);
1738 : } else {
1739 : /* Suppose given memory is static tensor. */
1740 60726 : is_static = TRUE;
1741 :
1742 : /* Error case if given tensor-info is invalid. */
1743 60726 : if (!gst_tensor_info_validate (info)) {
1744 0 : nns_loge ("Failed to get tensor info (invalid input info).");
1745 0 : goto failed;
1746 : }
1747 : }
1748 :
1749 61284 : num_mems = gst_buffer_n_memory (buffer);
1750 :
1751 : /* trivial call to gst_buffer_append_memory */
1752 61284 : if (num_mems < NNS_TENSOR_MEMORY_MAX) {
1753 60696 : gst_buffer_append_memory (buffer, memory);
1754 61284 : return TRUE;
1755 : }
1756 :
1757 : /* given buffer has NNS_TENSOR_MEMORY_MAX memory blocks */
1758 588 : last_memory = gst_buffer_peek_memory (buffer, num_mems - 1);
1759 588 : if (!last_memory) {
1760 0 : nns_loge ("Failed to get last memory");
1761 0 : goto failed;
1762 : }
1763 :
1764 588 : if (!gst_memory_map (last_memory, &last_memory_map, GST_MAP_READ)) {
1765 0 : nns_loge ("Failed to map last memory");
1766 0 : last_memory = NULL;
1767 0 : goto failed;
1768 : }
1769 :
1770 588 : new_mem_size = last_mem_size = gst_memory_get_sizes (last_memory, NULL, NULL);
1771 :
1772 : /* if the memory does not have proper header, append it */
1773 588 : is_extra = gst_memory_map_is_extra_tensor (&last_memory_map);
1774 588 : if (!is_extra) {
1775 28 : new_mem_size += sizeof (GstTensorExtraInfo);
1776 : }
1777 :
1778 588 : new_mem_size += gst_memory_get_sizes (memory, NULL, NULL);
1779 :
1780 588 : new_memory = gst_allocator_alloc (NULL, new_mem_size, NULL);
1781 588 : if (!new_memory) {
1782 0 : nns_loge ("Failed to allocate memory for extra tensors.");
1783 0 : goto failed;
1784 : }
1785 :
1786 588 : if (!gst_memory_map (new_memory, &new_memory_map, GST_MAP_WRITE)) {
1787 0 : nns_loge ("Failed to map extra memory");
1788 0 : gst_memory_unref (new_memory);
1789 0 : new_memory = NULL;
1790 0 : goto failed;
1791 : }
1792 :
1793 588 : if (!gst_memory_map (memory, &incoming_memory_map, GST_MAP_READ)) {
1794 0 : nns_loge ("Failed to map incoming memory");
1795 0 : goto failed;
1796 : }
1797 :
1798 588 : extra_info = (GstTensorExtraInfo *) new_memory_map.data;
1799 :
1800 : /* if the last_memory does not have proper header, append it */
1801 588 : if (!is_extra) {
1802 28 : gst_tensor_extra_info_init (extra_info, last_mem_size);
1803 28 : offset = sizeof (GstTensorExtraInfo);
1804 : } else {
1805 560 : offset = 0;
1806 : }
1807 :
1808 : /* copy last_memory into new_memory */
1809 588 : memcpy (new_memory_map.data + offset, last_memory_map.data,
1810 : last_memory_map.size);
1811 :
1812 : /* copy incoming_memory into new_memory */
1813 588 : new_mem_index = extra_info->num_extra_tensors;
1814 588 : extra_info->num_extra_tensors += 1;
1815 :
1816 : /* Copy tensor info into extra. */
1817 588 : if (is_static) {
1818 588 : gst_tensor_info_copy (&extra_info->infos[new_mem_index], info);
1819 :
1820 : /**
1821 : * Free the name string, cause it does not freed by gstreamer.
1822 : * @todo Make custom gst_allocator later?
1823 : */
1824 588 : g_clear_pointer (&extra_info->infos[new_mem_index].name, g_free);
1825 : } else {
1826 0 : gst_tensor_meta_info_convert (&meta, &extra_info->infos[new_mem_index]);
1827 : }
1828 :
1829 588 : memcpy (new_memory_map.data + offset + last_memory_map.size,
1830 588 : incoming_memory_map.data, incoming_memory_map.size);
1831 :
1832 588 : gst_memory_unmap (memory, &incoming_memory_map);
1833 588 : gst_memory_unmap (last_memory, &last_memory_map);
1834 588 : last_memory = NULL;
1835 :
1836 588 : gst_buffer_replace_memory (buffer, num_mems - 1, new_memory);
1837 588 : appended = TRUE;
1838 :
1839 588 : failed:
1840 588 : if (new_memory) {
1841 588 : gst_memory_unmap (new_memory, &new_memory_map);
1842 588 : if (!appended)
1843 0 : gst_memory_unref (new_memory);
1844 : }
1845 :
1846 588 : if (last_memory)
1847 0 : gst_memory_unmap (last_memory, &last_memory_map);
1848 :
1849 : /* Release incoming memory even if failed to append it into buffer. */
1850 588 : if (memory)
1851 588 : gst_memory_unref (memory);
1852 :
1853 588 : return appended;
1854 : }
1855 :
1856 : /**
1857 : * @brief Get the number of tensors in the buffer.
1858 : */
1859 : guint
1860 124312 : gst_tensor_buffer_get_count (GstBuffer * buffer)
1861 : {
1862 : guint num_mems;
1863 : GstMemory *mem;
1864 : GstMapInfo map;
1865 : GstTensorExtraInfo *extra_info;
1866 :
1867 248624 : g_return_val_if_fail (buffer != NULL, 0);
1868 :
1869 124312 : num_mems = gst_buffer_n_memory (buffer);
1870 124312 : if (num_mems < NNS_TENSOR_MEMORY_MAX) {
1871 122611 : return num_mems;
1872 : }
1873 :
1874 : /* num_mems == NNS_TENSOR_MEMORY_MAX */
1875 1701 : mem = gst_buffer_peek_memory (buffer, num_mems - 1);
1876 1701 : if (!mem) {
1877 0 : nns_loge ("Failed to get the last memory.");
1878 0 : return 0;
1879 : }
1880 :
1881 1701 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
1882 0 : nns_loge ("Failed to map the last memory.");
1883 0 : return 0;
1884 : }
1885 :
1886 1701 : if (gst_memory_map_is_extra_tensor (&map)) {
1887 1349 : extra_info = (GstTensorExtraInfo *) map.data;
1888 1349 : num_mems = extra_info->num_extra_tensors + NNS_TENSOR_MEMORY_MAX;
1889 : } else {
1890 352 : nns_logi ("The last memory does not have extra tensors header. "
1891 : "Assuming the number of tensors is %d.", num_mems);
1892 : }
1893 :
1894 1701 : gst_memory_unmap (mem, &map);
1895 :
1896 1701 : return num_mems;
1897 : }
1898 :
1899 : /**
1900 : * @brief Sets the value of a property based on the specified property value and GParamSpec.
1901 : *
1902 : * @param prop_value A pointer to the GValue where the property value will be set.
1903 : * @param param_spec A pointer to the GParamSpec that describes the property.
1904 : * @param property_value A string representing the value to be set for the property.
1905 : *
1906 : * @note This API is intended to be used by gst_tensor_parse_config_file ()
1907 : */
1908 : static void
1909 34 : set_property_value (GValue * prop_value, const GParamSpec * param_spec,
1910 : const gchar * property_value)
1911 : {
1912 34 : GType value_type = G_PARAM_SPEC_VALUE_TYPE (param_spec);
1913 :
1914 34 : g_value_init (prop_value, value_type);
1915 :
1916 34 : if (value_type == G_TYPE_BOOLEAN) {
1917 0 : gboolean value = g_ascii_strcasecmp (property_value, "true") == 0;
1918 0 : g_value_set_boolean (prop_value, value);
1919 34 : } else if (value_type == G_TYPE_INT) {
1920 0 : gint value = atoi (property_value);
1921 0 : g_value_set_int (prop_value, value);
1922 34 : } else if (value_type == G_TYPE_UINT) {
1923 0 : guint value = atoi (property_value);
1924 0 : g_value_set_uint (prop_value, value);
1925 34 : } else if (value_type == G_TYPE_FLOAT) {
1926 0 : gfloat value = atof (property_value);
1927 0 : g_value_set_float (prop_value, value);
1928 34 : } else if (value_type == G_TYPE_DOUBLE) {
1929 0 : gdouble value = atof (property_value);
1930 0 : g_value_set_double (prop_value, value);
1931 : } else {
1932 34 : g_value_set_string (prop_value, property_value); /** default is string */
1933 : }
1934 34 : }
1935 :
1936 : /**
1937 : * @brief Parses a configuration file and sets the corresponding properties on a GObject.
1938 : *
1939 : * This function reads the contents of the configuration file located at the given path
1940 : * and sets the properties of the specified GObject based on the configuration data.
1941 : *
1942 : * @param config_path The path to the configuration file.
1943 : * @param object The GObject on which to set the properties.
1944 : *
1945 : * @note The responsibility of managing the memory of the GObject passed as a parameter
1946 : * lies outside this function.
1947 : */
1948 : void
1949 9 : gst_tensor_parse_config_file (const gchar * config_path, const GObject * object)
1950 : {
1951 9 : g_autofree gchar *config_data = NULL;
1952 9 : g_auto (GStrv) lines = NULL;
1953 9 : GStrv line = NULL;
1954 9 : GError *error = NULL;
1955 9 : GObjectClass *g_object_class = G_OBJECT_GET_CLASS (object);
1956 :
1957 9 : if (!g_file_get_contents (config_path, &config_data, NULL, &error)) {
1958 0 : GST_DEBUG ("Failed to read config file: %s\n", error->message);
1959 0 : g_error_free (error);
1960 0 : return;
1961 : }
1962 :
1963 9 : lines = g_strsplit (config_data, "\n", -1);
1964 :
1965 : /** Iterate over each line */
1966 52 : for (line = lines; *line; ++line) {
1967 43 : g_auto (GStrv) parts = g_strsplit (*line, "=", 2);
1968 :
1969 43 : if (g_strv_length (parts) == 2) {
1970 68 : g_autofree gchar *property_name = g_strstrip (g_strdup (parts[0]));
1971 68 : g_autofree gchar *property_value = g_strstrip (g_strdup (parts[1]));
1972 :
1973 : GParamSpec *pdata =
1974 34 : g_object_class_find_property (g_object_class, property_name);
1975 :
1976 34 : if (pdata != NULL) {
1977 34 : GValue prop_value = G_VALUE_INIT;
1978 34 : set_property_value (&prop_value, pdata, property_value);
1979 34 : g_object_set_property (G_OBJECT (object), pdata->name, &prop_value);
1980 34 : g_value_unset (&prop_value);
1981 : }
1982 : }
1983 : }
1984 : }
|
