Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * NNStreamer Common Header's Contents (pipeline extension)
4 : * Copyright (C) 2020 MyungJoo Ham <myungjoo.ham@samsung.com>
5 : */
6 : /**
7 : * @file nnstreamer_plugin_api_impl.c
8 : * @date 14 Apr 2020
9 : * @brief Common data for NNStreamer, the GStreamer plugin for neural networks
10 : * @see https://github.com/nnstreamer/nnstreamer
11 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
12 : * @bug No known bugs except for NYI items
13 : *
14 : */
15 :
16 : #include <nnstreamer_util.h>
17 : #include <string.h>
18 : #include <tensor_common.h>
19 :
20 : static const gchar *gst_tensor_time_sync_mode_string[] = {
21 : [SYNC_NOSYNC] = "nosync",
22 : [SYNC_SLOWEST] = "slowest",
23 : [SYNC_BASEPAD] = "basepad",
24 : [SYNC_REFRESH] = "refresh",
25 : [SYNC_END] = NULL
26 : };
27 :
28 : /**
29 : * @brief The old rank of tensor.
30 : */
31 : #define NNS_TENSOR_RANK_LIMIT_PREV (4)
32 :
33 : #define NNS_TENSOR_EXTRA_MAGIC 0xf00dc0de
34 :
35 : /**
36 : * @brief Data structure to describe a "extra" tensor data.
37 : * This represents the information of the NNS_TENSOR_SIZE_LIMIT-th memory block for tensor stream.
38 : */
39 : typedef struct
40 : {
41 : uint32_t magic;
42 : uint32_t version;
43 : uint32_t num_extra_tensors;
44 : uint64_t reserved;
45 : GstTensorInfo infos[NNS_TENSOR_SIZE_EXTRA_LIMIT];
46 : } GstTensorExtraInfo;
47 :
48 : /**
49 : * @brief Check if given memory has extra tensors.
50 : * @param[in] map GstMapInfo of GstMemory to be checked.
51 : * @return TRUE if @map has extra tensors, otherwise FALSE.
52 : */
53 : static gboolean
54 2910 : gst_memory_map_is_extra_tensor (GstMapInfo * map)
55 : {
56 : GstTensorExtraInfo *extra_info;
57 : gboolean is_extra;
58 :
59 2910 : g_return_val_if_fail (map != NULL, FALSE);
60 :
61 2910 : if (map->size < sizeof (GstTensorExtraInfo))
62 380 : return FALSE;
63 :
64 2530 : extra_info = (GstTensorExtraInfo *) map->data;
65 :
66 : /* check magic in header (extra info) of the memory */
67 2530 : is_extra = (extra_info && extra_info->magic == NNS_TENSOR_EXTRA_MAGIC);
68 :
69 2530 : return is_extra;
70 : }
71 :
72 : /**
73 : * @brief Initialize GstTensorExtraInfo structure with given @a memory.
74 : * @param[in/out] extra GstTensorExtraInfo to be initialized.
75 : * @param[in] reserved_size The memory size of extra memory block.
76 : */
77 : static void
78 28 : gst_tensor_extra_info_init (GstTensorExtraInfo * extra, gsize reserved_size)
79 : {
80 : guint i;
81 :
82 28 : g_return_if_fail (extra != NULL);
83 :
84 28 : extra->magic = NNS_TENSOR_EXTRA_MAGIC;
85 28 : extra->version = 0;
86 28 : extra->num_extra_tensors = 0;
87 :
88 : /* set reserved size of NNS_TENSOR_SIZE_LIMIT-th memory */
89 28 : extra->reserved = reserved_size;
90 6748 : for (i = 0; i < NNS_TENSOR_SIZE_EXTRA_LIMIT; ++i) {
91 6720 : gst_tensor_info_init (&extra->infos[i]);
92 : }
93 : }
94 :
95 : /**
96 : * @brief Get the corresponding mode from the string value.
97 : * @param[in] str The string value for the mode.
98 : * @return Corresponding mode for the string. SYNC_END for errors.
99 : */
100 : tensor_time_sync_mode
101 66 : gst_tensor_time_sync_get_mode (const gchar * str)
102 : {
103 : gint index;
104 :
105 66 : index = find_key_strv (gst_tensor_time_sync_mode_string, str);
106 :
107 66 : return (index < 0) ? SYNC_END : index;
108 : }
109 :
110 : /**
111 : * @brief Get the time-sync mode string.
112 : * @return Corresponding mode string.
113 : */
114 : const gchar *
115 2 : gst_tensor_time_sync_get_mode_string (tensor_time_sync_mode mode)
116 : {
117 2 : return gst_tensor_time_sync_mode_string[mode];
118 : }
119 :
120 : /**
121 : * @brief Setup time sync option.
122 : * @param[in/out] filter "this" pointer. Sync mode & option MUST BE set already.
123 : * @return True if successfully set the option.
124 : */
125 : gboolean
126 95 : gst_tensor_time_sync_set_option_data (tensor_time_sync_data * sync)
127 : {
128 95 : g_return_val_if_fail (sync != NULL, FALSE);
129 :
130 95 : if (sync->mode == SYNC_END || sync->option == NULL)
131 66 : return FALSE;
132 :
133 29 : switch (sync->mode) {
134 0 : case SYNC_NOSYNC:
135 0 : break;
136 0 : case SYNC_SLOWEST:
137 0 : break;
138 29 : case SYNC_BASEPAD:
139 : {
140 29 : g_auto (GStrv) strv = g_strsplit (sync->option, ":", 2);
141 : guint sink_id;
142 : guint duration;
143 :
144 29 : if (strv[0] != NULL)
145 29 : sink_id = (guint) g_ascii_strtoull (strv[0], NULL, 10);
146 : else
147 0 : sink_id = 0;
148 :
149 29 : if (strv[1] != NULL)
150 27 : duration = (guint) g_ascii_strtoull (strv[1], NULL, 10);
151 : else
152 2 : duration = G_MAXINT;
153 :
154 29 : sync->data_basepad.sink_id = sink_id;
155 29 : sync->data_basepad.duration = duration;
156 29 : break;
157 : }
158 0 : default:
159 : /* unknown mode */
160 0 : GST_WARNING ("Unknown mode = %d", sync->mode);
161 0 : return FALSE;
162 : }
163 :
164 29 : return TRUE;
165 : }
166 :
167 : /**
168 : * @brief Internal function to detect EOS using the number of empty pads.
169 : * @param[in] collect Collect pad.
170 : * @param[in] sync Synchronization option.
171 : * @param[in] empty The number of empty pads (pad has no buffer).
172 : * @return True if EOS.
173 : */
174 : static gboolean
175 13297 : _gst_tensor_time_sync_is_eos (GstCollectPads * collect,
176 : tensor_time_sync_data * sync, guint empty)
177 : {
178 : guint total;
179 13297 : gboolean is_eos = FALSE;
180 :
181 13297 : total = g_slist_length (collect->data);
182 :
183 13297 : switch (sync->mode) {
184 11374 : case SYNC_REFRESH:
185 11374 : if (empty == total)
186 2 : is_eos = TRUE;
187 11374 : break;
188 1923 : default:
189 1923 : if (empty > 0)
190 209 : is_eos = TRUE;
191 1923 : break;
192 : }
193 :
194 13297 : return is_eos;
195 : }
196 :
197 : /**
198 : * @brief A function call to decide current timestamp among collected pads based on PTS.
199 : * It will decide current timestamp according to sync option.
200 : * GstMeta is also copied with same sync mode.
201 : */
202 : gboolean
203 6754 : gst_tensor_time_sync_get_current_time (GstCollectPads * collect,
204 : tensor_time_sync_data * sync, GstClockTime * current_time,
205 : GstBuffer * tensors_buf)
206 : {
207 6754 : GSList *walk = NULL;
208 : guint count, empty_pad;
209 :
210 6754 : g_return_val_if_fail (collect != NULL, FALSE);
211 6754 : g_return_val_if_fail (sync != NULL, FALSE);
212 6754 : g_return_val_if_fail (current_time != NULL, FALSE);
213 :
214 6754 : walk = collect->data;
215 6754 : count = empty_pad = 0;
216 :
217 21326 : while (walk) {
218 : GstCollectData *data;
219 : GstBuffer *buf;
220 14572 : gboolean need_update = FALSE;
221 :
222 14572 : data = (GstCollectData *) walk->data;
223 14572 : buf = gst_collect_pads_peek (collect, data);
224 14572 : walk = g_slist_next (walk);
225 :
226 14572 : if (buf) {
227 8354 : switch (sync->mode) {
228 7697 : case SYNC_NOSYNC:
229 : /* fall-through */
230 : case SYNC_SLOWEST:
231 : case SYNC_REFRESH:
232 7697 : if (*current_time < GST_BUFFER_PTS (buf))
233 6244 : need_update = TRUE;
234 7697 : break;
235 657 : case SYNC_BASEPAD:
236 657 : if (count == sync->data_basepad.sink_id)
237 296 : need_update = TRUE;
238 657 : break;
239 0 : default:
240 0 : break;
241 : }
242 8354 : if (need_update) {
243 6540 : *current_time = GST_BUFFER_PTS (buf);
244 6540 : gst_buffer_copy_into (tensors_buf, buf, GST_BUFFER_COPY_METADATA,
245 : 0, -1);
246 : }
247 8354 : gst_buffer_unref (buf);
248 : } else {
249 6218 : empty_pad++;
250 : }
251 :
252 14572 : count++;
253 : }
254 :
255 6754 : return _gst_tensor_time_sync_is_eos (collect, sync, empty_pad);
256 : }
257 :
258 : /**
259 : * @brief A function to be called while processing a flushing event.
260 : * It should clear old buffer and reset pad data.
261 : */
262 : void
263 152 : gst_tensor_time_sync_flush (GstCollectPads * collect)
264 : {
265 : GSList *walk;
266 : GstTensorCollectPadData *pad;
267 :
268 152 : g_return_if_fail (collect != NULL);
269 :
270 152 : walk = collect->data;
271 633 : while (walk) {
272 481 : pad = (GstTensorCollectPadData *) walk->data;
273 :
274 481 : if (pad->buffer) {
275 418 : gst_buffer_unref (pad->buffer);
276 418 : pad->buffer = NULL;
277 : }
278 :
279 481 : walk = g_slist_next (walk);
280 : }
281 : }
282 :
283 : /**
284 : * @brief Internal function to update buffer in pad data based on the sync mode.
285 : */
286 : static gboolean
287 3777 : _gst_tensor_time_sync_buffer_update (GstCollectPads * collect,
288 : GstCollectData * data, GstClockTime current, GstClockTime base,
289 : tensor_time_sync_data * sync)
290 : {
291 : GstTensorCollectPadData *pad;
292 : GstBuffer *buf;
293 :
294 3777 : pad = (GstTensorCollectPadData *) data;
295 :
296 3777 : buf = gst_collect_pads_peek (collect, data);
297 3777 : if (buf != NULL) {
298 3645 : if (GST_BUFFER_PTS (buf) < current) {
299 671 : gst_buffer_unref (buf);
300 671 : if (pad->buffer != NULL)
301 669 : gst_buffer_unref (pad->buffer);
302 671 : pad->buffer = gst_collect_pads_pop (collect, data);
303 671 : return FALSE;
304 : }
305 :
306 4399 : if ((sync->mode == SYNC_SLOWEST && pad->buffer != NULL &&
307 1425 : (ABS (GST_CLOCK_DIFF (current, GST_BUFFER_PTS (pad->buffer))) <
308 1425 : ABS (GST_CLOCK_DIFF (current, GST_BUFFER_PTS (buf))))) ||
309 3943 : (sync->mode == SYNC_BASEPAD && pad->buffer != NULL &&
310 1133 : (((GstClockTime) ABS (GST_CLOCK_DIFF (current,
311 : GST_BUFFER_PTS (buf)))) > base))) {
312 : /* keep last buffer */
313 : } else {
314 : /* update last buffer */
315 2240 : if (pad->buffer != NULL)
316 1824 : gst_buffer_unref (pad->buffer);
317 2240 : pad->buffer = gst_collect_pads_pop (collect, data);
318 : }
319 :
320 2974 : gst_buffer_unref (buf);
321 : }
322 :
323 3106 : return TRUE;
324 : }
325 :
326 : /**
327 : * @brief A function call to make tensors from collected pads.
328 : * It decide which buffer is going to be used according to sync option.
329 : * @return True to push buffer.
330 : */
331 : gboolean
332 7215 : gst_tensor_time_sync_buffer_from_collectpad (GstCollectPads * collect,
333 : tensor_time_sync_data * sync, GstClockTime current_time,
334 : GstBuffer * tensors_buf, GstTensorsConfig * configs, gboolean * is_eos)
335 : {
336 7215 : GSList *walk = NULL;
337 : GstCollectData *data;
338 : GstTensorCollectPadData *pad;
339 7215 : GstBuffer *buf = NULL;
340 : GstMemory *mem;
341 7215 : gint old_numerator = G_MAXINT;
342 7215 : gint old_denominator = G_MAXINT;
343 : guint counting, empty_pad;
344 : GstTensorsConfig in_configs;
345 7215 : GstClockTime base_time = 0;
346 : GstTensorInfo *_info;
347 : guint i, j;
348 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT];
349 : tensor_format in_formats[NNS_TENSOR_SIZE_LIMIT];
350 :
351 14430 : g_return_val_if_fail (collect != NULL, FALSE);
352 7215 : g_return_val_if_fail (sync != NULL, FALSE);
353 7215 : g_return_val_if_fail (tensors_buf != NULL, FALSE);
354 7215 : g_return_val_if_fail (configs != NULL, FALSE);
355 7215 : g_return_val_if_fail (is_eos != NULL, FALSE);
356 :
357 7215 : walk = collect->data;
358 7215 : counting = empty_pad = 0;
359 :
360 7215 : if (sync->mode == SYNC_BASEPAD) {
361 711 : walk = g_slist_nth (walk, sync->data_basepad.sink_id);
362 711 : if (walk == NULL) {
363 0 : GST_ERROR_OBJECT (collect, "Cannot get GstCollectData from GSList");
364 0 : return FALSE;
365 : }
366 :
367 711 : data = (GstCollectData *) walk->data;
368 711 : pad = (GstTensorCollectPadData *) data;
369 :
370 711 : buf = gst_collect_pads_peek (collect, data);
371 711 : if (buf != NULL) {
372 649 : if (pad->buffer != NULL)
373 624 : base_time =
374 624 : MIN ((GstClockTimeDiff) sync->data_basepad.duration,
375 : ABS (GST_CLOCK_DIFF (GST_BUFFER_PTS (buf),
376 : GST_BUFFER_PTS (pad->buffer))) - 1);
377 649 : gst_buffer_unref (buf);
378 : }
379 : }
380 :
381 7215 : walk = collect->data;
382 :
383 7215 : gst_tensors_config_init (&in_configs);
384 :
385 21950 : while (walk) {
386 15407 : gboolean configured = FALSE;
387 15407 : gboolean is_empty = FALSE;
388 :
389 15407 : data = (GstCollectData *) walk->data;
390 15407 : pad = (GstTensorCollectPadData *) data;
391 :
392 15407 : if (gst_pad_has_current_caps (data->pad)) {
393 15406 : GstCaps *caps = gst_pad_get_current_caps (data->pad);
394 :
395 15406 : gst_tensors_config_free (&in_configs);
396 15406 : configured = gst_tensors_config_from_caps (&in_configs, caps, TRUE);
397 :
398 15406 : gst_caps_unref (caps);
399 : }
400 :
401 : /**
402 : * This would be an internal logic error.
403 : * in_configs should be already confirmed valid at the negotiation phase
404 : * and this function should be called in a running pipeline.
405 : * If new sync mode is enabled (e.g., handle output when a pad gets new buffer),
406 : * this may cause unexpected exception.
407 : */
408 15407 : if (!configured) {
409 1 : return FALSE;
410 : }
411 :
412 15406 : if (in_configs.rate_d < old_denominator)
413 7214 : old_denominator = in_configs.rate_d;
414 15406 : if (in_configs.rate_n < old_numerator)
415 7422 : old_numerator = in_configs.rate_n;
416 :
417 15406 : walk = g_slist_next (walk);
418 :
419 15406 : switch (sync->mode) {
420 3777 : case SYNC_SLOWEST:
421 : /* fall-through */
422 : case SYNC_BASEPAD:
423 3777 : if (!_gst_tensor_time_sync_buffer_update (collect, data,
424 : current_time, base_time, sync))
425 671 : return FALSE;
426 3106 : buf = gst_buffer_ref (pad->buffer);
427 3106 : is_empty = (buf == NULL);
428 3106 : break;
429 257 : case SYNC_NOSYNC:
430 257 : buf = gst_collect_pads_pop (collect, data);
431 257 : is_empty = (buf == NULL);
432 257 : break;
433 11372 : case SYNC_REFRESH:
434 11372 : buf = gst_collect_pads_pop (collect, data);
435 11372 : if (buf != NULL) {
436 5687 : if (pad->buffer != NULL) {
437 5685 : gst_buffer_unref (pad->buffer);
438 : }
439 5687 : pad->buffer = gst_buffer_ref (buf);
440 : } else {
441 5685 : if (pad->buffer == NULL) {
442 0 : *is_eos = FALSE;
443 0 : ml_logd ("Not the all buffers are arrived yet.");
444 0 : return FALSE;
445 : }
446 5685 : is_empty = TRUE;
447 5685 : buf = gst_buffer_ref (pad->buffer);
448 : }
449 11372 : break;
450 0 : default:
451 0 : break;
452 : }
453 :
454 14735 : if (GST_IS_BUFFER (buf)) {
455 14735 : guint32 n_tensor = gst_tensor_buffer_get_count (buf);
456 14735 : buf = gst_tensor_buffer_from_config (buf, &in_configs);
457 :
458 : /** These are internal logic error. If given inputs are incorrect,
459 : the negotiation should have been failed before this stage. */
460 14735 : if (gst_tensors_config_is_static (&in_configs))
461 14709 : g_assert (n_tensor == in_configs.info.num_tensors);
462 14735 : g_assert ((counting + n_tensor) <= NNS_TENSOR_SIZE_LIMIT);
463 :
464 14735 : if (gst_tensors_config_is_flexible (&in_configs))
465 26 : configs->info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
466 :
467 29850 : for (i = 0; i < n_tensor; ++i) {
468 15115 : in_mem[counting] = gst_tensor_buffer_get_nth_memory (buf, i);
469 :
470 : /* set info */
471 15115 : gst_tensor_info_copy (gst_tensors_info_get_nth_info (&configs->info,
472 15115 : counting), gst_tensors_info_get_nth_info (&in_configs.info, i));
473 15115 : in_formats[counting] = in_configs.info.format;
474 15115 : counting++;
475 : }
476 :
477 14735 : gst_buffer_unref (buf);
478 : }
479 14735 : if (is_empty)
480 5685 : empty_pad++;
481 : }
482 :
483 : /* append memories to output buffer */
484 20864 : for (i = 0; i < counting; i++) {
485 14321 : _info = gst_tensors_info_get_nth_info (&configs->info, i);
486 14321 : mem = in_mem[i];
487 :
488 14321 : if (gst_tensors_config_is_flexible (configs)) {
489 : /* append header if input tensor is not flexible */
490 46 : if (in_formats[i] != _NNS_TENSOR_FORMAT_FLEXIBLE) {
491 : GstTensorMetaInfo meta;
492 :
493 20 : gst_tensor_info_convert_to_meta (_info, &meta);
494 20 : mem = gst_tensor_meta_info_append_header (&meta, in_mem[i]);
495 20 : gst_memory_unref (in_mem[i]);
496 : }
497 : }
498 :
499 14321 : if (!gst_tensor_buffer_append_memory (tensors_buf, mem, _info)) {
500 0 : for (j = i + 1; j < counting; j++)
501 0 : gst_memory_unref (in_mem[j]);
502 :
503 0 : nns_loge ("Failed to append memory to buffer.");
504 0 : return FALSE;
505 : }
506 : }
507 :
508 6543 : configs->info.num_tensors = counting;
509 6543 : configs->rate_d = old_denominator;
510 6543 : configs->rate_n = old_numerator;
511 :
512 6543 : GST_BUFFER_PTS (tensors_buf) = current_time;
513 :
514 6543 : gst_tensors_config_free (&in_configs);
515 :
516 : /* check eos */
517 6543 : *is_eos = _gst_tensor_time_sync_is_eos (collect, sync, empty_pad);
518 6543 : return !(*is_eos);
519 : }
520 :
521 : /**
522 : * @brief Configure gst-buffer with tensors information.
523 : * NNStreamer handles single memory chunk as single tensor.
524 : * If incoming buffer has invalid memories, separate it and generate new gst-buffer using tensors information.
525 : * Note that this function always takes the ownership of input buffer.
526 : * @param in input buffer
527 : * @param config tensors config structure
528 : * @return Newly allocated buffer. Null if failed. Caller should unref the buffer using gst_buffer_unref().
529 : */
530 : GstBuffer *
531 37030 : gst_tensor_buffer_from_config (GstBuffer * in, GstTensorsConfig * config)
532 : {
533 37030 : GstBuffer *out = NULL;
534 37030 : GstMemory *all = NULL;
535 : GstMapInfo map;
536 : guint i, num;
537 : gsize total, offset;
538 : gsize mem_size[NNS_TENSOR_MEMORY_MAX];
539 37030 : gboolean configured = FALSE;
540 37030 : gboolean is_extra = FALSE;
541 :
542 37030 : if (!GST_IS_BUFFER (in)) {
543 1 : nns_loge ("Failed to get tensor buffer, invalid input buffer.");
544 37030 : return NULL;
545 : }
546 :
547 37029 : if (!gst_tensors_config_validate (config)) {
548 2 : nns_loge ("Failed to get tensor buffer, invalid tensor configuration.");
549 2 : goto error;
550 : }
551 :
552 37027 : num = gst_buffer_n_memory (in);
553 37027 : total = gst_buffer_get_size (in);
554 :
555 : /* get memory size */
556 37027 : if (gst_tensors_config_is_static (config)) {
557 36948 : if (num == config->info.num_tensors) {
558 : /* Do nothing, pass input buffer. */
559 36941 : out = gst_buffer_ref (in);
560 36941 : goto done;
561 : }
562 :
563 7 : num = config->info.num_tensors;
564 7 : if ((is_extra = (num > NNS_TENSOR_MEMORY_MAX)))
565 5 : num = NNS_TENSOR_MEMORY_MAX;
566 93 : for (i = 0; i < num; i++)
567 86 : mem_size[i] = gst_tensors_info_get_size (&config->info, i);
568 7 : if (is_extra) {
569 5 : mem_size[num - 1] += sizeof (GstTensorExtraInfo);
570 25 : for (; i < config->info.num_tensors; i++)
571 20 : mem_size[num - 1] += gst_tensors_info_get_size (&config->info, i);
572 : }
573 : } else {
574 79 : if (num > 1) {
575 : /* Suppose it is already configured. */
576 20 : out = gst_buffer_ref (in);
577 20 : goto done;
578 : }
579 :
580 59 : if (!gst_buffer_map (in, &map, GST_MAP_READ)) {
581 0 : nns_loge ("Failed to get tensor buffer, cannot get the memory info.");
582 0 : goto error;
583 : }
584 :
585 59 : num = 0;
586 59 : offset = 0;
587 120 : while (offset < total) {
588 : GstTensorMetaInfo meta;
589 61 : gpointer h = map.data + offset;
590 :
591 61 : if (num >= NNS_TENSOR_MEMORY_MAX - 1) {
592 : /* Suppose remained memory may include extra tensors. */
593 0 : mem_size[num++] = total - offset;
594 0 : break;
595 : }
596 :
597 61 : gst_tensor_meta_info_parse_header (&meta, h);
598 61 : mem_size[num] = gst_tensor_meta_info_get_header_size (&meta);
599 61 : mem_size[num] += gst_tensor_meta_info_get_data_size (&meta);
600 :
601 61 : offset += mem_size[num];
602 61 : num++;
603 : }
604 :
605 59 : gst_buffer_unmap (in, &map);
606 :
607 59 : if (num == 1) {
608 : /* Do nothing, pass input buffer. */
609 58 : out = gst_buffer_ref (in);
610 58 : goto done;
611 : }
612 : }
613 :
614 : /* configure output buffer */
615 8 : out = gst_buffer_new ();
616 8 : all = gst_buffer_get_all_memory (in);
617 8 : offset = 0;
618 :
619 95 : for (i = 0; i < num; i++) {
620 : /* invalid memory size */
621 88 : if (offset + mem_size[i] > total) {
622 1 : nns_loge ("Failed to get tensor buffer, data size is mismatched.");
623 1 : goto error;
624 : }
625 :
626 87 : gst_buffer_append_memory (out, gst_memory_share (all, offset, mem_size[i]));
627 87 : offset += mem_size[i];
628 : }
629 :
630 7 : gst_buffer_copy_into (out, in, GST_BUFFER_COPY_METADATA, 0, -1);
631 :
632 37026 : done:
633 37026 : configured = TRUE;
634 37029 : error:
635 37029 : gst_buffer_unref (in);
636 :
637 37029 : if (all)
638 8 : gst_memory_unref (all);
639 :
640 37029 : if (!configured) {
641 3 : if (out) {
642 1 : gst_buffer_unref (out);
643 1 : out = NULL;
644 : }
645 : }
646 :
647 37029 : return out;
648 : }
649 :
650 : /**
651 : * @brief Internal struct to handle aggregation data in hash table.
652 : */
653 : typedef struct
654 : {
655 : GstAdapter *adapter;
656 : } gst_tensor_aggregation_data_s;
657 :
658 : /**
659 : * @brief Internal function to free aggregation data.
660 : */
661 : static void
662 868 : gst_tensor_aggregation_free_data (gpointer data)
663 : {
664 : gst_tensor_aggregation_data_s *aggr;
665 :
666 868 : aggr = (gst_tensor_aggregation_data_s *) data;
667 868 : if (aggr) {
668 868 : gst_adapter_clear (aggr->adapter);
669 868 : g_object_unref (aggr->adapter);
670 :
671 868 : g_free (aggr);
672 : }
673 868 : }
674 :
675 : /**
676 : * @brief Internal function to add new aggregation data.
677 : */
678 : static gst_tensor_aggregation_data_s *
679 927 : gst_tensor_aggregation_add_data (GHashTable * table, const gint64 key)
680 : {
681 : gst_tensor_aggregation_data_s *aggr;
682 : gint64 *hashkey;
683 :
684 927 : g_return_val_if_fail (table != NULL, NULL);
685 :
686 927 : hashkey = g_new (gint64, 1);
687 927 : *hashkey = key;
688 :
689 927 : aggr = g_new0 (gst_tensor_aggregation_data_s, 1);
690 927 : aggr->adapter = gst_adapter_new ();
691 :
692 927 : g_hash_table_insert (table, hashkey, aggr);
693 927 : return aggr;
694 : }
695 :
696 : /**
697 : * @brief Internal function to get aggregation data.
698 : */
699 : static gst_tensor_aggregation_data_s *
700 304 : gst_tensor_aggregation_get_data (GHashTable * table, const gint64 key)
701 : {
702 304 : g_return_val_if_fail (table != NULL, NULL);
703 :
704 304 : return (gst_tensor_aggregation_data_s *) g_hash_table_lookup (table, &key);
705 : }
706 :
707 : /**
708 : * @brief Internal function to remove all buffers from aggregation data.
709 : */
710 : static void
711 3493 : gst_tensor_aggregation_clear_internal (gpointer key, gpointer value,
712 : gpointer user_data)
713 : {
714 : gst_tensor_aggregation_data_s *aggr;
715 :
716 : UNUSED (key);
717 : UNUSED (user_data);
718 :
719 3493 : aggr = (gst_tensor_aggregation_data_s *) value;
720 3493 : if (aggr) {
721 3493 : gst_adapter_clear (aggr->adapter);
722 : }
723 3493 : }
724 :
725 : /**
726 : * @brief Gets new hash table for tensor aggregation.
727 : * @return Newly allocated hash table, caller should release this using g_hash_table_destroy().
728 : */
729 : GHashTable *
730 924 : gst_tensor_aggregation_init (void)
731 : {
732 : GHashTable *table;
733 :
734 924 : table = g_hash_table_new_full (g_int64_hash, g_int64_equal, g_free,
735 : gst_tensor_aggregation_free_data);
736 :
737 : /**
738 : * Add default adapter (for the case if buffer has no specific id).
739 : * If gst-buffer has tensor-meta which includes client-id,
740 : * e.g., aggregation frames from multiple clients on query-server pipeline,
741 : * nnstreamer element should parse meta and request adapter with this id.
742 : * However, on normal pipeline, gst-buffer does not contain tensor-meta,
743 : * then the element may request adapter with null key string.
744 : */
745 924 : gst_tensor_aggregation_add_data (table, 0);
746 :
747 924 : return table;
748 : }
749 :
750 : /**
751 : * @brief Clears buffers from adapter.
752 : * @param table a hash table instance initialized with gst_tensor_aggregation_init()
753 : * @param key the key to look up (set 0 to get default adapter)
754 : */
755 : void
756 1 : gst_tensor_aggregation_clear (GHashTable * table, const gint64 key)
757 : {
758 : gst_tensor_aggregation_data_s *aggr;
759 :
760 1 : g_return_if_fail (table != NULL);
761 1 : g_return_if_fail (key >= 0);
762 :
763 1 : aggr = gst_tensor_aggregation_get_data (table, key);
764 1 : gst_tensor_aggregation_clear_internal (NULL, aggr, NULL);
765 : }
766 :
767 : /**
768 : * @brief Clears buffers from all adapters in hash table.
769 : * @param table a hash table instance initialized with gst_tensor_aggregation_init()
770 : */
771 : void
772 3487 : gst_tensor_aggregation_clear_all (GHashTable * table)
773 : {
774 3487 : g_hash_table_foreach (table, gst_tensor_aggregation_clear_internal, NULL);
775 3487 : }
776 :
777 : /**
778 : * @brief Gets adapter from hash table.
779 : * @param table a hash table instance initialized with gst_tensor_aggregation_init()
780 : * @param key the key to look up (set 0 to get default adapter)
781 : * @return gst-adapter instance. DO NOT release this instance.
782 : */
783 : GstAdapter *
784 304 : gst_tensor_aggregation_get_adapter (GHashTable * table, const gint64 key)
785 : {
786 : gst_tensor_aggregation_data_s *aggr;
787 :
788 304 : g_return_val_if_fail (table != NULL, NULL);
789 303 : g_return_val_if_fail (key >= 0, NULL);
790 :
791 303 : aggr = gst_tensor_aggregation_get_data (table, key);
792 303 : if (!aggr) {
793 : /*append new data */
794 3 : aggr = gst_tensor_aggregation_add_data (table, key);
795 : }
796 :
797 303 : return aggr->adapter;
798 : }
799 :
800 : /**
801 : * @brief Internal function to check tensor dimensions to append old caps for backward compatibility (rank 4).
802 : */
803 : static gboolean
804 14356 : _append_prev_caps (const GstTensorsConfig * config)
805 : {
806 : GstTensorsInfo *info;
807 : GstTensorInfo *_info;
808 : guint i, min_rank;
809 :
810 14356 : g_return_val_if_fail (config != NULL, FALSE);
811 :
812 14356 : info = (GstTensorsInfo *) (&config->info);
813 14356 : if (!gst_tensors_info_validate (info))
814 7090 : return FALSE;
815 :
816 15538 : for (i = 0; i < info->num_tensors; i++) {
817 8354 : _info = gst_tensors_info_get_nth_info (info, i);
818 :
819 8354 : min_rank = gst_tensor_dimension_get_min_rank (_info->dimension);
820 :
821 8354 : if (min_rank > NNS_TENSOR_RANK_LIMIT_PREV)
822 82 : return FALSE;
823 : }
824 :
825 7184 : return TRUE;
826 : }
827 :
828 : /**
829 : * @brief Internal function to check tensors-info string includes item (dimension, type, or name).
830 : */
831 : static gboolean
832 12798 : _is_empty_info_string (const gchar * str)
833 : {
834 12798 : gboolean is_empty = TRUE;
835 :
836 12798 : if (str) {
837 : gchar **str_array;
838 : guint i, num;
839 :
840 12798 : str_array = g_strsplit (str, ",", -1);
841 12798 : num = g_strv_length (str_array);
842 :
843 12798 : for (i = 0; i < num; i++) {
844 8254 : g_strstrip (str_array[i]);
845 :
846 8254 : if (str_array[i] && str_array[i][0] != '\0') {
847 8254 : is_empty = FALSE;
848 8254 : break;
849 : }
850 : }
851 :
852 12798 : g_strfreev (str_array);
853 : }
854 :
855 12798 : return is_empty;
856 : }
857 :
858 : /**
859 : * @brief Internal function to get caps for single tensor from config.
860 : */
861 : static GstCaps *
862 7367 : _get_tensor_caps (const GstTensorsConfig * config)
863 : {
864 : GstCaps *caps, *prev1, *prev2;
865 : GstTensorsInfo *info;
866 : GstTensorInfo *_info;
867 : gboolean append_prev;
868 :
869 7367 : g_return_val_if_fail (config != NULL, NULL);
870 :
871 7367 : info = (GstTensorsInfo *) (&config->info);
872 7367 : if (info->num_tensors > 1)
873 135 : return NULL;
874 :
875 7232 : caps = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
876 7232 : _info = gst_tensors_info_get_nth_info (info, 0);
877 :
878 : /* caps for backward compatibility */
879 7232 : prev1 = prev2 = NULL;
880 7232 : append_prev = _append_prev_caps (config);
881 7232 : if (append_prev) {
882 3654 : prev1 = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
883 3654 : prev2 = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
884 : }
885 :
886 7232 : if (gst_tensor_dimension_is_valid (_info->dimension)) {
887 4028 : g_autofree gchar *dim_str =
888 4028 : gst_tensor_get_dimension_string (_info->dimension);
889 :
890 4028 : gst_caps_set_simple (caps, "dimension", G_TYPE_STRING, dim_str, NULL);
891 :
892 4028 : if (append_prev) {
893 3654 : g_autofree gchar *dim_prev1 =
894 3654 : gst_tensor_get_rank_dimension_string (_info->dimension,
895 : NNS_TENSOR_RANK_LIMIT_PREV, FALSE);
896 3654 : g_autofree gchar *dim_prev2 =
897 3654 : gst_tensor_get_rank_dimension_string (_info->dimension,
898 : NNS_TENSOR_RANK_LIMIT_PREV, TRUE);
899 :
900 3654 : gst_caps_set_simple (prev1, "dimension", G_TYPE_STRING, dim_prev1, NULL);
901 3654 : gst_caps_set_simple (prev2, "dimension", G_TYPE_STRING, dim_prev2, NULL);
902 : }
903 : }
904 :
905 7232 : if (_info->type != _NNS_END) {
906 4446 : const gchar *type_str = gst_tensor_get_type_string (_info->type);
907 :
908 4446 : gst_caps_set_simple (caps, "type", G_TYPE_STRING, type_str, NULL);
909 :
910 4446 : if (append_prev) {
911 3654 : gst_caps_set_simple (prev1, "type", G_TYPE_STRING, type_str, NULL);
912 3654 : gst_caps_set_simple (prev2, "type", G_TYPE_STRING, type_str, NULL);
913 : }
914 : }
915 :
916 7232 : if (config->rate_n >= 0 && config->rate_d > 0) {
917 2536 : gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION,
918 2536 : config->rate_n, config->rate_d, NULL);
919 :
920 2536 : if (append_prev) {
921 2370 : gst_caps_set_simple (prev1, "framerate", GST_TYPE_FRACTION,
922 2370 : config->rate_n, config->rate_d, NULL);
923 2370 : gst_caps_set_simple (prev2, "framerate", GST_TYPE_FRACTION,
924 2370 : config->rate_n, config->rate_d, NULL);
925 : }
926 : }
927 :
928 7232 : if (append_prev)
929 3654 : caps = gst_caps_merge (caps, gst_caps_merge (prev1, prev2));
930 :
931 7232 : return caps;
932 : }
933 :
934 : /**
935 : * @brief Internal function to get caps for multi tensors from config.
936 : */
937 : static GstCaps *
938 7124 : _get_tensors_caps (const GstTensorsConfig * config)
939 : {
940 : GstCaps *caps, *prev1, *prev2;
941 : gboolean append_prev;
942 :
943 7124 : g_return_val_if_fail (config != NULL, NULL);
944 :
945 7124 : caps = gst_caps_from_string (GST_TENSORS_CAP_DEFAULT);
946 :
947 : /* caps for backward compatibility */
948 7124 : prev1 = prev2 = NULL;
949 7124 : append_prev = _append_prev_caps (config);
950 7124 : if (append_prev) {
951 3530 : prev1 = gst_caps_from_string (GST_TENSORS_CAP_DEFAULT);
952 3530 : prev2 = gst_caps_from_string (GST_TENSORS_CAP_DEFAULT);
953 : }
954 :
955 7124 : if (config->info.num_tensors > 0) {
956 6399 : g_autofree gchar *type_str =
957 6399 : gst_tensors_info_get_types_string (&config->info);
958 6399 : g_autofree gchar *dim_str =
959 6399 : gst_tensors_info_get_dimensions_string (&config->info);
960 6399 : gboolean has_type = !_is_empty_info_string (type_str);
961 6399 : gboolean has_dim = !_is_empty_info_string (dim_str);
962 :
963 6399 : gst_caps_set_simple (caps, "num_tensors", G_TYPE_INT,
964 6399 : config->info.num_tensors, NULL);
965 6399 : if (has_dim) {
966 3918 : gst_caps_set_simple (caps, "dimensions", G_TYPE_STRING, dim_str, NULL);
967 : }
968 6399 : if (has_type) {
969 4336 : gst_caps_set_simple (caps, "types", G_TYPE_STRING, type_str, NULL);
970 : }
971 :
972 6399 : if (append_prev) {
973 3530 : gst_caps_set_simple (prev1, "num_tensors", G_TYPE_INT,
974 3530 : config->info.num_tensors, NULL);
975 3530 : if (has_dim) {
976 3530 : g_autofree gchar *dimstr =
977 3530 : gst_tensors_info_get_rank_dimensions_string (&config->info,
978 : NNS_TENSOR_RANK_LIMIT_PREV, FALSE);
979 :
980 3530 : gst_caps_set_simple (prev1, "dimensions", G_TYPE_STRING, dimstr, NULL);
981 : }
982 3530 : if (has_type) {
983 3530 : gst_caps_set_simple (prev1, "types", G_TYPE_STRING, type_str, NULL);
984 : }
985 :
986 3530 : gst_caps_set_simple (prev2, "num_tensors", G_TYPE_INT,
987 3530 : config->info.num_tensors, NULL);
988 3530 : if (has_dim) {
989 3530 : g_autofree gchar *dimstr =
990 3530 : gst_tensors_info_get_rank_dimensions_string (&config->info,
991 : NNS_TENSOR_RANK_LIMIT_PREV, TRUE);
992 :
993 3530 : gst_caps_set_simple (prev2, "dimensions", G_TYPE_STRING, dimstr, NULL);
994 : }
995 3530 : if (has_type) {
996 3530 : gst_caps_set_simple (prev2, "types", G_TYPE_STRING, type_str, NULL);
997 : }
998 : }
999 : }
1000 :
1001 7124 : if (config->rate_n >= 0 && config->rate_d > 0) {
1002 2383 : gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION,
1003 2383 : config->rate_n, config->rate_d, NULL);
1004 :
1005 2383 : if (append_prev) {
1006 2207 : gst_caps_set_simple (prev1, "framerate", GST_TYPE_FRACTION,
1007 2207 : config->rate_n, config->rate_d, NULL);
1008 2207 : gst_caps_set_simple (prev2, "framerate", GST_TYPE_FRACTION,
1009 2207 : config->rate_n, config->rate_d, NULL);
1010 : }
1011 : }
1012 :
1013 7124 : if (append_prev)
1014 3530 : caps = gst_caps_merge (caps, gst_caps_merge (prev1, prev2));
1015 :
1016 7124 : return caps;
1017 : }
1018 :
1019 : /**
1020 : * @brief Internal function to get caps for flexible tensor from config.
1021 : */
1022 : static GstCaps *
1023 7818 : _get_flexible_caps (const GstTensorsConfig * config)
1024 : {
1025 : GstCaps *caps;
1026 :
1027 7818 : caps = gst_caps_from_string (GST_TENSORS_FLEX_CAP_DEFAULT);
1028 :
1029 7818 : if (config->rate_n >= 0 && config->rate_d > 0) {
1030 2111 : gst_caps_set_simple (caps, "framerate", GST_TYPE_FRACTION,
1031 2111 : config->rate_n, config->rate_d, NULL);
1032 : }
1033 :
1034 7818 : return caps;
1035 : }
1036 :
1037 : /**
1038 : * @brief Check given mimetype is tensor stream.
1039 : * @param structure structure to be interpreted
1040 : * @return TRUE if mimetype is tensor stream
1041 : */
1042 : gboolean
1043 19172 : gst_structure_is_tensor_stream (const GstStructure * structure)
1044 : {
1045 : const gchar *name;
1046 :
1047 19172 : name = gst_structure_get_name (structure);
1048 19172 : g_return_val_if_fail (name != NULL, FALSE);
1049 :
1050 34126 : return (g_str_equal (name, NNS_MIMETYPE_TENSOR) ||
1051 14954 : g_str_equal (name, NNS_MIMETYPE_TENSORS));
1052 : }
1053 :
1054 : /**
1055 : * @brief Get media type from structure
1056 : * @param structure structure to be interpreted
1057 : * @return corresponding media type (returns _NNS_MEDIA_INVALID for unsupported type)
1058 : */
1059 : media_type
1060 20179 : gst_structure_get_media_type (const GstStructure * structure)
1061 : {
1062 : const gchar *name;
1063 :
1064 20179 : name = gst_structure_get_name (structure);
1065 :
1066 20179 : g_return_val_if_fail (name != NULL, _NNS_MEDIA_INVALID);
1067 :
1068 20179 : if (g_str_has_prefix (name, "video/")) {
1069 2768 : return _NNS_VIDEO;
1070 : }
1071 :
1072 17411 : if (g_str_has_prefix (name, "audio/")) {
1073 2185 : return _NNS_AUDIO;
1074 : }
1075 :
1076 15226 : if (g_str_has_prefix (name, "text/")) {
1077 2158 : return _NNS_TEXT;
1078 : }
1079 :
1080 13068 : if (g_str_equal (name, "application/octet-stream")) {
1081 4414 : return _NNS_OCTET;
1082 : }
1083 :
1084 8654 : if (gst_structure_is_tensor_stream (structure)) {
1085 2153 : return _NNS_TENSOR;
1086 : }
1087 :
1088 : /* unknown or unsupported type */
1089 6501 : return _NNS_MEDIA_INVALID;
1090 : }
1091 :
1092 : /**
1093 : * @brief Parse caps from peer pad and set tensors config.
1094 : * @param pad GstPad to get the capabilities
1095 : * @param config tensors config structure to be filled
1096 : * @param is_fixed flag to be updated when peer caps is fixed (not mandatory, do nothing when the param is null)
1097 : * @return TRUE if successfully configured from peer
1098 : */
1099 : gboolean
1100 4297 : gst_tensors_config_from_peer (GstPad * pad, GstTensorsConfig * config,
1101 : gboolean * is_fixed)
1102 : {
1103 : GstCaps *peer_caps;
1104 : GstStructure *structure;
1105 4297 : gboolean ret = FALSE;
1106 :
1107 4297 : g_return_val_if_fail (GST_IS_PAD (pad), FALSE);
1108 4297 : g_return_val_if_fail (config != NULL, FALSE);
1109 :
1110 4297 : gst_tensors_config_init (config);
1111 :
1112 4297 : if ((peer_caps = gst_pad_peer_query_caps (pad, NULL))) {
1113 4297 : if (gst_caps_get_size (peer_caps) > 0) {
1114 2187 : structure = gst_caps_get_structure (peer_caps, 0);
1115 2187 : ret = gst_tensors_config_from_structure (config, structure);
1116 : }
1117 :
1118 4297 : if (ret && is_fixed)
1119 30 : *is_fixed = gst_caps_is_fixed (peer_caps);
1120 :
1121 4297 : gst_caps_unref (peer_caps);
1122 : }
1123 :
1124 4297 : return ret;
1125 : }
1126 :
1127 : /**
1128 : * @brief Check whether two structures have the same dimension
1129 : */
1130 : static gboolean
1131 261 : _is_structure_dimension_same (GstStructure * st1, GstStructure * st2,
1132 : const gchar * fieldname)
1133 : {
1134 : const char *dim_str1;
1135 : const char *dim_str2;
1136 :
1137 261 : g_return_val_if_fail (gst_structure_has_field (st1, fieldname), FALSE);
1138 261 : g_return_val_if_fail (gst_structure_has_field (st2, fieldname), FALSE);
1139 :
1140 261 : dim_str1 = gst_structure_get_string (st1, fieldname);
1141 261 : dim_str2 = gst_structure_get_string (st2, fieldname);
1142 :
1143 261 : return gst_tensor_dimension_string_is_equal (dim_str1, dim_str2);
1144 : }
1145 :
1146 : /**
1147 : * @brief Update caps dimensions for negotiation
1148 : * @param caps caps to compare and update
1149 : * @param filter caps to compare
1150 : */
1151 : void
1152 1640 : gst_tensor_caps_update_dimension (GstCaps * caps, GstCaps * filter)
1153 : {
1154 : GstStructure *st_caps, *st_filter;
1155 : guint i, j;
1156 :
1157 1640 : g_return_if_fail (GST_IS_CAPS (caps));
1158 1640 : g_return_if_fail (GST_IS_CAPS (filter));
1159 :
1160 5008 : for (i = 0; i < gst_caps_get_size (caps); i++) {
1161 3368 : st_caps = gst_caps_get_structure (caps, i);
1162 :
1163 3368 : if (!gst_structure_is_tensor_stream (st_caps))
1164 0 : continue;
1165 :
1166 10477 : for (j = 0; j < gst_caps_get_size (filter); j++) {
1167 7109 : st_filter = gst_caps_get_structure (filter, j);
1168 :
1169 7109 : if (!gst_structure_is_tensor_stream (st_filter))
1170 0 : continue;
1171 :
1172 : /* other/tensor */
1173 7109 : if (gst_structure_has_field (st_caps, "dimension")
1174 3880 : && gst_structure_has_field (st_filter, "dimension")) {
1175 : /* update dimensions for negotiation */
1176 232 : if (_is_structure_dimension_same (st_caps, st_filter, "dimension")) {
1177 116 : gst_structure_set (st_caps, "dimension", G_TYPE_STRING,
1178 : gst_structure_get_string (st_filter, "dimension"), NULL);
1179 : }
1180 : }
1181 : /* other/tensors */
1182 6993 : else if (gst_structure_has_field (st_caps, "dimensions")
1183 2104 : && gst_structure_has_field (st_filter, "dimensions")) {
1184 : /* update dimensions for negotiation */
1185 142 : if (_is_structure_dimension_same (st_caps, st_filter, "dimensions")) {
1186 141 : gst_structure_set (st_caps, "dimensions", G_TYPE_STRING,
1187 : gst_structure_get_string (st_filter, "dimensions"), NULL);
1188 : }
1189 : }
1190 : }
1191 : }
1192 : }
1193 :
1194 : /**
1195 : * @brief Try intersecting @caps1 and @caps2 for tensor stream
1196 : * @param caps1 a GstCaps to intersect
1197 : * @param caps2 a GstCaps to intersect
1198 : * @return TRUE if intersection would be not empty.
1199 : */
1200 : gboolean
1201 14 : gst_tensor_caps_can_intersect (GstCaps * caps1, GstCaps * caps2)
1202 : {
1203 : GstStructure *structure1;
1204 : GstStructure *structure2;
1205 : GstStructure *structure_copy1;
1206 : GstStructure *structure_copy2;
1207 :
1208 : const gchar *name1;
1209 : const gchar *name2;
1210 :
1211 : gboolean intersectable;
1212 :
1213 14 : if (gst_caps_can_intersect (caps1, caps2))
1214 10 : return TRUE;
1215 :
1216 4 : structure1 = gst_caps_get_structure (caps1, 0);
1217 4 : structure2 = gst_caps_get_structure (caps2, 0);
1218 :
1219 4 : if (!gst_structure_is_tensor_stream (structure1)
1220 4 : || !gst_structure_is_tensor_stream (structure2))
1221 0 : return FALSE;
1222 :
1223 4 : name1 = gst_structure_get_name (structure1);
1224 4 : name2 = gst_structure_get_name (structure2);
1225 :
1226 4 : if (!g_str_equal (name1, name2))
1227 1 : return FALSE;
1228 :
1229 : /* other/tensor */
1230 3 : if (g_str_equal (name1, NNS_MIMETYPE_TENSOR)) {
1231 3 : if (gst_structure_has_field (structure1, "dimension")
1232 3 : && gst_structure_has_field (structure2, "dimension")) {
1233 3 : if (!_is_structure_dimension_same (structure1, structure2, "dimension"))
1234 1 : return FALSE;
1235 : }
1236 : }
1237 : /* other/tensors */
1238 0 : else if (gst_structure_has_field (structure1, "dimensions")
1239 0 : && gst_structure_has_field (structure2, "dimensions")) {
1240 0 : if (!_is_structure_dimension_same (structure1, structure2, "dimensions"))
1241 0 : return FALSE;
1242 : }
1243 :
1244 2 : structure_copy1 = gst_structure_copy (structure1);
1245 2 : structure_copy2 = gst_structure_copy (structure2);
1246 :
1247 2 : gst_structure_remove_field (structure_copy1, "dimension");
1248 2 : gst_structure_remove_field (structure_copy1, "dimensions");
1249 2 : gst_structure_remove_field (structure_copy2, "dimension");
1250 2 : gst_structure_remove_field (structure_copy2, "dimensions");
1251 :
1252 : intersectable =
1253 2 : gst_structure_can_intersect (structure_copy1, structure_copy2);
1254 :
1255 2 : gst_structure_free (structure_copy1);
1256 2 : gst_structure_free (structure_copy2);
1257 :
1258 2 : return intersectable;
1259 : }
1260 :
1261 : /**
1262 : * @brief Get pad caps from tensors config and caps of the peer connected to the pad.
1263 : * @param pad GstPad to get possible caps
1264 : * @param config tensors config structure
1265 : * @return caps for given config. Caller is responsible for unreffing the returned caps.
1266 : */
1267 : GstCaps *
1268 1210 : gst_tensor_pad_caps_from_config (GstPad * pad, const GstTensorsConfig * config)
1269 : {
1270 1210 : GstCaps *caps = NULL;
1271 : GstCaps *templ;
1272 : gboolean is_flexible, peer_is_flexible, peer_has_tensor_caps;
1273 : GstCaps *peer_caps;
1274 :
1275 2420 : g_return_val_if_fail (GST_IS_PAD (pad), NULL);
1276 1210 : g_return_val_if_fail (config != NULL, NULL);
1277 :
1278 1210 : templ = gst_pad_get_pad_template_caps (pad);
1279 :
1280 : /* check peer caps */
1281 1210 : peer_is_flexible = peer_has_tensor_caps = FALSE;
1282 :
1283 1210 : peer_caps = gst_pad_peer_query_caps (pad, NULL);
1284 1210 : if (peer_caps && gst_caps_get_size (peer_caps) > 0) {
1285 : GstCaps *tmp;
1286 : GstStructure *st;
1287 : GstTensorsConfig peer_config;
1288 :
1289 913 : tmp = gst_caps_from_string (GST_TENSOR_CAP_DEFAULT);
1290 913 : peer_has_tensor_caps = gst_caps_can_intersect (tmp, peer_caps);
1291 913 : gst_caps_unref (tmp);
1292 :
1293 913 : st = gst_caps_get_structure (peer_caps, 0);
1294 913 : if (gst_tensors_config_from_structure (&peer_config, st))
1295 913 : peer_is_flexible = gst_tensors_config_is_flexible (&peer_config);
1296 913 : gst_tensors_config_free (&peer_config);
1297 : }
1298 :
1299 : /* other/tensors (flexible) */
1300 1210 : is_flexible = gst_tensors_config_is_flexible (config);
1301 :
1302 1210 : if (is_flexible || peer_is_flexible) {
1303 71 : caps = _get_flexible_caps (config);
1304 71 : goto intersectable;
1305 : }
1306 :
1307 : /* other/tensor */
1308 1139 : if (config->info.num_tensors == 1 && peer_has_tensor_caps) {
1309 733 : caps = _get_tensor_caps (config);
1310 733 : if (peer_caps)
1311 733 : gst_tensor_caps_update_dimension (caps, peer_caps);
1312 :
1313 733 : if (gst_caps_can_intersect (caps, templ))
1314 729 : goto done;
1315 :
1316 4 : gst_caps_unref (caps);
1317 : }
1318 :
1319 : /* other/tensors (static) */
1320 410 : caps = _get_tensors_caps (config);
1321 410 : if (peer_caps)
1322 410 : gst_tensor_caps_update_dimension (caps, peer_caps);
1323 :
1324 0 : intersectable:
1325 481 : if (!gst_caps_can_intersect (caps, templ)) {
1326 0 : g_clear_pointer (&caps, gst_caps_unref);
1327 : }
1328 :
1329 481 : done:
1330 1210 : g_clear_pointer (&templ, gst_caps_unref);
1331 1210 : g_clear_pointer (&peer_caps, gst_caps_unref);
1332 :
1333 1210 : return gst_caps_truncate (caps);
1334 : }
1335 :
1336 : /**
1337 : * @brief Get all possible caps from tensors config. Unlike gst_tensor_pad_caps_from_config(), this function does not check peer caps.
1338 : * @param pad GstPad to get possible caps
1339 : * @param config tensors config structure
1340 : * @return caps for given config. Caller is responsible for unreffing the returned caps.
1341 : */
1342 : GstCaps *
1343 7740 : gst_tensor_pad_possible_caps_from_config (GstPad * pad,
1344 : const GstTensorsConfig * config)
1345 : {
1346 : GstCaps *caps, *tmp;
1347 : GstCaps *templ;
1348 :
1349 15480 : g_return_val_if_fail (GST_IS_PAD (pad), NULL);
1350 7740 : g_return_val_if_fail (config != NULL, NULL);
1351 :
1352 7740 : caps = gst_caps_new_empty ();
1353 7740 : templ = gst_pad_get_pad_template_caps (pad);
1354 :
1355 : /* append caps for static tensor */
1356 7740 : if (gst_tensors_config_is_static (config)) {
1357 : /* other/tensor */
1358 6615 : if ((tmp = _get_tensor_caps (config)) != NULL) {
1359 6480 : if (gst_caps_can_intersect (tmp, templ))
1360 6432 : gst_caps_append (caps, tmp);
1361 : else
1362 48 : gst_caps_unref (tmp);
1363 : }
1364 :
1365 : /* other/tensors */
1366 6615 : if ((tmp = _get_tensors_caps (config)) != NULL) {
1367 6615 : if (gst_caps_can_intersect (tmp, templ))
1368 6615 : gst_caps_append (caps, tmp);
1369 : else
1370 0 : gst_caps_unref (tmp);
1371 : }
1372 : }
1373 :
1374 : /* caps for flexible tensor */
1375 7740 : if ((tmp = _get_flexible_caps (config)) != NULL) {
1376 7740 : if (gst_caps_can_intersect (tmp, templ))
1377 7647 : gst_caps_append (caps, tmp);
1378 : else
1379 93 : gst_caps_unref (tmp);
1380 : }
1381 :
1382 : /* if no possible caps for given config, return null. */
1383 7740 : if (gst_caps_is_empty (caps)) {
1384 0 : g_clear_pointer (&caps, gst_caps_unref);
1385 : }
1386 :
1387 7740 : g_clear_pointer (&templ, gst_caps_unref);
1388 7740 : return caps;
1389 : }
1390 :
1391 : /**
1392 : * @brief Get tensor format of current pad caps.
1393 : * @param pad GstPad to check current caps.
1394 : * @return The tensor_format of current pad caps.
1395 : *
1396 : * If pad does not have tensor caps return _NNS_TENSOR_FORMAT_END
1397 : */
1398 : tensor_format
1399 129229 : gst_tensor_pad_get_format (GstPad * pad)
1400 : {
1401 : GstCaps *caps;
1402 129229 : tensor_format ret = _NNS_TENSOR_FORMAT_END;
1403 :
1404 129229 : g_return_val_if_fail (GST_IS_PAD (pad), _NNS_TENSOR_FORMAT_END);
1405 :
1406 129229 : caps = gst_pad_get_current_caps (pad);
1407 129229 : if (caps) {
1408 : GstTensorsConfig config;
1409 :
1410 129215 : if (gst_tensors_config_from_caps (&config, caps, TRUE)) {
1411 129104 : ret = config.info.format;
1412 : }
1413 129215 : gst_caps_unref (caps);
1414 129215 : gst_tensors_config_free (&config);
1415 : }
1416 :
1417 129229 : return ret;
1418 : }
1419 :
1420 : /**
1421 : * @brief Get caps from tensors config (for other/tensors)
1422 : * @param config tensors config info
1423 : * @return caps for given config
1424 : */
1425 : GstCaps *
1426 107 : gst_tensors_caps_from_config (const GstTensorsConfig * config)
1427 : {
1428 : GstCaps *caps;
1429 :
1430 107 : g_return_val_if_fail (config != NULL, NULL);
1431 :
1432 106 : if (gst_tensors_config_is_flexible (config)) {
1433 7 : caps = _get_flexible_caps (config);
1434 : } else {
1435 99 : caps = _get_tensors_caps (config);
1436 : }
1437 :
1438 106 : caps = gst_caps_truncate (caps);
1439 :
1440 106 : return caps;
1441 : }
1442 :
1443 : /**
1444 : * @brief Get tensor caps from tensors config
1445 : * @param config tensors config info
1446 : * @return caps for given config
1447 : */
1448 : GstCaps *
1449 20 : gst_tensor_caps_from_config (const GstTensorsConfig * config)
1450 : {
1451 : GstCaps *caps;
1452 :
1453 20 : g_return_val_if_fail (config != NULL, NULL);
1454 :
1455 19 : caps = _get_tensor_caps (config);
1456 19 : caps = gst_caps_truncate (caps);
1457 :
1458 19 : return caps;
1459 : }
1460 :
1461 : /**
1462 : * @brief Parse structure and set tensors config (for other/tensors)
1463 : * @param config tensors config structure to be filled
1464 : * @param structure structure to be interpreted
1465 : * @return TRUE if no error
1466 : */
1467 : gboolean
1468 159578 : gst_tensors_config_from_structure (GstTensorsConfig * config,
1469 : const GstStructure * structure)
1470 : {
1471 : const gchar *name;
1472 :
1473 159578 : g_return_val_if_fail (config != NULL, FALSE);
1474 159576 : gst_tensors_config_init (config);
1475 :
1476 159576 : g_return_val_if_fail (structure != NULL, FALSE);
1477 :
1478 159575 : name = gst_structure_get_name (structure);
1479 :
1480 159575 : if (g_str_equal (name, NNS_MIMETYPE_TENSOR)) {
1481 : /* other/tensor is always static */
1482 145806 : config->info.num_tensors = 1;
1483 :
1484 145806 : if (gst_structure_has_field (structure, "dimension")) {
1485 139831 : const gchar *dim_str = gst_structure_get_string (structure, "dimension");
1486 139831 : gst_tensor_parse_dimension (dim_str, config->info.info[0].dimension);
1487 : }
1488 :
1489 145806 : if (gst_structure_has_field (structure, "type")) {
1490 140004 : const gchar *type_str = gst_structure_get_string (structure, "type");
1491 140004 : config->info.info[0].type = gst_tensor_get_type (type_str);
1492 : }
1493 13769 : } else if (g_str_equal (name, NNS_MIMETYPE_TENSORS)) {
1494 13658 : tensor_format format = _NNS_TENSOR_FORMAT_END;
1495 :
1496 13658 : if (gst_structure_has_field (structure, "format")) {
1497 : const gchar *format_str;
1498 :
1499 13658 : format_str = gst_structure_get_string (structure, "format");
1500 13658 : format = gst_tensor_get_format (format_str);
1501 :
1502 13658 : if (format == _NNS_TENSOR_FORMAT_END) {
1503 1652 : GST_INFO
1504 : ("Invalid format %s, it should be one of %s. Suppose tensor format is unknown.",
1505 : _STR_NULL (format_str), GST_TENSOR_FORMAT_ALL);
1506 : }
1507 0 : } else if (gst_structure_has_field (structure, "num_tensors")) {
1508 0 : GST_WARNING
1509 : ("Found 'num_tensors' field but format is not described. Suppose tensor format is static.");
1510 0 : format = _NNS_TENSOR_FORMAT_STATIC;
1511 : }
1512 :
1513 13658 : config->info.format = format;
1514 :
1515 13658 : if (config->info.format == _NNS_TENSOR_FORMAT_STATIC) {
1516 9718 : gst_structure_get_int (structure, "num_tensors",
1517 9718 : (gint *) (&config->info.num_tensors));
1518 :
1519 : /* parse dimensions */
1520 9718 : if (gst_structure_has_field (structure, "dimensions")) {
1521 : const gchar *dims_str;
1522 : guint num_dims;
1523 :
1524 8777 : dims_str = gst_structure_get_string (structure, "dimensions");
1525 : num_dims =
1526 8777 : gst_tensors_info_parse_dimensions_string (&config->info, dims_str);
1527 :
1528 8777 : if (config->info.num_tensors != num_dims) {
1529 0 : nns_logw ("Invalid param, dimensions (%d) tensors (%d)\n",
1530 : num_dims, config->info.num_tensors);
1531 : }
1532 : }
1533 :
1534 : /* parse types */
1535 9718 : if (gst_structure_has_field (structure, "types")) {
1536 : const gchar *types_str;
1537 : guint num_types;
1538 :
1539 8871 : types_str = gst_structure_get_string (structure, "types");
1540 : num_types =
1541 8871 : gst_tensors_info_parse_types_string (&config->info, types_str);
1542 :
1543 8871 : if (config->info.num_tensors != num_types) {
1544 0 : nns_logw ("Invalid param, types (%d) tensors (%d)\n",
1545 : num_types, config->info.num_tensors);
1546 : }
1547 : }
1548 : }
1549 : } else {
1550 111 : nns_logw ("Unsupported type = %s\n", name ? name : "Unknown");
1551 111 : return FALSE;
1552 : }
1553 :
1554 159464 : if (gst_structure_has_field (structure, "framerate")) {
1555 159445 : gst_structure_get_fraction (structure, "framerate", &config->rate_n,
1556 159445 : &config->rate_d);
1557 : }
1558 :
1559 159464 : return TRUE;
1560 : }
1561 :
1562 : /**
1563 : * @brief Parse caps and set tensors config (for other/tensors)
1564 : * @param[out] config tensors config structure to be filled
1565 : * @param[in] caps incoming capability
1566 : * @param[in] validate TRUE to validate configuration
1567 : * @return TRUE/FALSE (if successfully configured, return TRUE)
1568 : */
1569 : gboolean
1570 149733 : gst_tensors_config_from_caps (GstTensorsConfig * config, const GstCaps * caps,
1571 : const gboolean validate)
1572 : {
1573 : GstStructure *structure;
1574 149733 : gboolean ret = FALSE;
1575 :
1576 149733 : gst_tensors_config_init (config);
1577 :
1578 149733 : if (validate && !gst_caps_is_fixed (caps)) {
1579 551 : nns_logw ("GstCaps is not fixed.");
1580 551 : return FALSE;
1581 : }
1582 :
1583 149182 : structure = gst_caps_get_structure (caps, 0);
1584 149182 : ret = gst_tensors_config_from_structure (config, structure);
1585 :
1586 149182 : if (ret && validate) {
1587 146298 : ret = gst_tensors_config_validate (config);
1588 : }
1589 :
1590 149182 : if (!ret) {
1591 114 : gst_tensors_config_free (config);
1592 : }
1593 :
1594 149182 : return ret;
1595 : }
1596 :
1597 : /**
1598 : * @brief Parse memory and fill the tensor meta.
1599 : * @param[out] meta tensor meta structure to be filled
1600 : * @param[in] mem pointer to GstMemory to be parsed
1601 : * @return TRUE if successfully set the meta
1602 : */
1603 : gboolean
1604 61029 : gst_tensor_meta_info_parse_memory (GstTensorMetaInfo * meta, GstMemory * mem)
1605 : {
1606 : GstMapInfo map;
1607 : gsize hsize, msize;
1608 : gboolean ret;
1609 :
1610 122057 : g_return_val_if_fail (mem != NULL, FALSE);
1611 61028 : g_return_val_if_fail (meta != NULL, FALSE);
1612 :
1613 61027 : gst_tensor_meta_info_init (meta);
1614 :
1615 : /* Check header size of tensor-meta. */
1616 61027 : hsize = gst_tensor_meta_info_get_header_size (meta);
1617 61027 : msize = gst_memory_get_sizes (mem, NULL, NULL);
1618 61027 : if (msize < hsize)
1619 8479 : return FALSE;
1620 :
1621 52548 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
1622 0 : nns_loge ("Failed to get the meta, cannot map the memory.");
1623 0 : return FALSE;
1624 : }
1625 :
1626 52548 : ret = gst_tensor_meta_info_parse_header (meta, map.data);
1627 :
1628 52548 : gst_memory_unmap (mem, &map);
1629 52548 : return ret;
1630 : }
1631 :
1632 : /**
1633 : * @brief Append header to memory.
1634 : * @param[in] meta tensor meta structure
1635 : * @param[in] mem pointer to GstMemory
1636 : * @return Newly allocated GstMemory (Caller should free returned memory using gst_memory_unref())
1637 : */
1638 : GstMemory *
1639 284 : gst_tensor_meta_info_append_header (GstTensorMetaInfo * meta, GstMemory * mem)
1640 : {
1641 284 : GstMemory *new_mem = NULL;
1642 : gsize msize, hsize;
1643 : GstMapInfo old_map, new_map;
1644 :
1645 567 : g_return_val_if_fail (mem != NULL, NULL);
1646 283 : g_return_val_if_fail (gst_tensor_meta_info_validate (meta), NULL);
1647 :
1648 282 : if (!gst_memory_map (mem, &old_map, GST_MAP_READ)) {
1649 0 : nns_loge ("Failed to append header, cannot map the old memory.");
1650 0 : return NULL;
1651 : }
1652 :
1653 : /* memory size (header + old memory) */
1654 282 : hsize = gst_tensor_meta_info_get_header_size (meta);
1655 282 : msize = hsize + old_map.size;
1656 :
1657 282 : new_mem = gst_allocator_alloc (NULL, msize, NULL);
1658 282 : if (!gst_memory_map (new_mem, &new_map, GST_MAP_WRITE)) {
1659 0 : nns_loge ("Failed to append header, cannot map the new memory.");
1660 0 : gst_memory_unmap (mem, &old_map);
1661 0 : gst_memory_unref (new_mem);
1662 0 : return NULL;
1663 : }
1664 :
1665 : /* set header and copy old data */
1666 282 : gst_tensor_meta_info_update_header (meta, new_map.data);
1667 282 : memcpy (new_map.data + hsize, old_map.data, old_map.size);
1668 :
1669 282 : gst_memory_unmap (mem, &old_map);
1670 282 : gst_memory_unmap (new_mem, &new_map);
1671 282 : return new_mem;
1672 : }
1673 :
1674 : /**
1675 : * @brief Get the nth GstMemory from given @a buffer.
1676 : * @param[in] buffer GstBuffer to be parsed.
1677 : * @param[in] index Index of GstMemory to be returned.
1678 : * @return GstMemory if found, otherwise NULL (Caller should free returned memory using gst_memory_unref()).
1679 : */
1680 : GstMemory *
1681 63217 : gst_tensor_buffer_get_nth_memory (GstBuffer * buffer, const guint index)
1682 : {
1683 : guint i, num_tensors;
1684 : gsize offset;
1685 63217 : GstMemory *extra_tensors_memory, *res_mem = NULL;
1686 : GstMapInfo extra_tensors_map;
1687 : GstTensorExtraInfo *extra_info;
1688 :
1689 63217 : if (!GST_IS_BUFFER (buffer)) {
1690 0 : nns_loge ("Failed to parse GstBuffer (invalid input buffer).");
1691 63217 : return NULL;
1692 : }
1693 :
1694 63217 : num_tensors = gst_tensor_buffer_get_count (buffer);
1695 63217 : if (index >= num_tensors) {
1696 0 : nns_loge ("Invalid index %u, the number of tensors in the buffer is %u.",
1697 : index, num_tensors);
1698 0 : return NULL;
1699 : }
1700 :
1701 : /* If num_tensors is less than or equal to NNS_TENSOR_MEMORY_MAX, it's trivial. */
1702 63217 : if (num_tensors <= NNS_TENSOR_MEMORY_MAX || index < NNS_TENSOR_MEMORY_MAX - 1) {
1703 62596 : return gst_buffer_get_memory (buffer, index);
1704 : }
1705 :
1706 : /* If num_tensors is greater than NNS_TENSOR_MEMORY_MAX, we need to parse extra info. */
1707 : extra_tensors_memory =
1708 621 : gst_buffer_peek_memory (buffer, NNS_TENSOR_MEMORY_MAX - 1);
1709 621 : if (!extra_tensors_memory) {
1710 0 : nns_loge ("Failed to get %d-th memory", NNS_TENSOR_MEMORY_MAX);
1711 0 : return NULL;
1712 : }
1713 :
1714 621 : if (!gst_memory_map (extra_tensors_memory, &extra_tensors_map, GST_MAP_READ)) {
1715 0 : nns_loge ("Failed to map %d-th memory", NNS_TENSOR_MEMORY_MAX);
1716 0 : return NULL;
1717 : }
1718 :
1719 : /* check header (extra info) of the memory */
1720 621 : if (!gst_memory_map_is_extra_tensor (&extra_tensors_map)) {
1721 0 : nns_loge ("Invalid extra header");
1722 0 : goto done;
1723 : }
1724 :
1725 : /* parse the memory */
1726 621 : extra_info = (GstTensorExtraInfo *) extra_tensors_map.data;
1727 621 : offset = sizeof (GstTensorExtraInfo);
1728 :
1729 : /* If index is NNS_TENSOR_MEMORY_MAX - 1 */
1730 621 : if (index == NNS_TENSOR_MEMORY_MAX - 1) {
1731 : res_mem =
1732 29 : gst_memory_share (extra_tensors_memory, offset, extra_info->reserved);
1733 29 : goto done;
1734 : }
1735 :
1736 592 : offset += extra_info->reserved;
1737 :
1738 31720 : for (i = 1; i <= index - NNS_TENSOR_MEMORY_MAX; ++i) {
1739 31128 : offset += gst_tensor_info_get_size (&extra_info->infos[i - 1]);
1740 : }
1741 :
1742 : /* wrap it as GstMemory */
1743 : res_mem =
1744 592 : gst_memory_share (extra_tensors_memory, offset,
1745 592 : gst_tensor_info_get_size (&extra_info->infos[index -
1746 : NNS_TENSOR_MEMORY_MAX]));
1747 :
1748 621 : done:
1749 621 : gst_memory_unmap (extra_tensors_memory, &extra_tensors_map);
1750 621 : return res_mem;
1751 : }
1752 :
1753 : /**
1754 : * @brief Append @a memory to given @a buffer.
1755 : * @param[in/out] buffer GstBuffer to be appended.
1756 : * @param[in] memory GstMemory to append. This function takes ownership of this, even if it returns failure.
1757 : * @param[in] info GstTensorInfo of given @a memory.
1758 : * @return TRUE if successfully appended, otherwise FALSE.
1759 : */
1760 : gboolean
1761 61004 : gst_tensor_buffer_append_memory (GstBuffer * buffer, GstMemory * memory,
1762 : const GstTensorInfo * info)
1763 : {
1764 : guint num_mems, new_mem_index;
1765 61004 : GstMemory *new_memory = NULL, *last_memory = NULL;
1766 : gsize offset, new_mem_size, last_mem_size;
1767 : GstMapInfo new_memory_map, last_memory_map, incoming_memory_map;
1768 : GstTensorExtraInfo *extra_info;
1769 : GstTensorMetaInfo meta;
1770 : gboolean is_extra, is_static;
1771 61004 : gboolean appended = FALSE;
1772 :
1773 61004 : if (!GST_IS_BUFFER (buffer)) {
1774 0 : nns_loge ("Failed to append memory, given buffer is invalid.");
1775 0 : goto failed;
1776 : }
1777 :
1778 61004 : if (!memory) {
1779 0 : nns_loge ("Failed to append memory, given memory is NULL.");
1780 0 : goto failed;
1781 : }
1782 :
1783 61004 : if (gst_tensor_meta_info_parse_memory (&meta, memory)) {
1784 563 : is_static = (meta.format == _NNS_TENSOR_FORMAT_STATIC);
1785 : } else {
1786 : /* Suppose given memory is static tensor. */
1787 60441 : is_static = TRUE;
1788 :
1789 : /* Error case if given tensor-info is invalid. */
1790 60441 : if (!gst_tensor_info_validate (info)) {
1791 0 : nns_loge ("Failed to get tensor info (invalid input info).");
1792 0 : goto failed;
1793 : }
1794 : }
1795 :
1796 61004 : num_mems = gst_buffer_n_memory (buffer);
1797 :
1798 : /* trivial call to gst_buffer_append_memory */
1799 61004 : if (num_mems < NNS_TENSOR_MEMORY_MAX) {
1800 60416 : gst_buffer_append_memory (buffer, memory);
1801 61004 : return TRUE;
1802 : }
1803 :
1804 : /* given buffer has NNS_TENSOR_MEMORY_MAX memory blocks */
1805 588 : last_memory = gst_buffer_peek_memory (buffer, num_mems - 1);
1806 588 : if (!last_memory) {
1807 0 : nns_loge ("Failed to get last memory");
1808 0 : goto failed;
1809 : }
1810 :
1811 588 : if (!gst_memory_map (last_memory, &last_memory_map, GST_MAP_READ)) {
1812 0 : nns_loge ("Failed to map last memory");
1813 0 : last_memory = NULL;
1814 0 : goto failed;
1815 : }
1816 :
1817 588 : new_mem_size = last_mem_size = gst_memory_get_sizes (last_memory, NULL, NULL);
1818 :
1819 : /* if the memory does not have proper header, append it */
1820 588 : is_extra = gst_memory_map_is_extra_tensor (&last_memory_map);
1821 588 : if (!is_extra) {
1822 28 : new_mem_size += sizeof (GstTensorExtraInfo);
1823 : }
1824 :
1825 588 : new_mem_size += gst_memory_get_sizes (memory, NULL, NULL);
1826 :
1827 588 : new_memory = gst_allocator_alloc (NULL, new_mem_size, NULL);
1828 588 : if (!new_memory) {
1829 0 : nns_loge ("Failed to allocate memory for extra tensors.");
1830 0 : goto failed;
1831 : }
1832 :
1833 588 : if (!gst_memory_map (new_memory, &new_memory_map, GST_MAP_WRITE)) {
1834 0 : nns_loge ("Failed to map extra memory");
1835 0 : gst_memory_unref (new_memory);
1836 0 : new_memory = NULL;
1837 0 : goto failed;
1838 : }
1839 :
1840 588 : if (!gst_memory_map (memory, &incoming_memory_map, GST_MAP_READ)) {
1841 0 : nns_loge ("Failed to map incoming memory");
1842 0 : goto failed;
1843 : }
1844 :
1845 588 : extra_info = (GstTensorExtraInfo *) new_memory_map.data;
1846 :
1847 : /* if the last_memory does not have proper header, append it */
1848 588 : if (!is_extra) {
1849 28 : gst_tensor_extra_info_init (extra_info, last_mem_size);
1850 28 : offset = sizeof (GstTensorExtraInfo);
1851 : } else {
1852 560 : offset = 0;
1853 : }
1854 :
1855 : /* copy last_memory into new_memory */
1856 588 : memcpy (new_memory_map.data + offset, last_memory_map.data,
1857 : last_memory_map.size);
1858 :
1859 : /* copy incoming_memory into new_memory */
1860 588 : new_mem_index = extra_info->num_extra_tensors;
1861 588 : extra_info->num_extra_tensors += 1;
1862 :
1863 : /* Copy tensor info into extra. */
1864 588 : if (is_static) {
1865 588 : gst_tensor_info_copy (&extra_info->infos[new_mem_index], info);
1866 :
1867 : /**
1868 : * Free the name string, cause it does not freed by gstreamer.
1869 : * @todo Make custom gst_allocator later?
1870 : */
1871 588 : g_clear_pointer (&extra_info->infos[new_mem_index].name, g_free);
1872 : } else {
1873 0 : gst_tensor_meta_info_convert (&meta, &extra_info->infos[new_mem_index]);
1874 : }
1875 :
1876 588 : memcpy (new_memory_map.data + offset + last_memory_map.size,
1877 588 : incoming_memory_map.data, incoming_memory_map.size);
1878 :
1879 588 : gst_memory_unmap (memory, &incoming_memory_map);
1880 588 : gst_memory_unmap (last_memory, &last_memory_map);
1881 588 : last_memory = NULL;
1882 :
1883 588 : gst_buffer_replace_memory (buffer, num_mems - 1, new_memory);
1884 588 : appended = TRUE;
1885 :
1886 588 : failed:
1887 588 : if (new_memory) {
1888 588 : gst_memory_unmap (new_memory, &new_memory_map);
1889 588 : if (!appended)
1890 0 : gst_memory_unref (new_memory);
1891 : }
1892 :
1893 588 : if (last_memory)
1894 0 : gst_memory_unmap (last_memory, &last_memory_map);
1895 :
1896 : /* Release incoming memory even if failed to append it into buffer. */
1897 588 : if (memory)
1898 588 : gst_memory_unref (memory);
1899 :
1900 588 : return appended;
1901 : }
1902 :
1903 : /**
1904 : * @brief Get the number of tensors in the buffer.
1905 : */
1906 : guint
1907 123736 : gst_tensor_buffer_get_count (GstBuffer * buffer)
1908 : {
1909 : guint num_mems;
1910 : GstMemory *mem;
1911 : GstMapInfo map;
1912 : GstTensorExtraInfo *extra_info;
1913 :
1914 247472 : g_return_val_if_fail (buffer != NULL, 0);
1915 :
1916 123736 : num_mems = gst_buffer_n_memory (buffer);
1917 123736 : if (num_mems < NNS_TENSOR_MEMORY_MAX) {
1918 122035 : return num_mems;
1919 : }
1920 :
1921 : /* num_mems == NNS_TENSOR_MEMORY_MAX */
1922 1701 : mem = gst_buffer_peek_memory (buffer, num_mems - 1);
1923 1701 : if (!mem) {
1924 0 : nns_loge ("Failed to get the last memory.");
1925 0 : return 0;
1926 : }
1927 :
1928 1701 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
1929 0 : nns_loge ("Failed to map the last memory.");
1930 0 : return 0;
1931 : }
1932 :
1933 1701 : if (gst_memory_map_is_extra_tensor (&map)) {
1934 1349 : extra_info = (GstTensorExtraInfo *) map.data;
1935 1349 : num_mems = extra_info->num_extra_tensors + NNS_TENSOR_MEMORY_MAX;
1936 : } else {
1937 352 : nns_logi ("The last memory does not have extra tensors header. "
1938 : "Assuming the number of tensors is %d.", num_mems);
1939 : }
1940 :
1941 1701 : gst_memory_unmap (mem, &map);
1942 :
1943 1701 : return num_mems;
1944 : }
1945 :
1946 : /**
1947 : * @brief Sets the value of a property based on the specified property value and GParamSpec.
1948 : *
1949 : * @param prop_value A pointer to the GValue where the property value will be set.
1950 : * @param param_spec A pointer to the GParamSpec that describes the property.
1951 : * @param property_value A string representing the value to be set for the property.
1952 : *
1953 : * @note This API is intended to be used by gst_tensor_parse_config_file ()
1954 : */
1955 : static void
1956 34 : set_property_value (GValue * prop_value, const GParamSpec * param_spec,
1957 : const gchar * property_value)
1958 : {
1959 34 : GType value_type = G_PARAM_SPEC_VALUE_TYPE (param_spec);
1960 :
1961 34 : g_value_init (prop_value, value_type);
1962 :
1963 34 : if (value_type == G_TYPE_BOOLEAN) {
1964 0 : gboolean value = g_ascii_strcasecmp (property_value, "true") == 0;
1965 0 : g_value_set_boolean (prop_value, value);
1966 34 : } else if (value_type == G_TYPE_INT) {
1967 0 : gint value = atoi (property_value);
1968 0 : g_value_set_int (prop_value, value);
1969 34 : } else if (value_type == G_TYPE_UINT) {
1970 0 : guint value = atoi (property_value);
1971 0 : g_value_set_uint (prop_value, value);
1972 34 : } else if (value_type == G_TYPE_FLOAT) {
1973 0 : gfloat value = atof (property_value);
1974 0 : g_value_set_float (prop_value, value);
1975 34 : } else if (value_type == G_TYPE_DOUBLE) {
1976 0 : gdouble value = atof (property_value);
1977 0 : g_value_set_double (prop_value, value);
1978 : } else {
1979 34 : g_value_set_string (prop_value, property_value); /** default is string */
1980 : }
1981 34 : }
1982 :
1983 : /**
1984 : * @brief Parses a configuration file and sets the corresponding properties on a GObject.
1985 : *
1986 : * This function reads the contents of the configuration file located at the given path
1987 : * and sets the properties of the specified GObject based on the configuration data.
1988 : *
1989 : * @param config_path The path to the configuration file.
1990 : * @param object The GObject on which to set the properties.
1991 : *
1992 : * @note The responsibility of managing the memory of the GObject passed as a parameter
1993 : * lies outside this function.
1994 : */
1995 : void
1996 9 : gst_tensor_parse_config_file (const gchar * config_path, const GObject * object)
1997 : {
1998 9 : g_autofree gchar *config_data = NULL;
1999 9 : g_auto (GStrv) lines = NULL;
2000 9 : GStrv line = NULL;
2001 9 : GError *error = NULL;
2002 9 : GObjectClass *g_object_class = G_OBJECT_GET_CLASS (object);
2003 :
2004 9 : if (!g_file_get_contents (config_path, &config_data, NULL, &error)) {
2005 0 : GST_DEBUG ("Failed to read config file: %s\n", error->message);
2006 0 : g_error_free (error);
2007 0 : return;
2008 : }
2009 :
2010 9 : lines = g_strsplit (config_data, "\n", -1);
2011 :
2012 : /** Iterate over each line */
2013 52 : for (line = lines; *line; ++line) {
2014 43 : g_auto (GStrv) parts = g_strsplit (*line, "=", 2);
2015 :
2016 43 : if (g_strv_length (parts) == 2) {
2017 68 : g_autofree gchar *property_name = g_strstrip (g_strdup (parts[0]));
2018 68 : g_autofree gchar *property_value = g_strstrip (g_strdup (parts[1]));
2019 :
2020 : GParamSpec *pdata =
2021 34 : g_object_class_find_property (g_object_class, property_name);
2022 :
2023 34 : if (pdata != NULL) {
2024 34 : GValue prop_value = G_VALUE_INIT;
2025 34 : set_property_value (&prop_value, pdata, property_value);
2026 34 : g_object_set_property (G_OBJECT (object), pdata->name, &prop_value);
2027 34 : g_value_unset (&prop_value);
2028 : }
2029 : }
2030 : }
2031 : }
|
