Line data Source code
1 : /**
2 : * GStreamer
3 : * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
4 : * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
5 : * Copyright (C) 2018 Samsung Electronics Co., Ltd.
6 : *
7 : * This library is free software; you can redistribute it and/or
8 : * modify it under the terms of the GNU Library General Public
9 : * License as published by the Free Software Foundation;
10 : * version 2.1 of the License.
11 : *
12 : * This library is distributed in the hope that it will be useful,
13 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : * Library General Public License for more details.
16 : */
17 :
18 : /**
19 : * SECTION:element-tensor_aggregator
20 : *
21 : * @file gsttensor_aggregator.c
22 : * @date 29 August 2018
23 : * @brief GStreamer plugin to aggregate tensor stream
24 : * @see https://github.com/nnstreamer/nnstreamer
25 : * @author Jaeyun Jung <jy1210.jung@samsung.com>
26 : * @bug No known bugs except for NYI items
27 : */
28 :
29 : #ifdef HAVE_CONFIG_H
30 : #include <config.h>
31 : #endif
32 :
33 : #include <string.h>
34 : #include "gsttensor_aggregator.h"
35 : #include "tensor_meta.h"
36 : #include <nnstreamer_util.h>
37 :
38 : /**
39 : * @brief Macro for debug mode.
40 : */
41 : #ifndef DBG
42 : #define DBG (!self->silent)
43 : #endif
44 :
45 : #define silent_debug_config(self,c,msg) do { \
46 : if (DBG) { \
47 : if (c) { \
48 : gchar *dim_str; \
49 : dim_str = gst_tensor_get_dimension_string ((c)->info.info[0].dimension); \
50 : GST_DEBUG_OBJECT (self, msg " type=%d dim=%s rate=%d/%d", (c)->info.info[0].type, dim_str, (c)->rate_n, (c)->rate_d); \
51 : g_free (dim_str); \
52 : } \
53 : } \
54 : } while (0)
55 :
56 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_aggregator_debug);
57 : #define GST_CAT_DEFAULT gst_tensor_aggregator_debug
58 :
59 : /**
60 : * @brief tensor_aggregator properties
61 : */
62 : enum
63 : {
64 : PROP_0,
65 : PROP_FRAMES_IN,
66 : PROP_FRAMES_OUT,
67 : PROP_FRAMES_FLUSH,
68 : PROP_FRAMES_DIMENSION,
69 : PROP_CONCAT,
70 : PROP_SILENT
71 : };
72 :
73 : /**
74 : * @brief Flag to print minimized log.
75 : */
76 : #define DEFAULT_SILENT TRUE
77 :
78 : /**
79 : * @brief The number of frames in input buffer.
80 : */
81 : #define DEFAULT_FRAMES_IN 1
82 :
83 : /**
84 : * @brief The number of frames in output buffer.
85 : */
86 : #define DEFAULT_FRAMES_OUT 1
87 :
88 : /**
89 : * @brief The number of frames to flush.
90 : */
91 : #define DEFAULT_FRAMES_FLUSH 0
92 :
93 : /**
94 : * @brief The dimension index of frames in configured tensor.
95 : */
96 : #define DEFAULT_FRAMES_DIMENSION (NNS_TENSOR_RANK_LIMIT - 1)
97 :
98 : /**
99 : * @brief Flag to concatenate output buffer.
100 : */
101 : #define DEFAULT_CONCAT TRUE
102 :
103 : /**
104 : * @brief Template caps string for pads.
105 : */
106 : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_WITH_NUM ("1")
107 :
108 : /**
109 : * @brief Template for sink pad.
110 : */
111 : static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
112 : GST_PAD_SINK,
113 : GST_PAD_ALWAYS,
114 : GST_STATIC_CAPS (CAPS_STRING));
115 :
116 : /**
117 : * @brief Template for src pad.
118 : */
119 : static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
120 : GST_PAD_SRC,
121 : GST_PAD_ALWAYS,
122 : GST_STATIC_CAPS (CAPS_STRING));
123 :
124 : #define gst_tensor_aggregator_parent_class parent_class
125 1409 : G_DEFINE_TYPE (GstTensorAggregator, gst_tensor_aggregator, GST_TYPE_ELEMENT);
126 :
127 : static void gst_tensor_aggregator_finalize (GObject * object);
128 : static void gst_tensor_aggregator_set_property (GObject * object,
129 : guint prop_id, const GValue * value, GParamSpec * pspec);
130 : static void gst_tensor_aggregator_get_property (GObject * object,
131 : guint prop_id, GValue * value, GParamSpec * pspec);
132 :
133 : static gboolean gst_tensor_aggregator_sink_event (GstPad * pad,
134 : GstObject * parent, GstEvent * event);
135 : static gboolean gst_tensor_aggregator_sink_query (GstPad * pad,
136 : GstObject * parent, GstQuery * query);
137 : static gboolean gst_tensor_aggregator_src_query (GstPad * pad,
138 : GstObject * parent, GstQuery * query);
139 : static GstFlowReturn gst_tensor_aggregator_chain (GstPad * pad,
140 : GstObject * parent, GstBuffer * buf);
141 : static GstStateChangeReturn
142 : gst_tensor_aggregator_change_state (GstElement * element,
143 : GstStateChange transition);
144 :
145 : static void gst_tensor_aggregator_reset (GstTensorAggregator * self);
146 : static GstCaps *gst_tensor_aggregator_query_caps (GstTensorAggregator * self,
147 : GstPad * pad, GstCaps * filter);
148 : static gboolean gst_tensor_aggregator_parse_caps (GstTensorAggregator * self,
149 : const GstCaps * caps);
150 :
151 : /**
152 : * @brief Initialize the tensor_aggregator's class.
153 : */
154 : static void
155 2 : gst_tensor_aggregator_class_init (GstTensorAggregatorClass * klass)
156 : {
157 : GObjectClass *object_class;
158 : GstElementClass *element_class;
159 :
160 2 : GST_DEBUG_CATEGORY_INIT (gst_tensor_aggregator_debug, "tensor_aggregator", 0,
161 : "Element to aggregate tensor stream");
162 :
163 2 : object_class = (GObjectClass *) klass;
164 2 : element_class = (GstElementClass *) klass;
165 :
166 2 : object_class->set_property = gst_tensor_aggregator_set_property;
167 2 : object_class->get_property = gst_tensor_aggregator_get_property;
168 2 : object_class->finalize = gst_tensor_aggregator_finalize;
169 :
170 : /**
171 : * GstTensorAggregator::frames-in:
172 : *
173 : * The number of frames in incoming buffer.
174 : * GstTensorAggregator itself cannot get the frames in buffer. (buffer is a single tensor instance)
175 : * GstTensorAggregator calculates the size of single frame with this property.
176 : */
177 2 : g_object_class_install_property (object_class, PROP_FRAMES_IN,
178 : g_param_spec_uint ("frames-in", "Frames in input",
179 : "The number of frames in incoming buffer", 1, G_MAXUINT,
180 : DEFAULT_FRAMES_IN, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
181 :
182 : /**
183 : * GstTensorAggregator::frames-out:
184 : *
185 : * The number of frames in outgoing buffer. (buffer is a single tensor instance)
186 : * GstTensorAggregator calculates the size of outgoing frames and pushes a buffer to source pad.
187 : */
188 2 : g_object_class_install_property (object_class, PROP_FRAMES_OUT,
189 : g_param_spec_uint ("frames-out", "Frames in output",
190 : "The number of frames in outgoing buffer", 1, G_MAXUINT,
191 : DEFAULT_FRAMES_OUT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
192 :
193 : /**
194 : * GstTensorAggregator::frames-flush:
195 : *
196 : * The number of frames to flush.
197 : * GstTensorAggregator flushes the bytes (N frames) in GstAdapter after pushing a buffer.
198 : * If set 0 (default value), all outgoing frames will be flushed.
199 : */
200 2 : g_object_class_install_property (object_class, PROP_FRAMES_FLUSH,
201 : g_param_spec_uint ("frames-flush", "Frames to flush",
202 : "The number of frames to flush (0 to flush all output)", 0, G_MAXUINT,
203 : DEFAULT_FRAMES_FLUSH, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
204 :
205 : /**
206 : * GstTensorAggregator::frames-dim:
207 : *
208 : * The dimension index of frames in tensor.
209 : * If frames-in and frames-out are different, GstTensorAggregator has to change the dimension of tensor.
210 : * With this property, GstTensorAggregator changes the out-caps.
211 : */
212 2 : g_object_class_install_property (object_class, PROP_FRAMES_DIMENSION,
213 : g_param_spec_uint ("frames-dim", "Dimension index of frames",
214 : "The dimension index of frames in tensor",
215 : 0, (NNS_TENSOR_RANK_LIMIT - 1), DEFAULT_FRAMES_DIMENSION,
216 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
217 :
218 : /**
219 : * GstTensorAggregator::concat:
220 : *
221 : * The flag to concatenate output buffer.
222 : * If concat is true and frames-out is larger than 1, GstTensorAggregator will concatenate the output buffer with the axis frames-dim.
223 : */
224 2 : g_object_class_install_property (object_class, PROP_CONCAT,
225 : g_param_spec_boolean ("concat", "Concat", "Concatenate output buffer",
226 : DEFAULT_CONCAT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
227 :
228 : /**
229 : * GstTensorAggregator::silent:
230 : *
231 : * The flag to enable/disable debugging messages.
232 : */
233 2 : g_object_class_install_property (object_class, PROP_SILENT,
234 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
235 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
236 :
237 2 : gst_element_class_set_static_metadata (element_class,
238 : "TensorAggregator",
239 : "Filter/Tensor",
240 : "Element to aggregate tensor stream", "Samsung Electronics Co., Ltd.");
241 :
242 2 : gst_element_class_add_pad_template (element_class,
243 : gst_static_pad_template_get (&src_template));
244 2 : gst_element_class_add_pad_template (element_class,
245 : gst_static_pad_template_get (&sink_template));
246 :
247 2 : element_class->change_state = gst_tensor_aggregator_change_state;
248 2 : }
249 :
250 : /**
251 : * @brief Initialize tensor_aggregator element.
252 : */
253 : static void
254 19 : gst_tensor_aggregator_init (GstTensorAggregator * self)
255 : {
256 : /** setup sink pad */
257 19 : self->sinkpad = gst_pad_new_from_static_template (&sink_template, "sink");
258 19 : gst_pad_set_event_function (self->sinkpad,
259 : GST_DEBUG_FUNCPTR (gst_tensor_aggregator_sink_event));
260 19 : gst_pad_set_query_function (self->sinkpad,
261 : GST_DEBUG_FUNCPTR (gst_tensor_aggregator_sink_query));
262 19 : gst_pad_set_chain_function (self->sinkpad,
263 : GST_DEBUG_FUNCPTR (gst_tensor_aggregator_chain));
264 19 : GST_PAD_SET_PROXY_CAPS (self->sinkpad);
265 19 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);
266 :
267 : /** setup src pad */
268 19 : self->srcpad = gst_pad_new_from_static_template (&src_template, "src");
269 19 : gst_pad_set_query_function (self->srcpad,
270 : GST_DEBUG_FUNCPTR (gst_tensor_aggregator_src_query));
271 19 : GST_PAD_SET_PROXY_CAPS (self->srcpad);
272 19 : gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
273 :
274 : /** init properties */
275 19 : self->silent = DEFAULT_SILENT;
276 19 : self->frames_in = DEFAULT_FRAMES_IN;
277 19 : self->frames_out = DEFAULT_FRAMES_OUT;
278 19 : self->frames_flush = DEFAULT_FRAMES_FLUSH;
279 19 : self->frames_dim = DEFAULT_FRAMES_DIMENSION;
280 19 : self->concat = DEFAULT_CONCAT;
281 :
282 19 : self->tensor_configured = FALSE;
283 19 : gst_tensors_config_init (&self->in_config);
284 19 : gst_tensors_config_init (&self->out_config);
285 :
286 19 : self->adapter_table = gst_tensor_aggregation_init ();
287 19 : gst_tensor_aggregator_reset (self);
288 19 : }
289 :
290 : /**
291 : * @brief Function to finalize instance.
292 : */
293 : static void
294 19 : gst_tensor_aggregator_finalize (GObject * object)
295 : {
296 : GstTensorAggregator *self;
297 :
298 19 : self = GST_TENSOR_AGGREGATOR (object);
299 :
300 19 : gst_tensor_aggregator_reset (self);
301 :
302 19 : gst_tensors_config_free (&self->in_config);
303 19 : gst_tensors_config_free (&self->out_config);
304 19 : g_hash_table_destroy (self->adapter_table);
305 :
306 19 : G_OBJECT_CLASS (parent_class)->finalize (object);
307 19 : }
308 :
309 : /**
310 : * @brief Setter for tensor_aggregator properties.
311 : */
312 : static void
313 51 : gst_tensor_aggregator_set_property (GObject * object, guint prop_id,
314 : const GValue * value, GParamSpec * pspec)
315 : {
316 : GstTensorAggregator *self;
317 :
318 51 : self = GST_TENSOR_AGGREGATOR (object);
319 :
320 51 : switch (prop_id) {
321 6 : case PROP_FRAMES_IN:
322 6 : self->frames_in = g_value_get_uint (value);
323 6 : break;
324 18 : case PROP_FRAMES_OUT:
325 18 : self->frames_out = g_value_get_uint (value);
326 18 : break;
327 5 : case PROP_FRAMES_FLUSH:
328 5 : self->frames_flush = g_value_get_uint (value);
329 5 : break;
330 18 : case PROP_FRAMES_DIMENSION:
331 18 : self->frames_dim = g_value_get_uint (value);
332 18 : break;
333 3 : case PROP_CONCAT:
334 3 : self->concat = g_value_get_boolean (value);
335 3 : break;
336 1 : case PROP_SILENT:
337 1 : self->silent = g_value_get_boolean (value);
338 1 : break;
339 0 : default:
340 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
341 0 : break;
342 : }
343 51 : }
344 :
345 : /**
346 : * @brief Getter for tensor_aggregator properties.
347 : */
348 : static void
349 12 : gst_tensor_aggregator_get_property (GObject * object, guint prop_id,
350 : GValue * value, GParamSpec * pspec)
351 : {
352 : GstTensorAggregator *self;
353 :
354 12 : self = GST_TENSOR_AGGREGATOR (object);
355 :
356 12 : switch (prop_id) {
357 2 : case PROP_FRAMES_IN:
358 2 : g_value_set_uint (value, self->frames_in);
359 2 : break;
360 2 : case PROP_FRAMES_OUT:
361 2 : g_value_set_uint (value, self->frames_out);
362 2 : break;
363 2 : case PROP_FRAMES_FLUSH:
364 2 : g_value_set_uint (value, self->frames_flush);
365 2 : break;
366 2 : case PROP_FRAMES_DIMENSION:
367 2 : g_value_set_uint (value, self->frames_dim);
368 2 : break;
369 2 : case PROP_CONCAT:
370 2 : g_value_set_boolean (value, self->concat);
371 2 : break;
372 2 : case PROP_SILENT:
373 2 : g_value_set_boolean (value, self->silent);
374 2 : break;
375 0 : default:
376 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
377 0 : break;
378 : }
379 12 : }
380 :
381 : /**
382 : * @brief This function handles sink events.
383 : */
384 : static gboolean
385 64 : gst_tensor_aggregator_sink_event (GstPad * pad, GstObject * parent,
386 : GstEvent * event)
387 : {
388 : GstTensorAggregator *self;
389 :
390 64 : self = GST_TENSOR_AGGREGATOR (parent);
391 :
392 64 : GST_DEBUG_OBJECT (self, "Received %s event: %" GST_PTR_FORMAT,
393 : GST_EVENT_TYPE_NAME (event), event);
394 :
395 64 : switch (GST_EVENT_TYPE (event)) {
396 18 : case GST_EVENT_CAPS:
397 : {
398 : GstCaps *in_caps;
399 : GstCaps *out_caps;
400 :
401 18 : gst_event_parse_caps (event, &in_caps);
402 18 : silent_debug_caps (self, in_caps, "in-caps");
403 :
404 18 : if (gst_tensor_aggregator_parse_caps (self, in_caps)) {
405 18 : gboolean ret = FALSE;
406 :
407 : out_caps =
408 18 : gst_tensor_pad_caps_from_config (self->srcpad, &self->out_config);
409 18 : silent_debug_caps (self, out_caps, "out-caps");
410 :
411 18 : ret = gst_pad_set_caps (self->srcpad, out_caps);
412 :
413 18 : gst_event_unref (event);
414 18 : gst_caps_unref (out_caps);
415 :
416 18 : return ret;
417 : }
418 0 : break;
419 : }
420 1 : case GST_EVENT_FLUSH_STOP:
421 1 : gst_tensor_aggregator_reset (self);
422 1 : break;
423 45 : default:
424 45 : break;
425 : }
426 :
427 46 : return gst_pad_event_default (pad, parent, event);
428 : }
429 :
430 : /**
431 : * @brief This function handles sink pad query.
432 : */
433 : static gboolean
434 76 : gst_tensor_aggregator_sink_query (GstPad * pad, GstObject * parent,
435 : GstQuery * query)
436 : {
437 : GstTensorAggregator *self;
438 :
439 76 : self = GST_TENSOR_AGGREGATOR (parent);
440 :
441 76 : GST_DEBUG_OBJECT (self, "Received %s query: %" GST_PTR_FORMAT,
442 : GST_QUERY_TYPE_NAME (query), query);
443 :
444 76 : switch (GST_QUERY_TYPE (query)) {
445 41 : case GST_QUERY_CAPS:
446 : {
447 : GstCaps *caps;
448 : GstCaps *filter;
449 :
450 41 : gst_query_parse_caps (query, &filter);
451 41 : caps = gst_tensor_aggregator_query_caps (self, pad, filter);
452 :
453 41 : gst_query_set_caps_result (query, caps);
454 41 : gst_caps_unref (caps);
455 41 : return TRUE;
456 : }
457 23 : case GST_QUERY_ACCEPT_CAPS:
458 : {
459 : GstCaps *caps;
460 : GstCaps *template_caps;
461 23 : gboolean res = FALSE;
462 :
463 23 : gst_query_parse_accept_caps (query, &caps);
464 23 : silent_debug_caps (self, caps, "accept-caps");
465 :
466 23 : if (gst_caps_is_fixed (caps)) {
467 23 : template_caps = gst_pad_get_pad_template_caps (pad);
468 :
469 23 : res = gst_caps_can_intersect (template_caps, caps);
470 23 : gst_caps_unref (template_caps);
471 : }
472 :
473 23 : gst_query_set_accept_caps_result (query, res);
474 23 : return TRUE;
475 : }
476 12 : default:
477 12 : break;
478 : }
479 :
480 12 : return gst_pad_query_default (pad, parent, query);
481 : }
482 :
483 : /**
484 : * @brief This function handles src pad query.
485 : */
486 : static gboolean
487 36 : gst_tensor_aggregator_src_query (GstPad * pad, GstObject * parent,
488 : GstQuery * query)
489 : {
490 : GstTensorAggregator *self;
491 :
492 36 : self = GST_TENSOR_AGGREGATOR (parent);
493 :
494 36 : GST_DEBUG_OBJECT (self, "Received %s query: %" GST_PTR_FORMAT,
495 : GST_QUERY_TYPE_NAME (query), query);
496 :
497 36 : switch (GST_QUERY_TYPE (query)) {
498 30 : case GST_QUERY_CAPS:
499 : {
500 : GstCaps *caps;
501 : GstCaps *filter;
502 :
503 30 : gst_query_parse_caps (query, &filter);
504 30 : caps = gst_tensor_aggregator_query_caps (self, pad, filter);
505 :
506 30 : gst_query_set_caps_result (query, caps);
507 30 : gst_caps_unref (caps);
508 30 : return TRUE;
509 : }
510 6 : default:
511 6 : break;
512 : }
513 :
514 6 : return gst_pad_query_default (pad, parent, query);
515 : }
516 :
517 : /**
518 : * @brief Internal function to get adapter.
519 : */
520 : static GstAdapter *
521 169 : gst_tensor_aggregator_get_adapter (GstTensorAggregator * self, GstBuffer * buf)
522 : {
523 : GstMetaQuery *meta;
524 169 : guint32 key = 0;
525 :
526 169 : meta = gst_buffer_get_meta_query (buf);
527 169 : if (meta)
528 4 : key = meta->client_id;
529 :
530 169 : return gst_tensor_aggregation_get_adapter (self->adapter_table, key);
531 : }
532 :
533 : /**
534 : * @brief Check tensor dimension and axis to concatenate data.
535 : * @param self this pointer to GstTensorAggregator
536 : * @param info tensor info for one frame
537 : * @return True if needed to concatenate
538 : */
539 : static gboolean
540 90 : gst_tensor_aggregator_check_concat_axis (GstTensorAggregator * self,
541 : const GstTensorInfo * info)
542 : {
543 : guint i;
544 :
545 90 : g_assert (info != NULL); /** Internal error. Caller should've checked it */
546 :
547 : /**
548 : * Check condition to concatenate data.
549 : */
550 90 : if (self->concat && self->frames_out > 1) {
551 1009 : for (i = self->frames_dim + 1; i < NNS_TENSOR_RANK_LIMIT; i++) {
552 942 : if (info->dimension[i] > 1) {
553 : /** concatenate data */
554 17 : return TRUE;
555 : }
556 : }
557 : }
558 :
559 73 : return FALSE;
560 : }
561 :
562 : /**
563 : * @brief Change the data in buffer with given axis.
564 : * @param self this pointer to GstTensorAggregator
565 : * @param outbuf buffer to be concatenated
566 : * @param info tensor info for one frame
567 : */
568 : static gboolean
569 17 : gst_tensor_aggregator_concat (GstTensorAggregator * self, GstBuffer * outbuf,
570 : const GstTensorInfo * info)
571 : {
572 : GstBuffer *srcbuf;
573 : GstMapInfo src_info, dest_info;
574 : guint f;
575 : gsize block_size;
576 : gsize src_idx, dest_idx;
577 : gsize frame_size;
578 :
579 17 : frame_size = gst_tensor_info_get_size (info);
580 17 : g_assert (frame_size > 0); /** Internal error */
581 :
582 17 : srcbuf = gst_buffer_copy (outbuf);
583 17 : outbuf = gst_buffer_make_writable (outbuf);
584 :
585 17 : if (!gst_buffer_map (srcbuf, &src_info, GST_MAP_READ)) {
586 0 : ml_logf ("Failed to map source buffer with tensor_aggregator.\n");
587 0 : gst_buffer_unref (srcbuf);
588 17 : return FALSE;
589 : }
590 17 : if (!gst_buffer_map (outbuf, &dest_info, GST_MAP_WRITE)) {
591 0 : ml_logf ("Failed to map destination buffer with tensor_aggregator.\n");
592 0 : gst_buffer_unmap (srcbuf, &src_info);
593 0 : gst_buffer_unref (srcbuf);
594 0 : return FALSE;
595 : }
596 :
597 : /**
598 : * Concatenate output buffer with given axis (frames-dim)
599 : * If frames-dim is equal to (NNS_TENSOR_RANK_LIMIT - 1), nothing to do.
600 : * (In this case, this function will not be called. See gst_tensor_aggregator_check_concat_axis ())
601 : ********************************************************************
602 : * Ex1) concatenate 2 frames with dimension 3:4:2:1
603 : ********************************************************************
604 : * frame 1
605 : * [
606 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ]
607 : * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ]
608 : * ]
609 : *
610 : * frame 2
611 : * [
612 : * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
613 : * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ]
614 : * ]
615 : ********************************************************************
616 : * 1-1. result with frames-dim 3 (3:4:2:2)
617 : * [
618 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ]
619 : * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ]
620 : * ]
621 : * [
622 : * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
623 : * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ]
624 : * ]
625 : ********************************************************************
626 : * 1-2. result with frames-dim 2 (3:4:4:1)
627 : * [
628 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ]
629 : * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ]
630 : * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
631 : * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ]
632 : * ]
633 : ********************************************************************
634 : * 1-3. result with frames-dim 1 (3:8:2:1)
635 : * [
636 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112]
637 : * [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
638 : * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124]
639 : * [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ]
640 : * ]
641 : ********************************************************************
642 : * 1-4. result with frames-dim 0 (6:4:2:1)
643 : * [
644 : * [ [1101 1102 1103 2101 2102 2103] [1104 1105 1106 2104 2105 2106]
645 : * [1107 1108 1109 2107 2108 2109] [1110 1111 1112 2110 2111 2112] ]
646 : * [ [1113 1114 1115 2113 2114 2115] [1116 1117 1118 2116 2117 2118]
647 : * [1119 1120 1121 2119 2120 2121] [1122 1123 1124 2122 2123 2124] ]
648 : * ]
649 : ********************************************************************
650 : * Ex2) concatenate 2 frames with dimension 3:4:2:2
651 : ********************************************************************
652 : * frame 1
653 : * [
654 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ]
655 : * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ]
656 : * ]
657 : * [
658 : * [ [1201 1202 1203] [1204 1205 1206] [1207 1208 1209] [1210 1211 1212] ]
659 : * [ [1213 1214 1215] [1216 1217 1218] [1219 1220 1221] [1222 1223 1224] ]
660 : * ]
661 : *
662 : * frame 2
663 : * [
664 : * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
665 : * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ]
666 : * ]
667 : * [
668 : * [ [2201 2202 2203] [2204 2205 2206] [2207 2208 2209] [2210 2211 2212] ]
669 : * [ [2213 2214 2215] [2216 2217 2218] [2219 2220 2221] [2222 2223 2224] ]
670 : * ]
671 : ********************************************************************
672 : * 2-1. result with frames-dim 3 (3:4:2:4)
673 : * [
674 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ]
675 : * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ]
676 : * ]
677 : * [
678 : * [ [1201 1202 1203] [1204 1205 1206] [1207 1208 1209] [1210 1211 1212] ]
679 : * [ [1213 1214 1215] [1216 1217 1218] [1219 1220 1221] [1222 1223 1224] ]
680 : * ]
681 : * [
682 : * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
683 : * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ]
684 : * ]
685 : * [
686 : * [ [2201 2202 2203] [2204 2205 2206] [2207 2208 2209] [2210 2211 2212] ]
687 : * [ [2213 2214 2215] [2216 2217 2218] [2219 2220 2221] [2222 2223 2224] ]
688 : * ]
689 : ********************************************************************
690 : * 2-2. result with frames-dim 2 (3:4:4:2)
691 : * [
692 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ]
693 : * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124] ]
694 : * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
695 : * [ [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ]
696 : * ]
697 : * [
698 : * [ [1201 1202 1203] [1204 1205 1206] [1207 1208 1209] [1210 1211 1212] ]
699 : * [ [1213 1214 1215] [1216 1217 1218] [1219 1220 1221] [1222 1223 1224] ]
700 : * [ [2201 2202 2203] [2204 2205 2206] [2207 2208 2209] [2210 2211 2212] ]
701 : * [ [2213 2214 2215] [2216 2217 2218] [2219 2220 2221] [2222 2223 2224] ]
702 : * ]
703 : ********************************************************************
704 : * 2-3. result with frames-dim 1 (3:8:2:2)
705 : * [
706 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112]
707 : * [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
708 : * [ [1113 1114 1115] [1116 1117 1118] [1119 1120 1121] [1122 1123 1124]
709 : * [2113 2114 2115] [2116 2117 2118] [2119 2120 2121] [2122 2123 2124] ]
710 : * ]
711 : * [
712 : * [ [1201 1202 1203] [1204 1205 1206] [1207 1208 1209] [1210 1211 1212]
713 : * [2201 2202 2203] [2204 2205 2206] [2207 2208 2209] [2210 2211 2212] ]
714 : * [ [1213 1214 1215] [1216 1217 1218] [1219 1220 1221] [1222 1223 1224]
715 : * [2213 2214 2215] [2216 2217 2218] [2219 2220 2221] [2222 2223 2224] ]
716 : * ]
717 : ********************************************************************
718 : * 2-4. result with frames-dim 0 (6:4:2:2)
719 : * [
720 : * [ [1101 1102 1103 2101 2102 2103] [1104 1105 1106 2104 2105 2106]
721 : * [1107 1108 1109 2107 2108 2109] [1110 1111 1112 2110 2111 2112] ]
722 : * [ [1113 1114 1115 2113 2114 2115] [1116 1117 1118 2116 2117 2118]
723 : * [1119 1120 1121 2119 2120 2121] [1122 1123 1124 2122 2123 2124] ]
724 : * ]
725 : * [
726 : * [ [1201 1202 1203 2201 2202 2203] [1204 1205 1206 2204 2205 2206]
727 : * [1207 1208 1209 2207 2208 2209] [1210 1211 1212 2210 2211 2212] ]
728 : * [ [1213 1214 1215 2213 2214 2215] [1216 1217 1218 2216 2217 2218]
729 : * [1219 1220 1221 2219 2220 2221] [1222 1223 1224 2222 2223 2224] ]
730 : * ]
731 : ********************************************************************
732 : * Ex3) concatenate 2 frames with dimension 3:4:1:1
733 : ********************************************************************
734 : * frame 1
735 : * [
736 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ]
737 : * ]
738 : *
739 : * frame 2
740 : * [
741 : * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
742 : * ]
743 : ********************************************************************
744 : * 3-1. result with frames-dim 3 (3:4:1:2)
745 : * [
746 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ]
747 : * ]
748 : * [
749 : * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
750 : * ]
751 : ********************************************************************
752 : * 3-2. result with frames-dim 2 (3:4:2:1)
753 : * [
754 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112] ]
755 : * [ [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
756 : * ]
757 : ********************************************************************
758 : * 3-3. result with frames-dim 1 (3:8:1:1)
759 : * [
760 : * [ [1101 1102 1103] [1104 1105 1106] [1107 1108 1109] [1110 1111 1112]
761 : * [2101 2102 2103] [2104 2105 2106] [2107 2108 2109] [2110 2111 2112] ]
762 : * ]
763 : ********************************************************************
764 : * 3-4. result with frames-dim 0 (6:4:1:1)
765 : * [
766 : * [ [1101 1102 1103 2101 2102 2103] [1104 1105 1106 2104 2105 2106]
767 : * [1107 1108 1109 2107 2108 2109] [1110 1111 1112 2110 2111 2112] ]
768 : * ]
769 : ********************************************************************
770 : */
771 :
772 : /** get block size */
773 17 : block_size = gst_tensor_get_element_size (info->type);
774 53 : for (f = 0; f <= self->frames_dim; f++) {
775 36 : block_size *= info->dimension[f];
776 : }
777 :
778 17 : src_idx = dest_idx = 0;
779 :
780 : do {
781 9804 : for (f = 0; f < self->frames_out; f++) {
782 8840 : nns_memcpy (dest_info.data + dest_idx,
783 : src_info.data + src_idx + (frame_size * f), block_size);
784 8840 : dest_idx += block_size;
785 : }
786 :
787 964 : src_idx += block_size;
788 :
789 964 : g_assert (src_idx <= frame_size);
790 964 : g_assert (dest_idx <= dest_info.size);
791 964 : } while (src_idx < frame_size);
792 :
793 17 : gst_buffer_unmap (srcbuf, &src_info);
794 17 : gst_buffer_unmap (outbuf, &dest_info);
795 :
796 17 : gst_buffer_unref (srcbuf);
797 :
798 17 : return TRUE;
799 : }
800 :
801 : /**
802 : * @brief Push the buffer to source pad. (Concatenate the buffer if needed)
803 : */
804 : static GstFlowReturn
805 90 : gst_tensor_aggregator_push (GstTensorAggregator * self, GstBuffer * outbuf,
806 : gsize frame_size)
807 : {
808 : GstTensorInfo info;
809 :
810 : /** tensor info for one frame */
811 90 : info = self->out_config.info.info[0];
812 90 : g_assert (self->frames_dim < NNS_TENSOR_RANK_LIMIT);
813 90 : info.dimension[self->frames_dim] /= self->frames_out;
814 :
815 90 : if (frame_size != gst_tensor_info_get_size (&info) || frame_size == 0U) {
816 0 : ml_logf
817 : ("Invalid output capability of tensor_aggregator. Frame size = %"
818 : G_GSIZE_FORMAT "\n", frame_size);
819 90 : return GST_FLOW_ERROR;
820 : }
821 :
822 90 : if (gst_tensor_aggregator_check_concat_axis (self, &info)) {
823 : /** change data in buffer with given axis */
824 17 : if (!gst_tensor_aggregator_concat (self, outbuf, &info))
825 0 : return GST_FLOW_ERROR;
826 : }
827 :
828 90 : return gst_pad_push (self->srcpad, outbuf);
829 : }
830 :
831 : /**
832 : * @brief Chain function, this function does the actual processing.
833 : */
834 : static GstFlowReturn
835 171 : gst_tensor_aggregator_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
836 : {
837 : GstTensorAggregator *self;
838 171 : GstFlowReturn ret = GST_FLOW_OK;
839 : GstAdapter *adapter;
840 : gsize avail, buf_size, frame_size, out_size;
841 : guint frames_in, frames_out, frames_flush;
842 : GstClockTime duration;
843 : UNUSED (pad);
844 :
845 171 : self = GST_TENSOR_AGGREGATOR (parent);
846 171 : g_assert (self->tensor_configured);
847 :
848 171 : buf_size = gst_buffer_get_size (buf);
849 171 : g_return_val_if_fail (buf_size > 0, GST_FLOW_ERROR);
850 :
851 171 : frames_in = self->frames_in;
852 171 : frames_out = self->frames_out;
853 171 : frames_flush = self->frames_flush;
854 171 : frame_size = buf_size / frames_in;
855 :
856 171 : if (frames_in == frames_out) {
857 : /** push the incoming buffer (do concat if needed) */
858 2 : return gst_tensor_aggregator_push (self, buf, frame_size);
859 : }
860 :
861 169 : adapter = gst_tensor_aggregator_get_adapter (self, buf);
862 169 : g_assert (adapter != NULL);
863 :
864 169 : duration = GST_BUFFER_DURATION (buf);
865 169 : if (GST_CLOCK_TIME_IS_VALID (duration)) {
866 : /** supposed same duration for incoming buffer */
867 145 : duration = gst_util_uint64_scale_int (duration, frames_out, frames_in);
868 : }
869 :
870 169 : gst_adapter_push (adapter, buf);
871 :
872 169 : out_size = frame_size * frames_out;
873 169 : g_assert (out_size > 0);
874 :
875 257 : while ((avail = gst_adapter_available (adapter)) >= out_size &&
876 : ret == GST_FLOW_OK) {
877 : GstBuffer *outbuf;
878 : GstClockTime pts, dts;
879 : guint64 pts_dist, dts_dist;
880 : gsize flush;
881 :
882 88 : pts = gst_adapter_prev_pts (adapter, &pts_dist);
883 88 : dts = gst_adapter_prev_dts (adapter, &dts_dist);
884 :
885 : /**
886 : * Update timestamp.
887 : * If frames-in is larger then frames-out, the same timestamp (pts and dts) would be returned.
888 : */
889 88 : if (frames_in > 1) {
890 : gint fn, fd;
891 :
892 63 : fn = self->in_config.rate_n;
893 63 : fd = self->in_config.rate_d;
894 :
895 63 : if (fn > 0 && fd > 0) {
896 59 : if (GST_CLOCK_TIME_IS_VALID (pts)) {
897 59 : pts +=
898 59 : gst_util_uint64_scale_int (pts_dist * fd, GST_SECOND,
899 : fn * frame_size);
900 : }
901 :
902 59 : if (GST_CLOCK_TIME_IS_VALID (dts)) {
903 0 : dts +=
904 0 : gst_util_uint64_scale_int (dts_dist * fd, GST_SECOND,
905 : fn * frame_size);
906 : }
907 : }
908 : }
909 :
910 88 : outbuf = gst_adapter_get_buffer (adapter, out_size);
911 88 : outbuf = gst_buffer_make_writable (outbuf);
912 :
913 : /** set timestamp */
914 88 : GST_BUFFER_PTS (outbuf) = pts;
915 88 : GST_BUFFER_DTS (outbuf) = dts;
916 88 : GST_BUFFER_DURATION (outbuf) = duration;
917 :
918 88 : ret = gst_tensor_aggregator_push (self, outbuf, frame_size);
919 :
920 : /** flush data */
921 88 : if (frames_flush > 0) {
922 18 : flush = frame_size * frames_flush;
923 :
924 18 : if (flush > avail) {
925 : /**
926 : * @todo flush data
927 : * Invalid state, tried to flush large size.
928 : * We have to determine how to handle this case. (flush the out-size or all available bytes)
929 : * Now all available bytes in adapter will be flushed.
930 : */
931 0 : flush = avail;
932 : }
933 : } else {
934 70 : flush = out_size;
935 : }
936 :
937 88 : gst_adapter_flush (adapter, flush);
938 : }
939 :
940 169 : return ret;
941 : }
942 :
943 : /**
944 : * @brief Called to perform state change.
945 : */
946 : static GstStateChangeReturn
947 114 : gst_tensor_aggregator_change_state (GstElement * element,
948 : GstStateChange transition)
949 : {
950 : GstTensorAggregator *self;
951 : GstStateChangeReturn ret;
952 :
953 114 : self = GST_TENSOR_AGGREGATOR (element);
954 :
955 114 : switch (transition) {
956 19 : case GST_STATE_CHANGE_READY_TO_PAUSED:
957 19 : gst_tensor_aggregator_reset (self);
958 19 : break;
959 95 : default:
960 95 : break;
961 : }
962 :
963 114 : ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
964 :
965 114 : switch (transition) {
966 19 : case GST_STATE_CHANGE_PAUSED_TO_READY:
967 19 : gst_tensor_aggregator_reset (self);
968 19 : break;
969 95 : default:
970 95 : break;
971 : }
972 :
973 114 : return ret;
974 : }
975 :
976 : /**
977 : * @brief Clear and reset data.
978 : */
979 : static void
980 77 : gst_tensor_aggregator_reset (GstTensorAggregator * self)
981 : {
982 : /* remove all buffers from adapter */
983 77 : gst_tensor_aggregation_clear_all (self->adapter_table);
984 77 : }
985 :
986 : /**
987 : * @brief Get pad caps for caps negotiation.
988 : */
989 : static GstCaps *
990 71 : gst_tensor_aggregator_query_caps (GstTensorAggregator * self, GstPad * pad,
991 : GstCaps * filter)
992 : {
993 : GstCaps *caps;
994 : GstTensorsConfig *config;
995 :
996 : /* tensor config info for given pad */
997 71 : if (pad == self->sinkpad) {
998 41 : config = &self->in_config;
999 : } else {
1000 30 : config = &self->out_config;
1001 : }
1002 :
1003 : /* caps from tensor config info */
1004 71 : caps = gst_tensor_pad_possible_caps_from_config (pad, config);
1005 :
1006 71 : silent_debug_caps (self, caps, "caps");
1007 71 : silent_debug_caps (self, filter, "filter");
1008 :
1009 71 : if (caps && filter) {
1010 : GstCaps *intersection;
1011 :
1012 : intersection =
1013 0 : gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
1014 :
1015 0 : gst_caps_unref (caps);
1016 0 : caps = intersection;
1017 : }
1018 :
1019 71 : return caps;
1020 : }
1021 :
1022 : /**
1023 : * @brief Parse caps and set tensor info.
1024 : */
1025 : static gboolean
1026 18 : gst_tensor_aggregator_parse_caps (GstTensorAggregator * self,
1027 : const GstCaps * caps)
1028 : {
1029 : GstStructure *structure;
1030 : GstTensorsConfig config;
1031 : GstTensorInfo *_info;
1032 : uint32_t per_frame;
1033 : guint count;
1034 :
1035 36 : g_return_val_if_fail (caps != NULL, FALSE);
1036 18 : g_return_val_if_fail (gst_caps_is_fixed (caps), FALSE);
1037 :
1038 18 : structure = gst_caps_get_structure (caps, 0);
1039 :
1040 18 : if (!gst_tensors_config_from_structure (&config, structure) ||
1041 18 : !gst_tensors_config_validate (&config)) {
1042 0 : GST_ERROR_OBJECT (self, "Cannot configure tensor info");
1043 0 : return FALSE;
1044 : }
1045 :
1046 : /**
1047 : * @todo flush data
1048 : * Check properties to detect invalid case.
1049 : * Assertion when in=5 out=10 flush=20 or in=10 out=5 flush=20
1050 : */
1051 18 : count = (self->frames_out + self->frames_in - 1) / self->frames_in;
1052 18 : if (self->frames_in * count < self->frames_flush) {
1053 0 : GST_ERROR_OBJECT (self, "Cannot flush frames");
1054 0 : return FALSE;
1055 : }
1056 :
1057 18 : self->in_config = config;
1058 : /* tensor-aggregator now handles single tensor. */
1059 18 : _info = &config.info.info[0];
1060 :
1061 : /**
1062 : * update dimension in output tensor.
1063 : * e.g, in-dimension 2:200:200:1
1064 : * if frames_out=10 and frames_dim=3, then out-dimension is 2:200:200:10.
1065 : * if frames_out=10 and frames_dim=2, then out-dimension is 2:200:2000:1.
1066 : */
1067 18 : if (self->frames_dim >= NNS_TENSOR_RANK_LIMIT ||
1068 18 : (_info->dimension[self->frames_dim] % self->frames_in) != 0) {
1069 0 : GST_ERROR_OBJECT (self, "Cannot update dimension in output tensor");
1070 0 : return FALSE;
1071 : }
1072 18 : per_frame = _info->dimension[self->frames_dim] / self->frames_in;
1073 :
1074 18 : _info->dimension[self->frames_dim] = per_frame * self->frames_out;
1075 18 : self->out_config = config;
1076 18 : self->tensor_configured = TRUE;
1077 :
1078 18 : silent_debug_config (self, &self->in_config, "in-tensor");
1079 18 : silent_debug_config (self, &self->out_config, "out-tensor");
1080 18 : return TRUE;
1081 : }
|
