Line data Source code
1 : /**
2 : * GStreamer / NNStreamer tensor_decoder main
3 : * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
4 : * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
5 : * Copyright (C) 2018 Jijoong Moon <jijoong.moon@samsung.com>
6 : *
7 : * This library is free software; you can redistribute it and/or
8 : * modify it under the terms of the GNU Library General Public
9 : * License as published by the Free Software Foundation;
10 : * version 2.1 of the License.
11 : *
12 : * This library is distributed in the hope that it will be useful,
13 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : * Library General Public License for more details.
16 : *
17 : */
18 : /**
19 : * @file gsttensor_decoder.c
20 : * @date 26 Mar 2018
21 : * @brief GStreamer plugin to convert tensors (as a filter for other general neural network filters) to other media types
22 : * @see https://github.com/nnstreamer/nnstreamer
23 : * @author Jijoong Moon <jijoong.moon@samsung.com>
24 : * @bug gst_tensordec_transform_size () may be incorrect if direction is SINK.
25 : * @bug If configured = TRUE, it holds TRUE until exit. What if configuration changes in run-time?
26 : *
27 : */
28 :
29 : /**
30 : * SECTION:element-tensordec
31 : *
32 : * A filter that converts tensor stream for NN frameworks to media stream.
33 : * The input is always in the format of other/tensor
34 : *
35 : * <refsect2>
36 : * <title>Example launch line</title>
37 : * |[
38 : * gst-launch -v -m fakesink ! tensor_decoder ! fakesrc silent=TRUE
39 : * ]|
40 : * </refsect2>
41 : */
42 :
43 : #ifdef HAVE_CONFIG_H
44 : #include <config.h>
45 : #endif
46 :
47 : #include <string.h>
48 : #include "gsttensor_decoder.h"
49 :
50 : /**
51 : * @brief Macro for debug mode.
52 : */
53 : #ifndef DBG
54 : #define DBG (!self->silent)
55 : #endif
56 :
57 : GST_DEBUG_CATEGORY_STATIC (gst_tensordec_debug);
58 : #define GST_CAT_DEFAULT gst_tensordec_debug
59 :
60 : /**
61 : * @brief Properties.
62 : */
63 : enum
64 : {
65 : PROP_0,
66 : PROP_SILENT,
67 : PROP_MODE,
68 : PROP_MODE_OPTION1,
69 : PROP_MODE_OPTION2,
70 : PROP_MODE_OPTION3,
71 : PROP_MODE_OPTION4,
72 : PROP_MODE_OPTION5,
73 : PROP_MODE_OPTION6,
74 : PROP_MODE_OPTION7,
75 : PROP_MODE_OPTION8,
76 : PROP_MODE_OPTION9,
77 : PROP_SUBPLUGINS,
78 : PROP_CONFIG
79 : };
80 :
81 : /**
82 : * @brief Flag to print minimized log.
83 : */
84 : #define DEFAULT_SILENT TRUE
85 :
86 : /**
87 : * @brief Support multi-tensor along with single-tensor as the input
88 : */
89 : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_DEFAULT ";" GST_TENSORS_FLEX_CAP_DEFAULT
90 :
91 : /**
92 : * @brief The capabilities of the inputs
93 : */
94 : static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
95 : GST_PAD_SINK,
96 : GST_PAD_ALWAYS,
97 : GST_STATIC_CAPS (CAPS_STRING));
98 :
99 : /**
100 : * @brief The capabilities of the outputs
101 : */
102 : static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
103 : GST_PAD_SRC,
104 : GST_PAD_ALWAYS,
105 : GST_STATIC_CAPS ("ANY"));
106 :
107 : #define gst_tensordec_parent_class parent_class
108 1512 : G_DEFINE_TYPE (GstTensorDecoder, gst_tensordec, GST_TYPE_BASE_TRANSFORM);
109 :
110 : /** GObject vmethod implementations */
111 : static void gst_tensordec_set_property (GObject * object, guint prop_id,
112 : const GValue * value, GParamSpec * pspec);
113 : static void gst_tensordec_get_property (GObject * object, guint prop_id,
114 : GValue * value, GParamSpec * pspec);
115 : static void gst_tensordec_class_finalize (GObject * object);
116 :
117 : /** GstBaseTransform vmethod implementations */
118 : static GstFlowReturn gst_tensordec_transform (GstBaseTransform * trans,
119 : GstBuffer * inbuf, GstBuffer * outbuf);
120 : static GstCaps *gst_tensordec_transform_caps (GstBaseTransform * trans,
121 : GstPadDirection direction, GstCaps * caps, GstCaps * filter);
122 : static GstCaps *gst_tensordec_fixate_caps (GstBaseTransform * trans,
123 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
124 : static gboolean gst_tensordec_set_caps (GstBaseTransform * trans,
125 : GstCaps * incaps, GstCaps * outcaps);
126 : static gboolean gst_tensordec_transform_size (GstBaseTransform * trans,
127 : GstPadDirection direction, GstCaps * caps, gsize size,
128 : GstCaps * othercaps, gsize * othersize);
129 :
130 : /**
131 : * @brief Validate decoder sub-plugin's data.
132 : */
133 : static gboolean
134 492 : nnstreamer_decoder_validate (const GstTensorDecoderDef * decoder)
135 : {
136 492 : if (!decoder || !decoder->modename) {
137 : /* invalid name */
138 2 : return FALSE;
139 : }
140 :
141 490 : if (!decoder->init || !decoder->getOutCaps || !decoder->decode) {
142 : /* invalid methods in decoder sub-plugin */
143 3 : return FALSE;
144 : }
145 :
146 487 : return TRUE;
147 : }
148 :
149 : /**
150 : * @brief Decoder's sub-plugin should call this function to register itself.
151 : * @param[in] decoder Decoder sub-plugin to be registered.
152 : * @return TRUE if registered. FALSE is failed or duplicated.
153 : */
154 : int
155 348 : nnstreamer_decoder_probe (GstTensorDecoderDef * decoder)
156 : {
157 348 : g_return_val_if_fail (nnstreamer_decoder_validate (decoder), FALSE);
158 343 : return register_subplugin (NNS_SUBPLUGIN_DECODER, decoder->modename, decoder);
159 : }
160 :
161 : /**
162 : * @brief Decoder's sub-plugin may call this to unregister itself.
163 : * @param[in] name The name of decoder sub-plugin.
164 : */
165 : void
166 342 : nnstreamer_decoder_exit (const char *name)
167 : {
168 342 : unregister_subplugin (NNS_SUBPLUGIN_DECODER, name);
169 342 : }
170 :
171 : /**
172 : * @brief Find decoder sub-plugin with the name.
173 : * @param[in] name The name of decoder sub-plugin.
174 : * @return NULL if not found or the sub-plugin object has an error.
175 : */
176 : const GstTensorDecoderDef *
177 157 : nnstreamer_decoder_find (const char *name)
178 : {
179 157 : return get_subplugin (NNS_SUBPLUGIN_DECODER, name);
180 : }
181 :
182 : /**
183 : * @brief set custom property description for tensor decoder sub-plugin
184 : */
185 : void
186 136 : nnstreamer_decoder_set_custom_property_desc (const char *name, const char *prop,
187 : ...)
188 : {
189 : va_list varargs;
190 :
191 136 : va_start (varargs, prop);
192 136 : subplugin_set_custom_property_desc (NNS_SUBPLUGIN_DECODER, name, prop,
193 : varargs);
194 136 : va_end (varargs);
195 136 : }
196 :
197 : /**
198 : * @brief Macro to clean sub-plugin data
199 : */
200 : #define gst_tensor_decoder_clean_plugin(self) do { \
201 : if (self->decoder) { \
202 : if (self->decoder->exit) \
203 : self->decoder->exit (&self->plugin_data); \
204 : else \
205 : g_free (self->plugin_data); \
206 : self->plugin_data = NULL; \
207 : } \
208 : } while (0)
209 :
210 : /**
211 : * @brief Get media caps from tensor config
212 : * @param self "this" pointer
213 : * @param config tensor config info
214 : * @return caps for media type
215 : */
216 : static GstCaps *
217 693 : gst_tensordec_get_media_caps_from_config (GstTensorDecoder * self,
218 : const GstTensorsConfig * config)
219 : {
220 693 : g_return_val_if_fail (config != NULL, NULL);
221 :
222 693 : if (self->decoder == NULL) {
223 6 : if (self->is_custom) {
224 : GstCaps *caps;
225 6 : caps = gst_caps_from_string ("application/octet-stream");
226 6 : if (config->rate_n >= 0 && config->rate_d > 0)
227 6 : gst_caps_set_simple (caps, "framerate",
228 6 : GST_TYPE_FRACTION, config->rate_n, config->rate_d, NULL);
229 6 : return caps;
230 : }
231 0 : GST_ERROR_OBJECT (self, "Decoder plugin is not yet configured.");
232 0 : return NULL;
233 : }
234 :
235 : /* call sub-plugin vmethod */
236 687 : return self->decoder->getOutCaps (&self->plugin_data, config);
237 : }
238 :
239 : /**
240 : * @brief Parse tensor caps and return media caps
241 : * @param self "this" pointer
242 : * @param caps tensor caps to be interpreted
243 : */
244 : static GstCaps *
245 809 : gst_tensordec_get_media_caps (GstTensorDecoder * self, const GstCaps * caps)
246 : {
247 : GstTensorsConfig config;
248 809 : GstCaps *result = NULL;
249 :
250 809 : if (gst_tensors_config_from_caps (&config, caps, TRUE)) {
251 375 : result = gst_tensordec_get_media_caps_from_config (self, &config);
252 : }
253 :
254 809 : if (result == NULL) {
255 : /* we cannot specify the media type */
256 458 : result = gst_caps_new_any ();
257 : }
258 :
259 809 : return result;
260 : }
261 :
262 : /**
263 : * @brief Check tensor config is consistent
264 : * @param self "this" pointer to check consistency
265 : * @param t_info newly configured tensor metadata
266 : */
267 : static gboolean
268 169 : gst_tensordec_check_consistency (GstTensorDecoder * self,
269 : GstTensorsConfig * config)
270 : {
271 169 : g_return_val_if_fail (self != NULL, FALSE);
272 169 : g_return_val_if_fail (config != NULL, FALSE);
273 :
274 169 : if (self->configured) {
275 169 : return gst_tensors_config_is_equal (&self->tensor_config, config);
276 : }
277 :
278 : /** not configured yet */
279 0 : return FALSE;
280 : }
281 :
282 : /**
283 : * @brief initialize the tensordec's class
284 : */
285 : static void
286 126 : gst_tensordec_class_init (GstTensorDecoderClass * klass)
287 : {
288 : GObjectClass *gobject_class;
289 : GstElementClass *gstelement_class;
290 : GstBaseTransformClass *trans_class;
291 126 : gchar **subplugins = NULL;
292 : gchar *strbuf;
293 : static gchar *strprint = NULL;
294 :
295 126 : GST_DEBUG_CATEGORY_INIT (gst_tensordec_debug, "tensor_decoder", 0,
296 : "Element to convert tensor to media stream");
297 :
298 126 : trans_class = (GstBaseTransformClass *) klass;
299 126 : gstelement_class = (GstElementClass *) trans_class;
300 126 : gobject_class = (GObjectClass *) gstelement_class;
301 :
302 126 : gobject_class->set_property = gst_tensordec_set_property;
303 126 : gobject_class->get_property = gst_tensordec_get_property;
304 126 : gobject_class->finalize = gst_tensordec_class_finalize;
305 :
306 126 : g_object_class_install_property (gobject_class, PROP_SILENT,
307 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
308 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
309 :
310 126 : subplugins = get_all_subplugins (NNS_SUBPLUGIN_DECODER);
311 126 : strbuf = g_strjoinv (", ", subplugins);
312 126 : g_free (strprint);
313 126 : strprint = g_strdup_printf
314 : ("Decoder mode. Other options (option1 to optionN) depend on the specified model. For more detail on optionX for each mode, please refer to the documentation or nnstreamer-check utility. Available modes (decoder subplugins) are: {%s}.",
315 : strbuf);
316 :
317 126 : g_object_class_install_property (gobject_class, PROP_MODE,
318 : g_param_spec_string ("mode", "Mode", strprint, "",
319 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
320 126 : g_free (strbuf);
321 126 : g_strfreev (subplugins);
322 :
323 126 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION1,
324 : g_param_spec_string ("option1", "Mode option 1",
325 : "option for specific decoder modes, 1st one.", "",
326 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
327 :
328 126 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION2,
329 : g_param_spec_string ("option2", "Mode option 2",
330 : "option for specific decoder modes, 2nd one.", "",
331 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
332 :
333 126 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION3,
334 : g_param_spec_string ("option3", "Mode option 3",
335 : "option for specific decoder modes, 3rd one.", "",
336 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
337 :
338 126 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION4,
339 : g_param_spec_string ("option4", "Mode option 4",
340 : "option for specific decoder modes, 4th one.", "",
341 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
342 :
343 126 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION5,
344 : g_param_spec_string ("option5", "Mode option 5",
345 : "option for specific decoder modes, 5th one.", "",
346 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
347 :
348 126 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION6,
349 : g_param_spec_string ("option6", "Mode option 6",
350 : "option for specific decoder modes, 6th one.", "",
351 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
352 :
353 126 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION7,
354 : g_param_spec_string ("option7", "Mode option 7",
355 : "option for specific decoder modes, 7th one.", "",
356 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
357 :
358 126 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION8,
359 : g_param_spec_string ("option8", "Mode option 8",
360 : "option for specific decoder modes, 8th one.", "",
361 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
362 :
363 126 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION9,
364 : g_param_spec_string ("option9", "Mode option 9",
365 : "option for specific decoder modes, 9th one.", "",
366 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
367 :
368 126 : g_object_class_install_property (gobject_class, PROP_SUBPLUGINS,
369 : g_param_spec_string ("sub-plugins", "Sub-plugins",
370 : "Registrable sub-plugins list", "",
371 : G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
372 :
373 126 : g_object_class_install_property (gobject_class, PROP_CONFIG,
374 : g_param_spec_string ("config-file", "Configuration-file",
375 : "Path to configuration file which contains plugins properties", "",
376 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
377 :
378 126 : gst_element_class_set_details_simple (gstelement_class,
379 : "TensorDecoder",
380 : "Converter/Tensor",
381 : "Converts tensor stream of C-Array for neural network framework filters to audio or video stream",
382 : "Jijoong Moon <jijoong.moon@samsung.com>");
383 :
384 126 : gst_element_class_add_pad_template (gstelement_class,
385 : gst_static_pad_template_get (&sink_factory));
386 :
387 126 : gst_element_class_add_pad_template (gstelement_class,
388 : gst_static_pad_template_get (&src_factory));
389 :
390 : /** Refer: https://gstreamer.freedesktop.org/documentation/design/element-transform.html */
391 126 : trans_class->passthrough_on_same_caps = FALSE;
392 126 : trans_class->transform_ip_on_passthrough = FALSE;
393 :
394 : /** Processing units */
395 126 : trans_class->transform = GST_DEBUG_FUNCPTR (gst_tensordec_transform);
396 :
397 : /** Negotiation units */
398 126 : trans_class->transform_caps =
399 126 : GST_DEBUG_FUNCPTR (gst_tensordec_transform_caps);
400 126 : trans_class->fixate_caps = GST_DEBUG_FUNCPTR (gst_tensordec_fixate_caps);
401 126 : trans_class->set_caps = GST_DEBUG_FUNCPTR (gst_tensordec_set_caps);
402 :
403 : /** Allocation units */
404 126 : trans_class->transform_size =
405 126 : GST_DEBUG_FUNCPTR (gst_tensordec_transform_size);
406 126 : }
407 :
408 : /**
409 : * @brief initialize the new element
410 : * instantiate pads and add them to element
411 : * set pad callback functions
412 : * initialize instance structure
413 : */
414 : static void
415 147 : gst_tensordec_init (GstTensorDecoder * self)
416 : {
417 : guint i;
418 :
419 147 : self->silent = DEFAULT_SILENT;
420 147 : self->configured = FALSE;
421 147 : self->negotiated = FALSE;
422 147 : self->decoder = NULL;
423 147 : self->plugin_data = NULL;
424 147 : self->is_custom = FALSE;
425 147 : self->custom.func = NULL;
426 147 : self->custom.data = NULL;
427 147 : self->config_path = NULL;
428 1470 : for (i = 0; i < TensorDecMaxOpNum; i++)
429 1323 : self->option[i] = NULL;
430 :
431 147 : gst_tensors_config_init (&self->tensor_config);
432 147 : }
433 :
434 : /**
435 : * @brief Process plugin (self->decoder) with given options if available
436 : * @retval FALSE if error. TRUE if OK (or SKIP)
437 : */
438 : static gboolean
439 1462 : gst_tensordec_process_plugin_options (GstTensorDecoder * self, guint opnum)
440 : {
441 1462 : g_assert (opnum < TensorDecMaxOpNum); /* Internal logic error! */
442 1462 : if (self->decoder == NULL)
443 5 : return TRUE; /* decoder plugin not available. */
444 1457 : if (self->decoder->setOption == NULL)
445 0 : return TRUE; /* This decoder cannot process options */
446 1457 : if (self->option[opnum] == NULL)
447 1292 : return TRUE; /* No option to process */
448 165 : return self->decoder->setOption (&self->plugin_data, opnum,
449 165 : self->option[opnum]);
450 : }
451 :
452 : /**
453 : * @brief A macro to process incoming per-mode option
454 : * @param[in] opnum The option number (1 to TensorDecMaxOpNum)
455 : */
456 : #define PROP_MODE_OPTION(opnum) \
457 : case PROP_MODE_OPTION ## opnum: \
458 : g_free (self->option[(opnum) - 1]); \
459 : self->option[(opnum) - 1] = g_value_dup_string (value); \
460 : if (!gst_tensordec_process_plugin_options (self, (opnum) - 1)) \
461 : GST_ERROR_OBJECT (self, "Configuring option for tensor-decoder failed (option %d = %s)", \
462 : (opnum), self->option[(opnum) - 1]); \
463 : break
464 :
465 : /**
466 : * @brief Set property (GObject vmethod)
467 : */
468 : static void
469 317 : gst_tensordec_set_property (GObject * object, guint prop_id,
470 : const GValue * value, GParamSpec * pspec)
471 : {
472 : GstTensorDecoder *self;
473 :
474 317 : self = GST_TENSOR_DECODER (object);
475 :
476 317 : switch (prop_id) {
477 1 : case PROP_SILENT:
478 1 : self->silent = g_value_get_boolean (value);
479 1 : break;
480 145 : case PROP_MODE:
481 : {
482 : const GstTensorDecoderDef *decoder;
483 : const gchar *mode_string;
484 : guint i;
485 :
486 145 : mode_string = g_value_get_string (value);
487 145 : if (g_ascii_strcasecmp (mode_string, "custom-code") == 0) {
488 1 : self->is_custom = TRUE;
489 1 : break;
490 : }
491 :
492 144 : decoder = nnstreamer_decoder_find (mode_string);
493 :
494 : /* See if we are using "plugin" */
495 144 : if (nnstreamer_decoder_validate (decoder)) {
496 144 : silent_debug (self, "tensor_decoder plugin mode (%s)\n", mode_string);
497 :
498 144 : if (decoder == self->decoder) {
499 : /* Already configured??? */
500 0 : GST_WARNING_OBJECT (self,
501 : "nnstreamer tensor_decoder %s is already configured.\n",
502 : mode_string);
503 : } else {
504 : /* Changing decoder. Deallocate the previous */
505 144 : gst_tensor_decoder_clean_plugin (self);
506 144 : self->decoder = decoder;
507 : }
508 :
509 144 : if (0 == self->decoder->init (&self->plugin_data)) {
510 0 : ml_loge ("Failed to initialize a decode subplugin, \"%s\".\n",
511 : mode_string);
512 0 : break;
513 : }
514 :
515 1440 : for (i = 0; i < TensorDecMaxOpNum; i++)
516 1296 : if (!gst_tensordec_process_plugin_options (self, i))
517 0 : GST_WARNING_OBJECT (self,
518 : "Failed to configure while setting the option %d.", (i + 1));
519 : } else {
520 0 : GST_ERROR_OBJECT (self,
521 : "The given mode for tensor_decoder, %s, is unrecognized.\n",
522 : mode_string);
523 0 : gst_tensor_decoder_clean_plugin (self);
524 0 : self->decoder = NULL;
525 : }
526 144 : break;
527 : }
528 5 : case PROP_CONFIG:
529 : {
530 5 : g_free (self->config_path);
531 5 : self->config_path = g_strdup (g_value_get_string (value));
532 5 : gst_tensor_parse_config_file (self->config_path, object);
533 5 : break;
534 : }
535 68 : PROP_MODE_OPTION (1);
536 23 : PROP_MODE_OPTION (2);
537 17 : PROP_MODE_OPTION (3);
538 19 : PROP_MODE_OPTION (4);
539 19 : PROP_MODE_OPTION (5);
540 7 : PROP_MODE_OPTION (6);
541 7 : PROP_MODE_OPTION (7);
542 3 : PROP_MODE_OPTION (8);
543 3 : PROP_MODE_OPTION (9);
544 :
545 0 : default:
546 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
547 0 : break;
548 : }
549 317 : }
550 :
551 : /**
552 : * @brief A macro to read per-mode option
553 : * @param[in] opnum The option number (1 to TensorDecMaxOpNum)
554 : */
555 : #define PROP_READ_OPTION(opnum) \
556 : case PROP_MODE_OPTION ## opnum: \
557 : g_value_set_string (value, self->option[opnum - 1]); \
558 : break
559 :
560 : /**
561 : * @brief Get property (GObject vmethod)
562 : */
563 : static void
564 39 : gst_tensordec_get_property (GObject * object, guint prop_id,
565 : GValue * value, GParamSpec * pspec)
566 : {
567 : GstTensorDecoder *self;
568 :
569 39 : self = GST_TENSOR_DECODER (object);
570 :
571 39 : switch (prop_id) {
572 4 : case PROP_SILENT:
573 4 : g_value_set_boolean (value, self->silent);
574 4 : break;
575 3 : case PROP_MODE:
576 3 : if (self->is_custom)
577 0 : g_value_set_string (value, "custom-code");
578 3 : else if (self->decoder)
579 1 : g_value_set_string (value, self->decoder->modename);
580 : else
581 2 : g_value_set_string (value, "");
582 3 : break;
583 3 : PROP_READ_OPTION (1);
584 3 : PROP_READ_OPTION (2);
585 3 : PROP_READ_OPTION (3);
586 3 : PROP_READ_OPTION (4);
587 3 : PROP_READ_OPTION (5);
588 3 : PROP_READ_OPTION (6);
589 3 : PROP_READ_OPTION (7);
590 3 : PROP_READ_OPTION (8);
591 3 : PROP_READ_OPTION (9);
592 3 : case PROP_SUBPLUGINS:
593 : {
594 3 : gchar **str_array = get_all_subplugins (NNS_SUBPLUGIN_DECODER);
595 :
596 3 : if (str_array) {
597 3 : g_value_take_string (value, g_strjoinv (",", str_array));
598 3 : g_strfreev (str_array);
599 : } else {
600 0 : g_value_set_string (value, "");
601 : }
602 3 : break;
603 : }
604 2 : case PROP_CONFIG:
605 2 : g_value_set_string (value, self->config_path ? self->config_path : "");
606 2 : break;
607 0 : default:
608 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
609 0 : break;
610 : }
611 39 : }
612 :
613 : /**
614 : * @brief Finalize instance (GObject vmethod)
615 : */
616 : static void
617 146 : gst_tensordec_class_finalize (GObject * object)
618 : {
619 : GstTensorDecoder *self;
620 : guint i;
621 :
622 146 : self = GST_TENSOR_DECODER (object);
623 :
624 146 : gst_tensor_decoder_clean_plugin (self);
625 :
626 146 : g_free (self->config_path);
627 1460 : for (i = 0; i < TensorDecMaxOpNum; ++i) {
628 1314 : g_free (self->option[i]);
629 : }
630 146 : self->custom.func = NULL;
631 146 : self->custom.data = NULL;
632 :
633 146 : G_OBJECT_CLASS (parent_class)->finalize (object);
634 146 : }
635 :
636 : /**
637 : * @brief Configure tensor metadata from sink caps
638 : */
639 : static gboolean
640 307 : gst_tensordec_configure (GstTensorDecoder * self, const GstCaps * in_caps,
641 : const GstCaps * out_caps)
642 : {
643 : GstTensorsConfig config;
644 :
645 307 : if (self->decoder == NULL && !self->is_custom) {
646 0 : GST_ERROR_OBJECT (self, "Decoder plugin is not yet configured.");
647 307 : return FALSE;
648 : }
649 :
650 307 : if (!gst_tensors_config_from_caps (&config, in_caps, TRUE)) {
651 0 : GST_ERROR_OBJECT (self, "Cannot configure tensor from in-caps.");
652 0 : return FALSE;
653 : }
654 :
655 : /**
656 : * If previous input configuration is set and is not compatible with incoming caps,
657 : * get possible media caps from sub-plugin and change input configuration.
658 : */
659 307 : if (self->configured && !gst_tensordec_check_consistency (self, &config)) {
660 : GstCaps *supposed;
661 : gboolean compatible;
662 :
663 11 : supposed = gst_tensordec_get_media_caps_from_config (self, &config);
664 11 : compatible = gst_caps_is_always_compatible (out_caps, supposed);
665 11 : gst_caps_unref (supposed);
666 :
667 : /** Check if outcaps is compatible with new caps */
668 11 : if (!compatible) {
669 0 : GST_ERROR_OBJECT (self, "The coming tensor config is not valid.");
670 0 : return FALSE;
671 : }
672 :
673 11 : gst_tensor_decoder_clean_plugin (self);
674 11 : self->decoder->init (&self->plugin_data);
675 : }
676 :
677 307 : self->tensor_config = config;
678 307 : self->configured = TRUE;
679 307 : return TRUE;
680 : }
681 :
682 : /**
683 : * @brief non-ip transform. required vmethod for BaseTransform class.
684 : */
685 : static GstFlowReturn
686 388 : gst_tensordec_transform (GstBaseTransform * trans,
687 : GstBuffer * inbuf, GstBuffer * outbuf)
688 : {
689 : GstTensorDecoder *self;
690 : GstFlowReturn res;
691 :
692 388 : self = GST_TENSOR_DECODER_CAST (trans);
693 :
694 388 : if (G_UNLIKELY (!self->negotiated))
695 0 : goto unknown_tensor;
696 388 : if (G_UNLIKELY (!self->configured))
697 0 : goto unknown_format;
698 :
699 776 : if (self->decoder || self->is_custom) {
700 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT];
701 : GstMapInfo in_info[NNS_TENSOR_SIZE_LIMIT];
702 : GstTensorMemory input[NNS_TENSOR_SIZE_LIMIT];
703 : guint i, num_tensors, num_mems;
704 :
705 388 : num_mems = gst_tensor_buffer_get_count (inbuf);
706 388 : if (gst_tensors_config_is_flexible (&self->tensor_config)) {
707 24 : self->tensor_config.info.num_tensors = num_mems;
708 : }
709 388 : num_tensors = self->tensor_config.info.num_tensors;
710 : /** Internal logic error. Negotiation process should prevent this! */
711 388 : g_assert (num_mems == num_tensors);
712 :
713 935 : for (i = 0; i < num_tensors; i++) {
714 547 : in_mem[i] = gst_tensor_buffer_get_nth_memory (inbuf, i);
715 547 : if (!gst_memory_map (in_mem[i], &in_info[i], GST_MAP_READ)) {
716 : guint j;
717 0 : ml_logf ("Failed to map in_mem[%u].\n", i);
718 :
719 0 : for (j = 0; j < i; j++) {
720 0 : gst_memory_unmap (in_mem[j], &in_info[j]);
721 0 : gst_memory_unref (in_mem[j]);
722 : }
723 0 : gst_memory_unref (in_mem[i]);
724 0 : return GST_FLOW_ERROR;
725 : }
726 :
727 547 : input[i].data = in_info[i].data;
728 547 : input[i].size = in_info[i].size;
729 : }
730 388 : if (!self->is_custom) {
731 387 : res = self->decoder->decode (&self->plugin_data, &self->tensor_config,
732 : input, outbuf);
733 1 : } else if (self->custom.func != NULL) {
734 1 : res = self->custom.func (input, &self->tensor_config, self->custom.data,
735 : outbuf);
736 : } else {
737 0 : GST_ERROR_OBJECT (self, "Custom decoder callback is not registered.");
738 0 : res = GST_FLOW_ERROR;
739 : }
740 :
741 935 : for (i = 0; i < num_tensors; i++) {
742 547 : gst_memory_unmap (in_mem[i], &in_info[i]);
743 547 : gst_memory_unref (in_mem[i]);
744 : }
745 : } else {
746 0 : GST_ERROR_OBJECT (self, "Decoder plugin not yet configured.");
747 0 : goto unknown_type;
748 : }
749 :
750 388 : return res;
751 :
752 0 : unknown_format:
753 0 : GST_ERROR_OBJECT (self, "Hit unknown_format");
754 0 : GST_ELEMENT_ERROR (self, CORE, NOT_IMPLEMENTED, (NULL), ("unknown format"));
755 0 : return GST_FLOW_NOT_NEGOTIATED;
756 0 : unknown_tensor:
757 0 : GST_ERROR_OBJECT (self, "Hit unknown_tensor");
758 0 : GST_ELEMENT_ERROR (self, CORE, NOT_IMPLEMENTED, (NULL),
759 : ("unknown format for tensor"));
760 0 : return GST_FLOW_NOT_NEGOTIATED;
761 0 : unknown_type:
762 0 : GST_ELEMENT_ERROR (self, CORE, NOT_IMPLEMENTED, (NULL),
763 : ("not implemented decoder mode"));
764 0 : return GST_FLOW_NOT_SUPPORTED;
765 : }
766 :
767 : /**
768 : * @brief configure tensor-srcpad cap from "proposed" cap.
769 : *
770 : * @trans ("this" pointer)
771 : * @direction (why do we need this?)
772 : * @caps sinkpad cap
773 : * @filter this element's cap (don't know specifically.)
774 : */
775 : static GstCaps *
776 2002 : gst_tensordec_transform_caps (GstBaseTransform * trans,
777 : GstPadDirection direction, GstCaps * caps, GstCaps * filter)
778 : {
779 : GstTensorDecoder *self;
780 : GstCaps *result;
781 :
782 2002 : self = GST_TENSOR_DECODER_CAST (trans);
783 :
784 : /* Not ready */
785 2002 : if (self->decoder == NULL && !self->is_custom)
786 0 : return NULL;
787 :
788 2002 : if (self->is_custom) {
789 12 : const decoder_custom_cb_s *ptr = NULL;
790 12 : if (self->option[0] == NULL) {
791 0 : nns_logw ("Tensor decoder custom option is not given.");
792 0 : return NULL;
793 : }
794 12 : self->custom.func = NULL;
795 12 : ptr = get_subplugin (NNS_CUSTOM_DECODER, self->option[0]);
796 12 : if (!ptr) {
797 0 : nns_logw ("Failed to find custom subplugin of the tensor_decoder");
798 0 : return NULL;
799 : }
800 12 : self->custom.func = ptr->func;
801 12 : self->custom.data = ptr->data;
802 : }
803 :
804 2002 : silent_debug (self, "Direction = %d\n", direction);
805 2002 : silent_debug_caps (self, caps, "from");
806 2002 : silent_debug_caps (self, filter, "filter");
807 :
808 2002 : if (direction == GST_PAD_SINK) {
809 : /** caps = sinkpad (other/tensor) return = srcpad (media) */
810 809 : result = gst_tensordec_get_media_caps (self, caps);
811 1193 : } else if (direction == GST_PAD_SRC) {
812 : /** caps = srcpad (media) return = sinkpad (other/tensor) */
813 : /** @todo We may do more specific actions here */
814 1193 : result = gst_caps_from_string (CAPS_STRING);
815 : } else {
816 0 : g_assert (0); /* Internal logic error! */
817 : return NULL;
818 : }
819 :
820 2002 : if (filter && gst_caps_get_size (filter) > 0) {
821 : GstCaps *intersection;
822 :
823 : intersection =
824 18 : gst_caps_intersect_full (filter, result, GST_CAPS_INTERSECT_FIRST);
825 :
826 18 : gst_caps_unref (result);
827 18 : result = intersection;
828 : }
829 :
830 2002 : silent_debug_caps (self, result, "to");
831 :
832 2002 : GST_DEBUG_OBJECT (self, "Direction[%d] transformed %" GST_PTR_FORMAT
833 : " into %" GST_PTR_FORMAT, direction, caps, result);
834 2002 : return result;
835 : }
836 :
837 : /**
838 : * @brief fixate caps. required vmethod of BaseTransform
839 : */
840 : static GstCaps *
841 161 : gst_tensordec_fixate_caps (GstBaseTransform * trans,
842 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
843 : {
844 : GstTensorDecoder *self;
845 : GstCaps *supposed;
846 : GstCaps *result;
847 :
848 161 : self = GST_TENSOR_DECODER_CAST (trans);
849 :
850 161 : silent_debug_caps (self, caps, "from caps");
851 161 : silent_debug_caps (self, othercaps, "from othercaps");
852 :
853 161 : GST_DEBUG_OBJECT (self, "trying to fixate othercaps %" GST_PTR_FORMAT
854 : " based on caps %" GST_PTR_FORMAT, othercaps, caps);
855 :
856 : /**
857 : * In gst_tensordec_transform_caps, we have refused to specify caps
858 : * if the direction is GST_PAD_SRC. Thus, gstreamer shouldn't fixate
859 : * it with GST_PAD_SRC. If this happens, it's either internal logic
860 : * error or GST bug.
861 : */
862 161 : if (direction != GST_PAD_SINK) {
863 0 : ml_logf_stacktrace ("Invalid direction for tensor-decoder fixate caps.\n");
864 161 : return NULL;
865 : }
866 :
867 161 : if (gst_tensordec_configure (self, caps, othercaps)) {
868 : supposed =
869 161 : gst_tensordec_get_media_caps_from_config (self, &self->tensor_config);
870 : } else {
871 0 : supposed = gst_tensordec_get_media_caps (self, caps);
872 : }
873 :
874 161 : result = gst_caps_intersect (othercaps, supposed);
875 161 : gst_caps_unref (supposed);
876 :
877 161 : if (gst_caps_is_empty (result)) {
878 0 : gst_caps_unref (result);
879 0 : result = othercaps;
880 : } else {
881 161 : gst_caps_unref (othercaps);
882 : }
883 :
884 161 : GST_DEBUG_OBJECT (self, "now fixating %" GST_PTR_FORMAT, result);
885 :
886 161 : result = gst_caps_make_writable (result);
887 161 : result = gst_caps_fixate (result);
888 :
889 161 : if (direction == GST_PAD_SINK) {
890 161 : if (gst_caps_is_subset (caps, result)) {
891 0 : gst_caps_replace (&result, caps);
892 : }
893 : }
894 161 : return result;
895 : }
896 :
897 : /**
898 : * @brief set caps. required vmethod of BaseTransform
899 : */
900 : static gboolean
901 146 : gst_tensordec_set_caps (GstBaseTransform * trans,
902 : GstCaps * incaps, GstCaps * outcaps)
903 : {
904 146 : GstTensorDecoder *self = GST_TENSOR_DECODER_CAST (trans);
905 :
906 146 : silent_debug_caps (self, incaps, "from incaps");
907 146 : silent_debug_caps (self, outcaps, "from outcaps");
908 :
909 146 : if (gst_tensordec_configure (self, incaps, outcaps)) {
910 292 : GstCaps *supposed = gst_tensordec_get_media_caps_from_config (self,
911 146 : &self->tensor_config);
912 :
913 : /** Check if outcaps ==equivalent== supposed */
914 146 : if (gst_caps_is_always_compatible (outcaps, supposed)) {
915 146 : self->negotiated = TRUE;
916 : } else {
917 0 : GST_ERROR_OBJECT (self,
918 : "This is not compatible with the supposed output pad cap");
919 : }
920 :
921 146 : gst_caps_unref (supposed);
922 : }
923 :
924 146 : return self->negotiated;
925 : }
926 :
927 : /**
928 : * @brief Tell the framework the required size of buffer based on the info of the other side pad. optional vmethod of BaseTransform
929 : *
930 : * This is called when non-ip mode is used.
931 : */
932 : static gboolean
933 371 : gst_tensordec_transform_size (GstBaseTransform * trans,
934 : GstPadDirection direction, GstCaps * caps, gsize size,
935 : GstCaps * othercaps, gsize * othersize)
936 : {
937 : GstTensorDecoder *self;
938 :
939 371 : if (direction == GST_PAD_SRC)
940 0 : return FALSE;
941 : /** @todo If direction = SRC, you may need different interpretation! */
942 371 : self = GST_TENSOR_DECODER_CAST (trans);
943 :
944 371 : g_assert (self->configured);
945 :
946 371 : if (!self->is_custom && self->decoder->getTransformSize)
947 126 : *othersize = self->decoder->getTransformSize (&self->plugin_data,
948 126 : &self->tensor_config, caps, size, othercaps, direction);
949 : else
950 245 : *othersize = 0;
951 :
952 371 : return TRUE;
953 : }
954 :
955 : /**
956 : * @brief Registers a callback for tensor_decoder custom condition
957 : * @return 0 if success. -ERRNO if error.
958 : */
959 : int
960 5 : nnstreamer_decoder_custom_register (const gchar * name,
961 : tensor_decoder_custom func, void *data)
962 : {
963 : decoder_custom_cb_s *ptr;
964 :
965 5 : g_return_val_if_fail (name && strlen (name), -EINVAL);
966 4 : g_return_val_if_fail (func, -EINVAL);
967 :
968 3 : if (!(ptr = g_try_new0 (decoder_custom_cb_s, 1)))
969 0 : return -ENOMEM;
970 :
971 3 : ptr->func = func;
972 3 : ptr->data = data;
973 :
974 3 : if (register_subplugin (NNS_CUSTOM_DECODER, name, ptr))
975 2 : return 0;
976 :
977 1 : g_free (ptr);
978 1 : return -EINVAL;
979 : }
980 :
981 : /**
982 : * @brief Unregisters a callback for tensor_decoder custom condition
983 : * @return 0 if success. -ERRNO if error.
984 : */
985 : int
986 4 : nnstreamer_decoder_custom_unregister (const gchar * name)
987 : {
988 : decoder_custom_cb_s *ptr;
989 :
990 4 : ptr = (decoder_custom_cb_s *) get_subplugin (NNS_CUSTOM_DECODER, name);
991 4 : if (!unregister_subplugin (NNS_CUSTOM_DECODER, name)) {
992 2 : ml_loge ("Failed to unregister custom callback %s.", name);
993 2 : return -EINVAL;
994 : }
995 2 : g_free (ptr);
996 :
997 2 : return 0;
998 : }
|
