Line data Source code
1 : /**
2 : * GStreamer / NNStreamer tensor_decoder main
3 : * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
4 : * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
5 : * Copyright (C) 2018 Jijoong Moon <jijoong.moon@samsung.com>
6 : *
7 : * This library is free software; you can redistribute it and/or
8 : * modify it under the terms of the GNU Library General Public
9 : * License as published by the Free Software Foundation;
10 : * version 2.1 of the License.
11 : *
12 : * This library is distributed in the hope that it will be useful,
13 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : * Library General Public License for more details.
16 : *
17 : */
18 : /**
19 : * @file gsttensor_decoder.c
20 : * @date 26 Mar 2018
21 : * @brief GStreamer plugin to convert tensors (as a filter for other general neural network filters) to other media types
22 : * @see https://github.com/nnstreamer/nnstreamer
23 : * @author Jijoong Moon <jijoong.moon@samsung.com>
24 : * @bug gst_tensordec_transform_size () may be incorrect if direction is SINK.
25 : * @bug If configured = TRUE, it holds TRUE until exit. What if configuration changes in run-time?
26 : *
27 : */
28 :
29 : /**
30 : * SECTION:element-tensordec
31 : *
32 : * A filter that converts tensor stream for NN frameworks to media stream.
33 : * The input is always in the format of other/tensor
34 : *
35 : * <refsect2>
36 : * <title>Example launch line</title>
37 : * |[
38 : * gst-launch -v -m fakesink ! tensor_decoder ! fakesrc silent=TRUE
39 : * ]|
40 : * </refsect2>
41 : */
42 :
43 : #ifdef HAVE_CONFIG_H
44 : #include <config.h>
45 : #endif
46 :
47 : #include <string.h>
48 : #include "gsttensor_decoder.h"
49 :
50 : /**
51 : * @brief Macro for debug mode.
52 : */
53 : #ifndef DBG
54 : #define DBG (!self->silent)
55 : #endif
56 :
57 : GST_DEBUG_CATEGORY_STATIC (gst_tensordec_debug);
58 : #define GST_CAT_DEFAULT gst_tensordec_debug
59 :
60 : /**
61 : * @brief Properties.
62 : */
63 : enum
64 : {
65 : PROP_0,
66 : PROP_SILENT,
67 : PROP_MODE,
68 : PROP_MODE_OPTION1,
69 : PROP_MODE_OPTION2,
70 : PROP_MODE_OPTION3,
71 : PROP_MODE_OPTION4,
72 : PROP_MODE_OPTION5,
73 : PROP_MODE_OPTION6,
74 : PROP_MODE_OPTION7,
75 : PROP_MODE_OPTION8,
76 : PROP_MODE_OPTION9,
77 : PROP_SUBPLUGINS,
78 : PROP_CONFIG
79 : };
80 :
81 : /**
82 : * @brief Flag to print minimized log.
83 : */
84 : #define DEFAULT_SILENT TRUE
85 :
86 : /**
87 : * @brief Support multi-tensor along with single-tensor as the input
88 : */
89 : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_DEFAULT ";" GST_TENSORS_FLEX_CAP_DEFAULT
90 :
91 : /**
92 : * @brief The capabilities of the inputs
93 : */
94 : static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
95 : GST_PAD_SINK,
96 : GST_PAD_ALWAYS,
97 : GST_STATIC_CAPS (CAPS_STRING));
98 :
99 : /**
100 : * @brief The capabilities of the outputs
101 : */
102 : static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
103 : GST_PAD_SRC,
104 : GST_PAD_ALWAYS,
105 : GST_STATIC_CAPS ("ANY"));
106 :
107 : #define gst_tensordec_parent_class parent_class
108 1482 : G_DEFINE_TYPE (GstTensorDecoder, gst_tensordec, GST_TYPE_BASE_TRANSFORM);
109 :
110 : /** GObject vmethod implementations */
111 : static void gst_tensordec_set_property (GObject * object, guint prop_id,
112 : const GValue * value, GParamSpec * pspec);
113 : static void gst_tensordec_get_property (GObject * object, guint prop_id,
114 : GValue * value, GParamSpec * pspec);
115 : static void gst_tensordec_class_finalize (GObject * object);
116 :
117 : /** GstBaseTransform vmethod implementations */
118 : static GstFlowReturn gst_tensordec_transform (GstBaseTransform * trans,
119 : GstBuffer * inbuf, GstBuffer * outbuf);
120 : static GstCaps *gst_tensordec_transform_caps (GstBaseTransform * trans,
121 : GstPadDirection direction, GstCaps * caps, GstCaps * filter);
122 : static GstCaps *gst_tensordec_fixate_caps (GstBaseTransform * trans,
123 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
124 : static gboolean gst_tensordec_set_caps (GstBaseTransform * trans,
125 : GstCaps * incaps, GstCaps * outcaps);
126 : static gboolean gst_tensordec_transform_size (GstBaseTransform * trans,
127 : GstPadDirection direction, GstCaps * caps, gsize size,
128 : GstCaps * othercaps, gsize * othersize);
129 :
130 : /**
131 : * @brief Validate decoder sub-plugin's data.
132 : */
133 : static gboolean
134 272 : nnstreamer_decoder_validate (const GstTensorDecoderDef * decoder)
135 : {
136 272 : if (!decoder || !decoder->modename) {
137 : /* invalid name */
138 2 : return FALSE;
139 : }
140 :
141 270 : if (!decoder->init || !decoder->getOutCaps || !decoder->decode) {
142 : /* invalid methods in decoder sub-plugin */
143 3 : return FALSE;
144 : }
145 :
146 267 : return TRUE;
147 : }
148 :
149 : /**
150 : * @brief Decoder's sub-plugin should call this function to register itself.
151 : * @param[in] decoder Decoder sub-plugin to be registered.
152 : * @return TRUE if registered. FALSE is failed or duplicated.
153 : */
154 : int
155 128 : nnstreamer_decoder_probe (GstTensorDecoderDef * decoder)
156 : {
157 128 : g_return_val_if_fail (nnstreamer_decoder_validate (decoder), FALSE);
158 123 : return register_subplugin (NNS_SUBPLUGIN_DECODER, decoder->modename, decoder);
159 : }
160 :
161 : /**
162 : * @brief Decoder's sub-plugin may call this to unregister itself.
163 : * @param[in] name The name of decoder sub-plugin.
164 : */
165 : void
166 122 : nnstreamer_decoder_exit (const char *name)
167 : {
168 122 : unregister_subplugin (NNS_SUBPLUGIN_DECODER, name);
169 122 : }
170 :
171 : /**
172 : * @brief Find decoder sub-plugin with the name.
173 : * @param[in] name The name of decoder sub-plugin.
174 : * @return NULL if not found or the sub-plugin object has an error.
175 : */
176 : const GstTensorDecoderDef *
177 157 : nnstreamer_decoder_find (const char *name)
178 : {
179 157 : return get_subplugin (NNS_SUBPLUGIN_DECODER, name);
180 : }
181 :
182 : /**
183 : * @brief set custom property description for tensor decoder sub-plugin
184 : */
185 : void
186 56 : nnstreamer_decoder_set_custom_property_desc (const char *name, const char *prop,
187 : ...)
188 : {
189 : va_list varargs;
190 :
191 56 : va_start (varargs, prop);
192 56 : subplugin_set_custom_property_desc (NNS_SUBPLUGIN_DECODER, name, prop,
193 : varargs);
194 56 : va_end (varargs);
195 56 : }
196 :
197 : /**
198 : * @brief Macro to clean sub-plugin data
199 : */
200 : #define gst_tensor_decoder_clean_plugin(self) do { \
201 : if (self->decoder) { \
202 : if (self->decoder->exit) \
203 : self->decoder->exit (&self->plugin_data); \
204 : else \
205 : g_free (self->plugin_data); \
206 : self->plugin_data = NULL; \
207 : } \
208 : } while (0)
209 :
210 : /**
211 : * @brief Get media caps from tensor config
212 : * @param self "this" pointer
213 : * @param config tensor config info
214 : * @return caps for media type
215 : */
216 : static GstCaps *
217 1127 : gst_tensordec_media_caps_from_tensor (GstTensorDecoder * self,
218 : const GstTensorsConfig * config)
219 : {
220 1127 : g_return_val_if_fail (config != NULL, NULL);
221 :
222 1127 : if (self->decoder == NULL) {
223 8 : if (self->is_custom) {
224 : GstCaps *caps;
225 8 : caps = gst_caps_from_string ("application/octet-stream");
226 8 : if (config->rate_n >= 0 && config->rate_d > 0)
227 6 : gst_caps_set_simple (caps, "framerate",
228 6 : GST_TYPE_FRACTION, config->rate_n, config->rate_d, NULL);
229 8 : return caps;
230 : }
231 0 : GST_ERROR_OBJECT (self, "Decoder plugin is not yet configured.");
232 0 : return NULL;
233 : }
234 :
235 : /* call sub-plugin vmethod */
236 1119 : return self->decoder->getOutCaps (&self->plugin_data, config);
237 : }
238 :
239 : /**
240 : * @brief Parse structure and return media caps
241 : * @param self "this" pointer
242 : * @param structure structure to be interpreted
243 : */
244 : static GstCaps *
245 809 : gst_tensordec_media_caps_from_structure (GstTensorDecoder * self,
246 : const GstStructure * structure)
247 : {
248 : GstTensorsConfig config;
249 809 : GstCaps *result = NULL;
250 :
251 809 : if (gst_tensors_config_from_structure (&config, structure)) {
252 809 : result = gst_tensordec_media_caps_from_tensor (self, &config);
253 : }
254 :
255 809 : if (result == NULL) {
256 : /* we cannot specify the media type */
257 207 : result = gst_caps_new_any ();
258 : }
259 :
260 809 : return result;
261 : }
262 :
263 : /**
264 : * @brief Check tensor config is consistent
265 : * @param self "this" pointer to check consistency
266 : * @param t_info newly configured tensor metadata
267 : */
268 : static gboolean
269 169 : gst_tensordec_check_consistency (GstTensorDecoder * self,
270 : GstTensorsConfig * config)
271 : {
272 169 : g_return_val_if_fail (self != NULL, FALSE);
273 169 : g_return_val_if_fail (config != NULL, FALSE);
274 :
275 169 : if (self->configured) {
276 169 : return gst_tensors_config_is_equal (&self->tensor_config, config);
277 : }
278 :
279 : /** not configured yet */
280 0 : return FALSE;
281 : }
282 :
283 : /**
284 : * @brief initialize the tensordec's class
285 : */
286 : static void
287 116 : gst_tensordec_class_init (GstTensorDecoderClass * klass)
288 : {
289 : GObjectClass *gobject_class;
290 : GstElementClass *gstelement_class;
291 : GstBaseTransformClass *trans_class;
292 116 : gchar **subplugins = NULL;
293 : gchar *strbuf;
294 : static gchar *strprint = NULL;
295 :
296 116 : GST_DEBUG_CATEGORY_INIT (gst_tensordec_debug, "tensor_decoder", 0,
297 : "Element to convert tensor to media stream");
298 :
299 116 : trans_class = (GstBaseTransformClass *) klass;
300 116 : gstelement_class = (GstElementClass *) trans_class;
301 116 : gobject_class = (GObjectClass *) gstelement_class;
302 :
303 116 : gobject_class->set_property = gst_tensordec_set_property;
304 116 : gobject_class->get_property = gst_tensordec_get_property;
305 116 : gobject_class->finalize = gst_tensordec_class_finalize;
306 :
307 116 : g_object_class_install_property (gobject_class, PROP_SILENT,
308 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
309 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
310 :
311 116 : subplugins = get_all_subplugins (NNS_SUBPLUGIN_DECODER);
312 116 : strbuf = g_strjoinv (", ", subplugins);
313 116 : g_free (strprint);
314 116 : strprint = g_strdup_printf
315 : ("Decoder mode. Other options (option1 to optionN) depend on the specified model. For more detail on optionX for each mode, please refer to the documentation or nnstreamer-check utility. Available modes (decoder subplugins) are: {%s}.",
316 : strbuf);
317 :
318 116 : g_object_class_install_property (gobject_class, PROP_MODE,
319 : g_param_spec_string ("mode", "Mode", strprint, "",
320 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
321 116 : g_free (strbuf);
322 116 : g_strfreev (subplugins);
323 :
324 116 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION1,
325 : g_param_spec_string ("option1", "Mode option 1",
326 : "option for specific decoder modes, 1st one.", "",
327 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
328 :
329 116 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION2,
330 : g_param_spec_string ("option2", "Mode option 2",
331 : "option for specific decoder modes, 2nd one.", "",
332 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
333 :
334 116 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION3,
335 : g_param_spec_string ("option3", "Mode option 3",
336 : "option for specific decoder modes, 3rd one.", "",
337 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
338 :
339 116 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION4,
340 : g_param_spec_string ("option4", "Mode option 4",
341 : "option for specific decoder modes, 4th one.", "",
342 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
343 :
344 116 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION5,
345 : g_param_spec_string ("option5", "Mode option 5",
346 : "option for specific decoder modes, 5th one.", "",
347 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
348 :
349 116 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION6,
350 : g_param_spec_string ("option6", "Mode option 6",
351 : "option for specific decoder modes, 6th one.", "",
352 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
353 :
354 116 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION7,
355 : g_param_spec_string ("option7", "Mode option 7",
356 : "option for specific decoder modes, 7th one.", "",
357 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
358 :
359 116 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION8,
360 : g_param_spec_string ("option8", "Mode option 8",
361 : "option for specific decoder modes, 8th one.", "",
362 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
363 :
364 116 : g_object_class_install_property (gobject_class, PROP_MODE_OPTION9,
365 : g_param_spec_string ("option9", "Mode option 9",
366 : "option for specific decoder modes, 9th one.", "",
367 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
368 :
369 116 : g_object_class_install_property (gobject_class, PROP_SUBPLUGINS,
370 : g_param_spec_string ("sub-plugins", "Sub-plugins",
371 : "Registrable sub-plugins list", "",
372 : G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
373 :
374 116 : g_object_class_install_property (gobject_class, PROP_CONFIG,
375 : g_param_spec_string ("config-file", "Configuration-file",
376 : "Path to configuration file which contains plugins properties", "",
377 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
378 :
379 116 : gst_element_class_set_details_simple (gstelement_class,
380 : "TensorDecoder",
381 : "Converter/Tensor",
382 : "Converts tensor stream of C-Array for neural network framework filters to audio or video stream",
383 : "Jijoong Moon <jijoong.moon@samsung.com>");
384 :
385 116 : gst_element_class_add_pad_template (gstelement_class,
386 : gst_static_pad_template_get (&sink_factory));
387 :
388 116 : gst_element_class_add_pad_template (gstelement_class,
389 : gst_static_pad_template_get (&src_factory));
390 :
391 : /** Refer: https://gstreamer.freedesktop.org/documentation/design/element-transform.html */
392 116 : trans_class->passthrough_on_same_caps = FALSE;
393 116 : trans_class->transform_ip_on_passthrough = FALSE;
394 :
395 : /** Processing units */
396 116 : trans_class->transform = GST_DEBUG_FUNCPTR (gst_tensordec_transform);
397 :
398 : /** Negotiation units */
399 116 : trans_class->transform_caps =
400 116 : GST_DEBUG_FUNCPTR (gst_tensordec_transform_caps);
401 116 : trans_class->fixate_caps = GST_DEBUG_FUNCPTR (gst_tensordec_fixate_caps);
402 116 : trans_class->set_caps = GST_DEBUG_FUNCPTR (gst_tensordec_set_caps);
403 :
404 : /** Allocation units */
405 116 : trans_class->transform_size =
406 116 : GST_DEBUG_FUNCPTR (gst_tensordec_transform_size);
407 116 : }
408 :
409 : /**
410 : * @brief initialize the new element
411 : * instantiate pads and add them to element
412 : * set pad callback functions
413 : * initialize instance structure
414 : */
415 : static void
416 147 : gst_tensordec_init (GstTensorDecoder * self)
417 : {
418 : guint i;
419 :
420 147 : self->silent = DEFAULT_SILENT;
421 147 : self->configured = FALSE;
422 147 : self->negotiated = FALSE;
423 147 : self->decoder = NULL;
424 147 : self->plugin_data = NULL;
425 147 : self->is_custom = FALSE;
426 147 : self->custom.func = NULL;
427 147 : self->custom.data = NULL;
428 147 : self->config_path = NULL;
429 1470 : for (i = 0; i < TensorDecMaxOpNum; i++)
430 1323 : self->option[i] = NULL;
431 :
432 147 : gst_tensors_config_init (&self->tensor_config);
433 147 : }
434 :
435 : /**
436 : * @brief Process plugin (self->decoder) with given options if available
437 : * @retval FALSE if error. TRUE if OK (or SKIP)
438 : */
439 : static gboolean
440 1462 : gst_tensordec_process_plugin_options (GstTensorDecoder * self, guint opnum)
441 : {
442 1462 : g_assert (opnum < TensorDecMaxOpNum); /* Internal logic error! */
443 1462 : if (self->decoder == NULL)
444 5 : return TRUE; /* decoder plugin not available. */
445 1457 : if (self->decoder->setOption == NULL)
446 0 : return TRUE; /* This decoder cannot process options */
447 1457 : if (self->option[opnum] == NULL)
448 1292 : return TRUE; /* No option to process */
449 165 : return self->decoder->setOption (&self->plugin_data, opnum,
450 165 : self->option[opnum]);
451 : }
452 :
453 : /**
454 : * @brief A macro to process incoming per-mode option
455 : * @param[in] opnum The option number (1 to TensorDecMaxOpNum)
456 : */
457 : #define PROP_MODE_OPTION(opnum) \
458 : case PROP_MODE_OPTION ## opnum: \
459 : g_free (self->option[(opnum) - 1]); \
460 : self->option[(opnum) - 1] = g_value_dup_string (value); \
461 : if (!gst_tensordec_process_plugin_options (self, (opnum) - 1)) \
462 : GST_ERROR_OBJECT (self, "Configuring option for tensor-decoder failed (option %d = %s)", \
463 : (opnum), self->option[(opnum) - 1]); \
464 : break
465 :
466 : /**
467 : * @brief Set property (GObject vmethod)
468 : */
469 : static void
470 317 : gst_tensordec_set_property (GObject * object, guint prop_id,
471 : const GValue * value, GParamSpec * pspec)
472 : {
473 : GstTensorDecoder *self;
474 :
475 317 : self = GST_TENSOR_DECODER (object);
476 :
477 317 : switch (prop_id) {
478 1 : case PROP_SILENT:
479 1 : self->silent = g_value_get_boolean (value);
480 1 : break;
481 145 : case PROP_MODE:
482 : {
483 : const GstTensorDecoderDef *decoder;
484 : const gchar *mode_string;
485 : guint i;
486 :
487 145 : mode_string = g_value_get_string (value);
488 145 : if (g_ascii_strcasecmp (mode_string, "custom-code") == 0) {
489 1 : self->is_custom = TRUE;
490 1 : break;
491 : }
492 :
493 144 : decoder = nnstreamer_decoder_find (mode_string);
494 :
495 : /* See if we are using "plugin" */
496 144 : if (nnstreamer_decoder_validate (decoder)) {
497 144 : silent_debug (self, "tensor_decoder plugin mode (%s)\n", mode_string);
498 :
499 144 : if (decoder == self->decoder) {
500 : /* Already configured??? */
501 0 : GST_WARNING_OBJECT (self,
502 : "nnstreamer tensor_decoder %s is already configured.\n",
503 : mode_string);
504 : } else {
505 : /* Changing decoder. Deallocate the previous */
506 144 : gst_tensor_decoder_clean_plugin (self);
507 144 : self->decoder = decoder;
508 : }
509 :
510 144 : if (0 == self->decoder->init (&self->plugin_data)) {
511 0 : ml_loge ("Failed to initialize a decode subplugin, \"%s\".\n",
512 : mode_string);
513 0 : break;
514 : }
515 :
516 1440 : for (i = 0; i < TensorDecMaxOpNum; i++)
517 1296 : if (!gst_tensordec_process_plugin_options (self, i))
518 0 : GST_WARNING_OBJECT (self,
519 : "Failed to configure while setting the option %d.", (i + 1));
520 : } else {
521 0 : GST_ERROR_OBJECT (self,
522 : "The given mode for tensor_decoder, %s, is unrecognized.\n",
523 : mode_string);
524 0 : gst_tensor_decoder_clean_plugin (self);
525 0 : self->decoder = NULL;
526 : }
527 144 : break;
528 : }
529 5 : case PROP_CONFIG:
530 : {
531 5 : g_free (self->config_path);
532 5 : self->config_path = g_strdup (g_value_get_string (value));
533 5 : gst_tensor_parse_config_file (self->config_path, object);
534 5 : break;
535 : }
536 68 : PROP_MODE_OPTION (1);
537 23 : PROP_MODE_OPTION (2);
538 17 : PROP_MODE_OPTION (3);
539 19 : PROP_MODE_OPTION (4);
540 19 : PROP_MODE_OPTION (5);
541 7 : PROP_MODE_OPTION (6);
542 7 : PROP_MODE_OPTION (7);
543 3 : PROP_MODE_OPTION (8);
544 3 : PROP_MODE_OPTION (9);
545 :
546 0 : default:
547 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
548 0 : break;
549 : }
550 317 : }
551 :
552 : /**
553 : * @brief A macro to read per-mode option
554 : * @param[in] opnum The option number (1 to TensorDecMaxOpNum)
555 : */
556 : #define PROP_READ_OPTION(opnum) \
557 : case PROP_MODE_OPTION ## opnum: \
558 : g_value_set_string (value, self->option[opnum - 1]); \
559 : break
560 :
561 : /**
562 : * @brief Get property (GObject vmethod)
563 : */
564 : static void
565 39 : gst_tensordec_get_property (GObject * object, guint prop_id,
566 : GValue * value, GParamSpec * pspec)
567 : {
568 : GstTensorDecoder *self;
569 :
570 39 : self = GST_TENSOR_DECODER (object);
571 :
572 39 : switch (prop_id) {
573 4 : case PROP_SILENT:
574 4 : g_value_set_boolean (value, self->silent);
575 4 : break;
576 3 : case PROP_MODE:
577 3 : if (self->is_custom)
578 0 : g_value_set_string (value, "custom-code");
579 3 : else if (self->decoder)
580 1 : g_value_set_string (value, self->decoder->modename);
581 : else
582 2 : g_value_set_string (value, "");
583 3 : break;
584 3 : PROP_READ_OPTION (1);
585 3 : PROP_READ_OPTION (2);
586 3 : PROP_READ_OPTION (3);
587 3 : PROP_READ_OPTION (4);
588 3 : PROP_READ_OPTION (5);
589 3 : PROP_READ_OPTION (6);
590 3 : PROP_READ_OPTION (7);
591 3 : PROP_READ_OPTION (8);
592 3 : PROP_READ_OPTION (9);
593 3 : case PROP_SUBPLUGINS:
594 : {
595 3 : gchar **str_array = get_all_subplugins (NNS_SUBPLUGIN_DECODER);
596 :
597 3 : if (str_array) {
598 3 : g_value_take_string (value, g_strjoinv (",", str_array));
599 3 : g_strfreev (str_array);
600 : } else {
601 0 : g_value_set_string (value, "");
602 : }
603 3 : break;
604 : }
605 2 : case PROP_CONFIG:
606 2 : g_value_set_string (value, self->config_path ? self->config_path : "");
607 2 : break;
608 0 : default:
609 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
610 0 : break;
611 : }
612 39 : }
613 :
614 : /**
615 : * @brief Finalize instance (GObject vmethod)
616 : */
617 : static void
618 146 : gst_tensordec_class_finalize (GObject * object)
619 : {
620 : GstTensorDecoder *self;
621 : guint i;
622 :
623 146 : self = GST_TENSOR_DECODER (object);
624 :
625 146 : gst_tensor_decoder_clean_plugin (self);
626 :
627 146 : g_free (self->config_path);
628 1460 : for (i = 0; i < TensorDecMaxOpNum; ++i) {
629 1314 : g_free (self->option[i]);
630 : }
631 146 : self->custom.func = NULL;
632 146 : self->custom.data = NULL;
633 :
634 146 : G_OBJECT_CLASS (parent_class)->finalize (object);
635 146 : }
636 :
637 : /**
638 : * @brief Configure tensor metadata from sink caps
639 : */
640 : static gboolean
641 307 : gst_tensordec_configure (GstTensorDecoder * self, const GstCaps * in_caps,
642 : const GstCaps * out_caps)
643 : {
644 : GstStructure *structure;
645 : GstTensorsConfig config;
646 :
647 : /** This caps is coming from tensor */
648 307 : structure = gst_caps_get_structure (in_caps, 0);
649 :
650 307 : if (!gst_tensors_config_from_structure (&config, structure)) {
651 0 : GST_ERROR_OBJECT (self, "Cannot configure tensor from structure");
652 307 : return FALSE;
653 : }
654 :
655 307 : if (!gst_tensors_config_validate (&config)) {
656 0 : GST_ERROR_OBJECT (self, "Not configured yet");
657 0 : return FALSE;
658 : }
659 :
660 307 : if (self->decoder == NULL && !self->is_custom) {
661 0 : GST_ERROR_OBJECT (self, "Decoder plugin is not yet configured.");
662 0 : return FALSE;
663 : }
664 :
665 : /**
666 : * If previous input configuration is set and is not compatible with incoming caps,
667 : * get possible media caps from sub-plugin and change input configuration.
668 : */
669 307 : if (self->configured && !gst_tensordec_check_consistency (self, &config)) {
670 : GstCaps *supposed;
671 : gboolean compatible;
672 :
673 11 : supposed = gst_tensordec_media_caps_from_tensor (self, &config);
674 11 : compatible = gst_caps_is_always_compatible (out_caps, supposed);
675 11 : gst_caps_unref (supposed);
676 :
677 : /** Check if outcaps is compatible with new caps */
678 11 : if (!compatible) {
679 0 : GST_ERROR_OBJECT (self, "The coming tensor config is not valid.");
680 0 : return FALSE;
681 : }
682 :
683 11 : gst_tensor_decoder_clean_plugin (self);
684 11 : self->decoder->init (&self->plugin_data);
685 : }
686 :
687 307 : self->tensor_config = config;
688 307 : self->configured = TRUE;
689 307 : return TRUE;
690 : }
691 :
692 : /**
693 : * @brief non-ip transform. required vmethod for BaseTransform class.
694 : */
695 : static GstFlowReturn
696 390 : gst_tensordec_transform (GstBaseTransform * trans,
697 : GstBuffer * inbuf, GstBuffer * outbuf)
698 : {
699 : GstTensorDecoder *self;
700 : GstFlowReturn res;
701 :
702 390 : self = GST_TENSOR_DECODER_CAST (trans);
703 :
704 390 : if (G_UNLIKELY (!self->negotiated))
705 0 : goto unknown_tensor;
706 390 : if (G_UNLIKELY (!self->configured))
707 0 : goto unknown_format;
708 :
709 780 : if (self->decoder || self->is_custom) {
710 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT];
711 : GstMapInfo in_info[NNS_TENSOR_SIZE_LIMIT];
712 : GstTensorMemory input[NNS_TENSOR_SIZE_LIMIT];
713 : guint i, num_tensors, num_mems;
714 :
715 390 : num_mems = gst_tensor_buffer_get_count (inbuf);
716 390 : if (gst_tensors_config_is_flexible (&self->tensor_config)) {
717 24 : self->tensor_config.info.num_tensors = num_mems;
718 : }
719 390 : num_tensors = self->tensor_config.info.num_tensors;
720 : /** Internal logic error. Negotiation process should prevent this! */
721 390 : g_assert (num_mems == num_tensors);
722 :
723 939 : for (i = 0; i < num_tensors; i++) {
724 549 : in_mem[i] = gst_tensor_buffer_get_nth_memory (inbuf, i);
725 549 : if (!gst_memory_map (in_mem[i], &in_info[i], GST_MAP_READ)) {
726 : guint j;
727 0 : ml_logf ("Failed to map in_mem[%u].\n", i);
728 :
729 0 : for (j = 0; j < i; j++) {
730 0 : gst_memory_unmap (in_mem[j], &in_info[j]);
731 0 : gst_memory_unref (in_mem[j]);
732 : }
733 0 : gst_memory_unref (in_mem[i]);
734 0 : return GST_FLOW_ERROR;
735 : }
736 :
737 549 : input[i].data = in_info[i].data;
738 549 : input[i].size = in_info[i].size;
739 : }
740 390 : if (!self->is_custom) {
741 389 : res = self->decoder->decode (&self->plugin_data, &self->tensor_config,
742 : input, outbuf);
743 1 : } else if (self->custom.func != NULL) {
744 1 : res = self->custom.func (input, &self->tensor_config, self->custom.data,
745 : outbuf);
746 : } else {
747 0 : GST_ERROR_OBJECT (self, "Custom decoder callback is not registered.");
748 0 : res = GST_FLOW_ERROR;
749 : }
750 :
751 939 : for (i = 0; i < num_tensors; i++) {
752 549 : gst_memory_unmap (in_mem[i], &in_info[i]);
753 549 : gst_memory_unref (in_mem[i]);
754 : }
755 : } else {
756 0 : GST_ERROR_OBJECT (self, "Decoder plugin not yet configured.");
757 0 : goto unknown_type;
758 : }
759 :
760 390 : return res;
761 :
762 0 : unknown_format:
763 0 : GST_ERROR_OBJECT (self, "Hit unknown_format");
764 0 : GST_ELEMENT_ERROR (self, CORE, NOT_IMPLEMENTED, (NULL), ("unknown format"));
765 0 : return GST_FLOW_NOT_NEGOTIATED;
766 0 : unknown_tensor:
767 0 : GST_ERROR_OBJECT (self, "Hit unknown_tensor");
768 0 : GST_ELEMENT_ERROR (self, CORE, NOT_IMPLEMENTED, (NULL),
769 : ("unknown format for tensor"));
770 0 : return GST_FLOW_NOT_NEGOTIATED;
771 0 : unknown_type:
772 0 : GST_ELEMENT_ERROR (self, CORE, NOT_IMPLEMENTED, (NULL),
773 : ("not implemented decoder mode"));
774 0 : return GST_FLOW_NOT_SUPPORTED;
775 : }
776 :
777 : /**
778 : * @brief configure tensor-srcpad cap from "proposed" cap.
779 : *
780 : * @trans ("this" pointer)
781 : * @direction (why do we need this?)
782 : * @caps sinkpad cap
783 : * @filter this element's cap (don't know specifically.)
784 : */
785 : static GstCaps *
786 2002 : gst_tensordec_transform_caps (GstBaseTransform * trans,
787 : GstPadDirection direction, GstCaps * caps, GstCaps * filter)
788 : {
789 : GstTensorDecoder *self;
790 : GstCaps *result;
791 :
792 2002 : self = GST_TENSOR_DECODER_CAST (trans);
793 :
794 : /* Not ready */
795 2002 : if (self->decoder == NULL && !self->is_custom)
796 0 : return NULL;
797 :
798 2002 : if (self->is_custom) {
799 12 : const decoder_custom_cb_s *ptr = NULL;
800 12 : if (self->option[0] == NULL) {
801 0 : nns_logw ("Tensor decoder custom option is not given.");
802 0 : return NULL;
803 : }
804 12 : self->custom.func = NULL;
805 12 : ptr = get_subplugin (NNS_CUSTOM_DECODER, self->option[0]);
806 12 : if (!ptr) {
807 0 : nns_logw ("Failed to find custom subplugin of the tensor_decoder");
808 0 : return NULL;
809 : }
810 12 : self->custom.func = ptr->func;
811 12 : self->custom.data = ptr->data;
812 : }
813 :
814 2002 : silent_debug (self, "Direction = %d\n", direction);
815 2002 : silent_debug_caps (self, caps, "from");
816 2002 : silent_debug_caps (self, filter, "filter");
817 :
818 2002 : if (direction == GST_PAD_SINK) {
819 : /** caps = sinkpad (other/tensor) return = srcpad (media) */
820 809 : GstStructure *s = gst_caps_get_structure (caps, 0);
821 809 : result = gst_tensordec_media_caps_from_structure (self, s);
822 1193 : } else if (direction == GST_PAD_SRC) {
823 : /** caps = srcpad (media) return = sinkpad (other/tensor) */
824 : /** @todo We may do more specific actions here */
825 1193 : result = gst_caps_from_string (CAPS_STRING);
826 : } else {
827 0 : g_assert (0); /* Internal logic error! */
828 : return NULL;
829 : }
830 :
831 2002 : if (filter && gst_caps_get_size (filter) > 0) {
832 : GstCaps *intersection;
833 :
834 : intersection =
835 18 : gst_caps_intersect_full (filter, result, GST_CAPS_INTERSECT_FIRST);
836 :
837 18 : gst_caps_unref (result);
838 18 : result = intersection;
839 : }
840 :
841 2002 : silent_debug_caps (self, result, "to");
842 :
843 2002 : GST_DEBUG_OBJECT (self, "Direction[%d] transformed %" GST_PTR_FORMAT
844 : " into %" GST_PTR_FORMAT, direction, caps, result);
845 2002 : return result;
846 : }
847 :
848 : /**
849 : * @brief fixate caps. required vmethod of BaseTransform
850 : */
851 : static GstCaps *
852 161 : gst_tensordec_fixate_caps (GstBaseTransform * trans,
853 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
854 : {
855 : GstTensorDecoder *self;
856 : GstCaps *supposed;
857 : GstCaps *result;
858 :
859 161 : self = GST_TENSOR_DECODER_CAST (trans);
860 :
861 161 : silent_debug_caps (self, caps, "from caps");
862 161 : silent_debug_caps (self, othercaps, "from othercaps");
863 :
864 161 : GST_DEBUG_OBJECT (self, "trying to fixate othercaps %" GST_PTR_FORMAT
865 : " based on caps %" GST_PTR_FORMAT, othercaps, caps);
866 :
867 : /**
868 : * In gst_tensordec_transform_caps, we have refused to specify caps
869 : * if the direction is GST_PAD_SRC. Thus, gstreamer shouldn't fixate
870 : * it with GST_PAD_SRC. If this happens, it's either internal logic
871 : * error or GST bug.
872 : */
873 161 : if (direction != GST_PAD_SINK) {
874 0 : ml_logf_stacktrace ("Invalid direction for tensor-decoder fixate caps.\n");
875 161 : return NULL;
876 : }
877 :
878 161 : if (gst_tensordec_configure (self, caps, othercaps)) {
879 : supposed =
880 161 : gst_tensordec_media_caps_from_tensor (self, &self->tensor_config);
881 : } else {
882 0 : GstStructure *s = gst_caps_get_structure (caps, 0);
883 0 : supposed = gst_tensordec_media_caps_from_structure (self, s);
884 : }
885 :
886 161 : result = gst_caps_intersect (othercaps, supposed);
887 161 : gst_caps_unref (supposed);
888 :
889 161 : if (gst_caps_is_empty (result)) {
890 0 : gst_caps_unref (result);
891 0 : result = othercaps;
892 : } else {
893 161 : gst_caps_unref (othercaps);
894 : }
895 :
896 161 : GST_DEBUG_OBJECT (self, "now fixating %" GST_PTR_FORMAT, result);
897 :
898 161 : result = gst_caps_make_writable (result);
899 161 : result = gst_caps_fixate (result);
900 :
901 161 : if (direction == GST_PAD_SINK) {
902 161 : if (gst_caps_is_subset (caps, result)) {
903 0 : gst_caps_replace (&result, caps);
904 : }
905 : }
906 161 : return result;
907 : }
908 :
909 : /**
910 : * @brief set caps. required vmethod of BaseTransform
911 : */
912 : static gboolean
913 146 : gst_tensordec_set_caps (GstBaseTransform * trans,
914 : GstCaps * incaps, GstCaps * outcaps)
915 : {
916 146 : GstTensorDecoder *self = GST_TENSOR_DECODER_CAST (trans);
917 :
918 146 : silent_debug_caps (self, incaps, "from incaps");
919 146 : silent_debug_caps (self, outcaps, "from outcaps");
920 :
921 146 : if (gst_tensordec_configure (self, incaps, outcaps)) {
922 292 : GstCaps *supposed = gst_tensordec_media_caps_from_tensor (self,
923 146 : &self->tensor_config);
924 :
925 : /** Check if outcaps ==equivalent== supposed */
926 146 : if (gst_caps_is_always_compatible (outcaps, supposed)) {
927 146 : self->negotiated = TRUE;
928 : } else {
929 0 : GST_ERROR_OBJECT (self,
930 : "This is not compatible with the supposed output pad cap");
931 : }
932 :
933 146 : gst_caps_unref (supposed);
934 : }
935 :
936 146 : return self->negotiated;
937 : }
938 :
939 : /**
940 : * @brief Tell the framework the required size of buffer based on the info of the other side pad. optional vmethod of BaseTransform
941 : *
942 : * This is called when non-ip mode is used.
943 : */
944 : static gboolean
945 373 : gst_tensordec_transform_size (GstBaseTransform * trans,
946 : GstPadDirection direction, GstCaps * caps, gsize size,
947 : GstCaps * othercaps, gsize * othersize)
948 : {
949 : GstTensorDecoder *self;
950 :
951 373 : if (direction == GST_PAD_SRC)
952 0 : return FALSE;
953 : /** @todo If direction = SRC, you may need different interpretation! */
954 373 : self = GST_TENSOR_DECODER_CAST (trans);
955 :
956 373 : g_assert (self->configured);
957 :
958 373 : if (!self->is_custom && self->decoder->getTransformSize)
959 126 : *othersize = self->decoder->getTransformSize (&self->plugin_data,
960 126 : &self->tensor_config, caps, size, othercaps, direction);
961 : else
962 247 : *othersize = 0;
963 :
964 373 : return TRUE;
965 : }
966 :
967 : /**
968 : * @brief Registers a callback for tensor_decoder custom condition
969 : * @return 0 if success. -ERRNO if error.
970 : */
971 : int
972 5 : nnstreamer_decoder_custom_register (const gchar * name,
973 : tensor_decoder_custom func, void *data)
974 : {
975 : decoder_custom_cb_s *ptr;
976 :
977 5 : g_return_val_if_fail (name && strlen (name), -EINVAL);
978 4 : g_return_val_if_fail (func, -EINVAL);
979 :
980 3 : if (!(ptr = g_try_new0 (decoder_custom_cb_s, 1)))
981 0 : return -ENOMEM;
982 :
983 3 : ptr->func = func;
984 3 : ptr->data = data;
985 :
986 3 : if (register_subplugin (NNS_CUSTOM_DECODER, name, ptr))
987 2 : return 0;
988 :
989 1 : g_free (ptr);
990 1 : return -EINVAL;
991 : }
992 :
993 : /**
994 : * @brief Unregisters a callback for tensor_decoder custom condition
995 : * @return 0 if success. -ERRNO if error.
996 : */
997 : int
998 4 : nnstreamer_decoder_custom_unregister (const gchar * name)
999 : {
1000 : decoder_custom_cb_s *ptr;
1001 :
1002 4 : ptr = (decoder_custom_cb_s *) get_subplugin (NNS_CUSTOM_DECODER, name);
1003 4 : if (!unregister_subplugin (NNS_CUSTOM_DECODER, name)) {
1004 2 : ml_loge ("Failed to unregister custom callback %s.", name);
1005 2 : return -EINVAL;
1006 : }
1007 2 : g_free (ptr);
1008 :
1009 2 : return 0;
1010 : }
|
