Line data Source code
1 : /**
2 : * GStreamer
3 : * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
4 : * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
5 : * Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>
6 : *
7 : * This library is free software; you can redistribute it and/or
8 : * modify it under the terms of the GNU Library General Public
9 : * License as published by the Free Software Foundation;
10 : * version 2.1 of the License.
11 : *
12 : * This library is distributed in the hope that it will be useful,
13 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : * Library General Public License for more details.
16 : *
17 : */
18 : /**
19 : * @file gsttensor_transform.c
20 : * @date 10 Jul 2018
21 : * @brief GStreamer plugin to transform tensor dimension or type
22 : * @see https://github.com/nnstreamer/nnstreamer
23 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
24 : * @bug This is NYI.
25 : *
26 : */
27 :
28 : /**
29 : * SECTION:element-tensor_transform
30 : *
31 : * A filter that transforms tensors dimension or type.
32 : * The input and output is always in the format of other/tensor or other/tensors.
33 : *
34 : * <refsect2>
35 : * <title>Example launch line</title>
36 : * |[
37 : * gst-launch -v -m fakesrc ! tensor_converter ! tensor_transform mode=dimchg option=0:2 ! fakesink silent=TRUE
38 : * ]|
39 : * <title>How to use dimchg</title>
40 : * |[
41 : * option=0:2 # Move 0th dim to 2nd dim. I.e., [a][H][W][C] ==> [a][C][H][W]
42 : * ]|
43 : * </refsect2>
44 : */
45 :
46 : #ifdef HAVE_CONFIG_H
47 : #include <config.h>
48 : #endif
49 :
50 : #include <string.h>
51 : #include <math.h>
52 : #include <nnstreamer_log.h>
53 : #include <nnstreamer_util.h>
54 : #include "gsttensor_transform.h"
55 :
56 : #ifdef HAVE_ORC
57 : #include "nnstreamer-orc.h"
58 : #endif
59 :
60 : /**
61 : * @brief Macro for debug mode.
62 : */
63 : #ifndef DBG
64 : #define DBG (!filter->silent)
65 : #endif
66 :
67 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_transform_debug);
68 : #define GST_CAT_DEFAULT gst_tensor_transform_debug
69 : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_MAKE ("{ static, flexible }")
70 : #define REGEX_DIMCHG_OPTION "^([0-9]|1[0-5]):([0-9]|1[0-5])$"
71 : #define REGEX_TYPECAST_OPTION "(^[u]?int(8|16|32|64)$|^float(16|32|64)$)"
72 : #define REGEX_TRANSPOSE_OPTION "^(?:([0-2]):(?!.*\\1)){3}3$"
73 : #define REGEX_STAND_OPTION "^(default|dc-average)(:([u]?int(8|16|32|64)|float(16|32|64)))?(,per-channel:(true|false))?$"
74 : #define REGEX_CLAMP_OPTION "^((([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?))):"\
75 : "((([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?)))$"
76 : #define REGEX_PADDING_OPTION "^((left|right|top|bottom|front|back):(\\d)(,)?)+(layout:(NCHW|NHWC))?$"
77 : #define REGEX_ARITH_OPTION "^(typecast:([u]?int(8|16|32|64)|float(16|32|64)),)?"\
78 : "(per-channel:(false|true@[0-9]+),)?"\
79 : "(((add|mul|div)(:([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?))+(@[0-9]+)?)(,|))+$"
80 :
81 : #define REGEX_ARITH_OPTION_TYPECAST "(typecast:([u]?int(8|16|32|64)|float(16|32|64)))"
82 :
83 : /**
84 : * @brief The transpose rank is fixed to 4.
85 : * This RANK does not affect other/tensors(s)'s NNS_TENSOR_RANK_LIMIT.
86 : */
87 : #define NNS_TENSOR_TRANSPOSE_RANK_LIMIT (4)
88 :
89 : /**
90 : * @brief The padding rank is fixed to 3.
91 : * This RANK does not affect other/tensors(s)'s NNS_TENSOR_RANK_LIMIT.
92 : */
93 : #define NNS_TENSOR_PADDING_RANK_LIMIT (3)
94 :
95 : /**
96 : * @brief tensor_transform properties
97 : */
98 : enum
99 : {
100 : PROP_0,
101 : PROP_SILENT,
102 : PROP_MODE,
103 : PROP_OPTION,
104 : PROP_ACCELERATION,
105 : PROP_APPLY,
106 : PROP_TRANSPOSE_RANK_LIMIT
107 : };
108 :
109 : /**
110 : * @brief Flag to set orc acceleration.
111 : */
112 : #ifdef HAVE_ORC
113 : #define DEFAULT_ACCELERATION TRUE
114 : #else
115 : #define DEFAULT_ACCELERATION FALSE
116 : #endif
117 :
118 : static const gchar *gst_tensor_transform_stand_string[] = {
119 : [STAND_DEFAULT] = "default",
120 : [STAND_DC_AVERAGE] = "dc-average",
121 : [STAND_END] = NULL
122 : };
123 :
124 : static const gchar *gst_tensor_transform_operator_string[] = {
125 : [GTT_OP_TYPECAST] = "typecast",
126 : [GTT_OP_ADD] = "add",
127 : [GTT_OP_MUL] = "mul",
128 : [GTT_OP_DIV] = "div",
129 : [GTT_OP_UNKNOWN] = NULL
130 : };
131 :
132 : /**
133 : * @brief The capabilities of the inputs
134 : */
135 : static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
136 : GST_PAD_SINK,
137 : GST_PAD_ALWAYS,
138 : GST_STATIC_CAPS (CAPS_STRING));
139 :
140 : /**
141 : * @brief The capabilities of the outputs
142 : */
143 : static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
144 : GST_PAD_SRC,
145 : GST_PAD_ALWAYS,
146 : GST_STATIC_CAPS (CAPS_STRING));
147 :
148 : #define gst_tensor_transform_parent_class parent_class
149 1841 : G_DEFINE_TYPE (GstTensorTransform, gst_tensor_transform,
150 : GST_TYPE_BASE_TRANSFORM);
151 :
152 : /* GObject vmethod implementations */
153 : static void gst_tensor_transform_set_property (GObject * object, guint prop_id,
154 : const GValue * value, GParamSpec * pspec);
155 : static void gst_tensor_transform_get_property (GObject * object, guint prop_id,
156 : GValue * value, GParamSpec * pspec);
157 : static void gst_tensor_transform_finalize (GObject * object);
158 :
159 : /* GstBaseTransformer vmethod implementations */
160 : static GstFlowReturn gst_tensor_transform_transform (GstBaseTransform * trans,
161 : GstBuffer * inbuf, GstBuffer * outbuf);
162 : static GstCaps *gst_tensor_transform_transform_caps (GstBaseTransform * trans,
163 : GstPadDirection direction, GstCaps * caps, GstCaps * filter);
164 : static GstCaps *gst_tensor_transform_fixate_caps (GstBaseTransform * trans,
165 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
166 : static gboolean gst_tensor_transform_set_caps (GstBaseTransform * trans,
167 : GstCaps * incaps, GstCaps * outcaps);
168 : static gboolean gst_tensor_transform_transform_size (GstBaseTransform * trans,
169 : GstPadDirection direction, GstCaps * caps, gsize size,
170 : GstCaps * othercaps, gsize * othersize);
171 :
172 : static gboolean gst_tensor_transform_convert_dimension (GstTensorTransform *
173 : filter, GstPadDirection direction, guint idx, const GstTensorInfo * in_info,
174 : GstTensorInfo * out_info);
175 :
176 : #define GST_TYPE_TENSOR_TRANSFORM_MODE (gst_tensor_transform_mode_get_type ())
177 : /**
178 : * @brief A private function to register GEnumValue array for the 'mode' property
179 : * to a GType and return it
180 : */
181 : static GType
182 105 : gst_tensor_transform_mode_get_type (void)
183 : {
184 : static GType mode_type = 0;
185 :
186 105 : if (mode_type == 0) {
187 : static GEnumValue mode_types[] = {
188 : {GTT_DIMCHG, "Mode for changing tensor dimensions, "
189 : "option=FROM_DIM:TO_DIM (with a regex, " REGEX_DIMCHG_OPTION
190 : ", where NNS_TENSOR_RANK_LIMIT is 16)",
191 : "dimchg"},
192 : {GTT_TYPECAST, "Mode for casting type of tensor, "
193 : "option=" REGEX_TYPECAST_OPTION, "typecast"},
194 : {GTT_ARITHMETIC, "Mode for arithmetic operations with tensor, "
195 : "option=[typecast:TYPE,][per-channel:(false|true@DIM),]add|mul|div:NUMBER[@CH_IDX], ...",
196 : "arithmetic"},
197 : {GTT_TRANSPOSE, "Mode for transposing shape of tensor, "
198 : "option=D1\':D2\':D3\':D4 (fixed to 3)",
199 : "transpose"},
200 : {GTT_STAND, "Mode for statistical standardization of tensor, "
201 : "option=(default|dc-average)[:TYPE][,per-channel:(false|true)]",
202 : "stand"},
203 : {GTT_CLAMP, "Mode for clamping all elements of tensor into the range, "
204 : "option=CLAMP_MIN:CLAMP_MAX",
205 : "clamp"},
206 : {GTT_PADDING, "Mode for padding of tensor, "
207 : "option=left|right|top|bottom|front|back:NUMBER[,layout:(NCHW|NHWC)]",
208 : "padding"},
209 : {GTT_UNKNOWN, "Unknown or not-implemented-yet mode",
210 : "unknown"},
211 : {0, NULL, NULL},
212 : };
213 :
214 105 : mode_type = g_enum_register_static ("gtt_mode_type", mode_types);
215 : }
216 :
217 105 : return mode_type;
218 : }
219 :
220 : /**
221 : * @brief initialize the tensor_transform's class
222 : */
223 : static void
224 105 : gst_tensor_transform_class_init (GstTensorTransformClass * klass)
225 : {
226 : GObjectClass *gobject_class;
227 : GstElementClass *gstelement_class;
228 : GstBaseTransformClass *trans_class;
229 :
230 105 : GST_DEBUG_CATEGORY_INIT (gst_tensor_transform_debug, "tensor_transform", 0,
231 : "Element to transforms tensor dimension or type");
232 :
233 105 : trans_class = (GstBaseTransformClass *) klass;
234 105 : gstelement_class = (GstElementClass *) trans_class;
235 105 : gobject_class = (GObjectClass *) gstelement_class;
236 :
237 105 : gobject_class->set_property = gst_tensor_transform_set_property;
238 105 : gobject_class->get_property = gst_tensor_transform_get_property;
239 105 : gobject_class->finalize = gst_tensor_transform_finalize;
240 :
241 105 : g_object_class_install_property (gobject_class, PROP_SILENT,
242 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output ?",
243 : FALSE, G_PARAM_READWRITE));
244 105 : g_object_class_install_property (gobject_class, PROP_MODE,
245 : g_param_spec_enum ("mode", "Mode", "Mode used for transforming tensor",
246 : GST_TYPE_TENSOR_TRANSFORM_MODE, GTT_UNKNOWN,
247 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
248 105 : g_object_class_install_property (gobject_class, PROP_OPTION,
249 : g_param_spec_string ("option", "Option",
250 : "Option for the tensor transform mode ?", "", G_PARAM_READWRITE));
251 105 : g_object_class_install_property (gobject_class, PROP_ACCELERATION,
252 : g_param_spec_boolean ("acceleration", "Acceleration", "Orc acceleration",
253 : DEFAULT_ACCELERATION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
254 105 : g_object_class_install_property (gobject_class, PROP_APPLY,
255 : g_param_spec_string ("apply", "Apply", "Select tensors to apply, "
256 : "separated with ',' in case of multiple tensors. Default to apply all tensors.",
257 : "", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
258 105 : g_object_class_install_property (gobject_class, PROP_TRANSPOSE_RANK_LIMIT,
259 : g_param_spec_uint ("transpose-rank-limit", "Transpose rank limit",
260 : "The rank limit of transpose, which varies per version of nnstreamer and may be lower than the global rank limit if it is over 4.",
261 : 0, NNS_TENSOR_RANK_LIMIT, NNS_TENSOR_TRANSPOSE_RANK_LIMIT,
262 : G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
263 :
264 105 : gst_element_class_set_details_simple (gstelement_class,
265 : "TensorTransform",
266 : "Filter/Tensor",
267 : "Transforms other/tensor dimensions for different models or frameworks",
268 : "MyungJoo Ham <myungjoo.ham@samsung.com>");
269 :
270 105 : gst_element_class_add_pad_template (gstelement_class,
271 : gst_static_pad_template_get (&src_factory));
272 105 : gst_element_class_add_pad_template (gstelement_class,
273 : gst_static_pad_template_get (&sink_factory));
274 : /* Refer: https://gstreamer.freedesktop.org/documentation/design/element-transform.html */
275 105 : trans_class->passthrough_on_same_caps = FALSE;
276 :
277 : /* Processing units */
278 105 : trans_class->transform = GST_DEBUG_FUNCPTR (gst_tensor_transform_transform);
279 :
280 : /* Negotiation units */
281 105 : trans_class->transform_caps =
282 105 : GST_DEBUG_FUNCPTR (gst_tensor_transform_transform_caps);
283 105 : trans_class->fixate_caps =
284 105 : GST_DEBUG_FUNCPTR (gst_tensor_transform_fixate_caps);
285 105 : trans_class->set_caps = GST_DEBUG_FUNCPTR (gst_tensor_transform_set_caps);
286 :
287 : /* Allocation units */
288 105 : trans_class->transform_size =
289 105 : GST_DEBUG_FUNCPTR (gst_tensor_transform_transform_size);
290 105 : }
291 :
292 : /**
293 : * @brief initialize the new element (G_DEFINE_TYPE requires this)
294 : * instantiate pads and add them to element
295 : * set pad callback functions
296 : * initialize instance structure
297 : */
298 : static void
299 254 : gst_tensor_transform_init (GstTensorTransform * filter)
300 : {
301 254 : filter->silent = TRUE;
302 254 : filter->mode = GTT_UNKNOWN;
303 254 : filter->option = NULL;
304 254 : filter->loaded = FALSE;
305 254 : filter->operators = NULL;
306 254 : filter->acceleration = DEFAULT_ACCELERATION;
307 254 : filter->apply = NULL;
308 :
309 254 : gst_tensors_config_init (&filter->in_config);
310 254 : gst_tensors_config_init (&filter->out_config);
311 254 : }
312 :
313 : /**
314 : * @brief Get the corresponding operator from the string value
315 : * @param[in] str The string value for the operator
316 : * @return corresponding operator for the string (GTT_OP_UNKNOWN for errors)
317 : */
318 : static tensor_transform_operator
319 175 : gst_tensor_transform_get_operator (const gchar * str)
320 : {
321 : int index;
322 :
323 175 : index = find_key_strv (gst_tensor_transform_operator_string, str);
324 :
325 175 : return (index < 0) ? GTT_OP_UNKNOWN : index;
326 : }
327 :
328 : /**
329 : * @brief Get the corresponding mode from the string value
330 : * @param[in] str The string value for the mode
331 : * @return corresponding mode for the string. STAND_END for errors
332 : */
333 : static tensor_transform_stand_mode
334 4 : gst_tensor_transform_get_stand_mode (const gchar * str)
335 : {
336 : int index;
337 :
338 4 : index = find_key_strv (gst_tensor_transform_stand_string, str);
339 :
340 4 : return (index < 0) ? STAND_END : index;
341 : }
342 :
343 : #ifndef FLOAT16_SUPPORT
344 : /**
345 : * @brief Generate error if float16 is required.
346 : */
347 : static void
348 0 : float16_not_supported (void)
349 : {
350 0 : ml_loge
351 : ("Tensor_transform does not support float16 operators. Apply -Denable-float16=true for meson build option if your architecture support float16. Note that tensor-transform's float16 is adhoc and does NOT perform good (slow!).\n");
352 0 : g_assert (0);
353 : }
354 : #endif
355 :
356 : #ifdef FLOAT16_SUPPORT
357 : /**
358 : * @brief Refrain from heavy operations on float16
359 : * @todo Remove this after applying SIMD or ORC
360 : */
361 : static void
362 : refrain_from_heavy_op_on_float16 (gulong n)
363 : {
364 : static int warned = 0;
365 : /* 1 million */
366 : if (n > 1000000) {
367 : if (warned)
368 : return;
369 : ml_logw
370 : ("Tensor_transform implementation for float16 does not support SIMD. Heavy tensor-transform operations of float16 is not recommended. Try to apply heavy ops with other types (e.g., float32) and convert it to float16 at the time when it's really needed.\n");
371 : warned = 1;
372 : }
373 : }
374 :
375 : /** @todo Make this use SIMD or ORC */
376 : #define _conv_to_f16(intype, o, i, n) \
377 : do { \
378 : float16 *op = (gpointer) (o); \
379 : intype *ip = (gpointer) (i); \
380 : gulong idx; \
381 : refrain_from_heavy_op_on_float16 (n); \
382 : for (idx = 0; idx < n; idx++) \
383 : *(op + idx) = (float16) *(ip + idx); \
384 : } while (0)
385 :
386 : /** @todo Make this use SIMD or ORC */
387 : #define _conv_from_f16_action(n, op, ip, otypename) \
388 : do { \
389 : gulong idx; \
390 : for (idx = 0; idx < n; idx++) \
391 : *(op + idx) = (otypename) *(ip + idx); \
392 : } while (0)
393 :
394 : /** @todo Make this use SIMD or ORC */
395 : #define _conv_from_f16(otype, o, i, n) \
396 : do { \
397 : float16 *ip = (gpointer) (i); \
398 : refrain_from_heavy_op_on_float16 (n); \
399 : switch (otype) { \
400 : case _NNS_INT32: { \
401 : int32_t *op = (gpointer) (o); \
402 : _conv_from_f16_action (n, op, ip, int32_t); \
403 : break; } \
404 : case _NNS_UINT32: { \
405 : uint32_t *op = (gpointer) (o); \
406 : _conv_from_f16_action (n, op, ip, uint32_t); \
407 : break; } \
408 : case _NNS_INT16: { \
409 : int16_t *op = (gpointer) (o); \
410 : _conv_from_f16_action (n, op, ip, int16_t); \
411 : break; } \
412 : case _NNS_UINT16: { \
413 : uint16_t *op = (gpointer) (o); \
414 : _conv_from_f16_action (n, op, ip, uint16_t); \
415 : break; } \
416 : case _NNS_INT8: { \
417 : int8_t *op = (gpointer) (o); \
418 : _conv_from_f16_action (n, op, ip, int8_t); \
419 : break; } \
420 : case _NNS_UINT8: { \
421 : uint8_t *op = (gpointer) (o); \
422 : _conv_from_f16_action (n, op, ip, uint8_t); \
423 : break; } \
424 : case _NNS_FLOAT64: { \
425 : double *op = (gpointer) (o); \
426 : _conv_from_f16_action (n, op, ip, double); \
427 : break; } \
428 : case _NNS_FLOAT32: { \
429 : float *op = (gpointer) (o); \
430 : _conv_from_f16_action (n, op, ip, float); \
431 : break; } \
432 : case _NNS_FLOAT16: { \
433 : float16 *op = (gpointer) (o); \
434 : _conv_from_f16_action (n, op, ip, float16); \
435 : break; } \
436 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (otype)); g_assert (0); \
437 : } \
438 : } while (0)
439 :
440 : /** @todo Make this use SIMD or ORC */
441 : #define _op_float16(i, n, v, op) \
442 : do { \
443 : gulong idx; \
444 : float16 *data_in = (float16 *) (i); \
445 : refrain_from_heavy_op_on_float16 (n); \
446 : switch (op) { \
447 : case GTT_OP_ADD: \
448 : for (idx = 0; idx < n; idx++) \
449 : data_in[idx] = data_in[idx] + (v); \
450 : break; \
451 : case GTT_OP_MUL: \
452 : for (idx = 0; idx < n; idx++) \
453 : data_in[idx] = data_in[idx] * (v); \
454 : break; \
455 : case GTT_OP_DIV: \
456 : for (idx = 0; idx < n; idx++) \
457 : data_in[idx] = data_in[idx] / (v); \
458 : break; \
459 : default: GST_ERROR_OBJECT (filter, "Unknown operator for float16: %d", op); break; \
460 : } \
461 : } while (0)
462 :
463 : #else /* ! FLOAT16_SUPPORT */
464 : #define _conv_to_f16(intype, o, i, n) do { float16_not_supported (); } while (0)
465 : #define _conv_from_f16(otype, o, i, n) do { float16_not_supported (); } while (0)
466 : #define _op_float16(i, n, v, op) do { float16_not_supported (); } while (0)
467 : #endif /* FLOAT16_SUPPORT */
468 :
469 : #ifdef HAVE_ORC
470 : /* define macros for orc */
471 : #define orc_func_conv(intype,outtype) nns_orc_conv_ ## intype ## _to_ ## outtype
472 : #define orc_func_add(intype) nns_orc_add_c_ ## intype
473 : #define orc_func_mul(intype) nns_orc_mul_c_ ## intype
474 : #define orc_func_div(intype) nns_orc_div_c_ ## intype
475 :
476 : #define orc_typecast_to(i,o,n,intype,otype,intypename) do { \
477 : switch (otype) { \
478 : case _NNS_INT32: orc_func_conv (intype, s32) ((gpointer) o, (gpointer) i, n); break; \
479 : case _NNS_UINT32: orc_func_conv (intype, u32) ((gpointer) o, (gpointer) i, n); break; \
480 : case _NNS_INT16: orc_func_conv (intype, s16) ((gpointer) o, (gpointer) i, n); break; \
481 : case _NNS_UINT16: orc_func_conv (intype, u16) ((gpointer) o, (gpointer) i, n); break; \
482 : case _NNS_INT8: orc_func_conv (intype, s8) ((gpointer) o, (gpointer) i, n); break; \
483 : case _NNS_UINT8: orc_func_conv (intype, u8) ((gpointer) o, (gpointer) i, n); break; \
484 : case _NNS_FLOAT64: orc_func_conv (intype, f64) ((gpointer) o, (gpointer) i, n); break; \
485 : case _NNS_FLOAT32: orc_func_conv (intype, f32) ((gpointer) o, (gpointer) i, n); break; \
486 : case _NNS_INT64: orc_func_conv (intype, s64) ((gpointer) o, (gpointer) i, n); break; \
487 : case _NNS_UINT64: orc_func_conv (intype, u64) ((gpointer) o, (gpointer) i, n); break; \
488 : case _NNS_FLOAT16: _conv_to_f16 (intypename, o, i, n); break; \
489 : default: GST_ERROR_OBJECT (filter, "Unsupported output type %d", otype); g_assert (0); break; \
490 : } \
491 : } while (0)
492 :
493 : #define orc_typecast(i,o,n,itype,otype) do { \
494 : switch (itype) { \
495 : case _NNS_INT32: orc_typecast_to (i, o, n, s32, otype, int32_t); break; \
496 : case _NNS_UINT32: orc_typecast_to (i, o, n, u32, otype, uint32_t); break; \
497 : case _NNS_INT16: orc_typecast_to (i, o, n, s16, otype, int16_t); break; \
498 : case _NNS_UINT16: orc_typecast_to (i, o, n, u16, otype, uint16_t); break; \
499 : case _NNS_INT8: orc_typecast_to (i, o, n, s8, otype, int8_t); break; \
500 : case _NNS_UINT8: orc_typecast_to (i, o, n, u8, otype, uint8_t); break; \
501 : case _NNS_FLOAT64: orc_typecast_to (i, o, n, f64, otype, double); break; \
502 : case _NNS_FLOAT32: orc_typecast_to (i, o, n, f32, otype, float); break; \
503 : case _NNS_INT64: orc_typecast_to (i, o, n, s64, otype, int64_t); break; \
504 : case _NNS_UINT64: orc_typecast_to (i, o, n, u64, otype, uint64_t); break; \
505 : case _NNS_FLOAT16: _conv_from_f16 (otype, o, i, n); break; \
506 : default: GST_ERROR_OBJECT (filter, "Unsupported input type %d", itype); g_assert (0); break; \
507 : } \
508 : } while (0)
509 :
510 : #define orc_typesize(size, type) do { \
511 : switch (type) { \
512 : case _NNS_INT32: size = sizeof(int32_t); break; \
513 : case _NNS_UINT32: size = sizeof(uint32_t); break; \
514 : case _NNS_INT16: size = sizeof(int16_t); break; \
515 : case _NNS_UINT16: size = sizeof(uint16_t); break; \
516 : case _NNS_INT8: size = sizeof(int8_t); break; \
517 : case _NNS_UINT8: size = sizeof(uint8_t); break; \
518 : case _NNS_FLOAT64: size = sizeof(double); break; \
519 : case _NNS_FLOAT32: size = sizeof(float); break; \
520 : case _NNS_INT64: size = sizeof(int64_t); break; \
521 : case _NNS_UINT64: size = sizeof(uint64_t); break; \
522 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", type); g_assert (0); break; \
523 : } \
524 : } while (0)
525 :
526 : #define orc_operator_func(i,n,v,opfunc,op) do { \
527 : switch ((v)->type) { \
528 : case _NNS_INT32: opfunc (s32) ((gpointer) i, (v)->data._int32_t, n); break; \
529 : case _NNS_UINT32: opfunc (u32) ((gpointer) i, (v)->data._uint32_t, n); break; \
530 : case _NNS_INT16: opfunc (s16) ((gpointer) i, (v)->data._int16_t, n); break; \
531 : case _NNS_UINT16: opfunc (u16) ((gpointer) i, (v)->data._uint16_t, n); break; \
532 : case _NNS_INT8: opfunc (s8) ((gpointer) i, (v)->data._int8_t, n); break; \
533 : case _NNS_UINT8: opfunc (u8) ((gpointer) i, (v)->data._uint8_t, n); break; \
534 : case _NNS_FLOAT64: opfunc (f64) ((gpointer) i, (v)->data._double, n); break; \
535 : case _NNS_FLOAT32: opfunc (f32) ((gpointer) i, (v)->data._float, n); break; \
536 : case _NNS_INT64: opfunc (s64) ((gpointer) i, (v)->data._int64_t, n); break; \
537 : case _NNS_UINT64: opfunc (u64) ((gpointer) i, (v)->data._uint64_t, n); break; \
538 : case _NNS_FLOAT16: _op_float16 (i, n, (v)->data._float16, op); break; \
539 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (v)->type); g_assert (0); break; \
540 : } \
541 : } while (0)
542 :
543 : #define orc_operator_div_loop(i,n,val,typename) do { \
544 : gsize idx_div; \
545 : typename *data_in = (typename *) (i); \
546 : for (idx_div = 0; idx_div < (n); ++idx_div) { \
547 : data_in[idx_div] = data_in[idx_div] / (val); \
548 : } \
549 : } while (0)
550 :
551 : #define orc_operator(i,n,v,op) do { \
552 : switch (op) { \
553 : case GTT_OP_ADD: orc_operator_func (i, n, v, orc_func_add, op); break; \
554 : case GTT_OP_MUL: orc_operator_func (i, n, v, orc_func_mul, op); break; \
555 : case GTT_OP_DIV: \
556 : switch ((v)->type) { \
557 : case _NNS_INT32: orc_operator_div_loop (i, n, (v)->data._int32_t, int32_t); break; \
558 : case _NNS_UINT32: orc_operator_div_loop (i, n, (v)->data._uint32_t, uint32_t); break; \
559 : case _NNS_INT16: orc_operator_div_loop (i, n, (v)->data._int16_t, int16_t); break; \
560 : case _NNS_UINT16: orc_operator_div_loop (i, n, (v)->data._uint16_t, uint16_t); break; \
561 : case _NNS_INT8: orc_operator_div_loop (i, n, (v)->data._int8_t, int8_t); break; \
562 : case _NNS_UINT8: orc_operator_div_loop (i, n, (v)->data._uint8_t, uint8_t); break; \
563 : case _NNS_FLOAT64: orc_func_div (f64) ((gpointer) i, (v)->data._double, n); break; \
564 : case _NNS_FLOAT32: orc_func_div (f32) ((gpointer) i, (v)->data._float, n); break; \
565 : case _NNS_INT64: orc_operator_div_loop (i, n, (v)->data._int64_t, int64_t); break; \
566 : case _NNS_UINT64: orc_operator_div_loop (i, n, (v)->data._uint64_t, uint64_t); break; \
567 : case _NNS_FLOAT16: _op_float16 (i, n, (v)->data._float16, op); break; \
568 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (v)->type); g_assert (0); break; \
569 : } \
570 : break; \
571 : default: GST_ERROR_OBJECT (filter, "Unknown operator %d", op); break; \
572 : } \
573 : } while (0)
574 : #endif /* HAVE_ORC */
575 :
576 : /**
577 : * @brief Macro for operator
578 : */
579 : #define handle_operator(d,v,oper,vtype) do { \
580 : switch (oper) { \
581 : case GTT_OP_ADD: \
582 : (d)->data._##vtype += (v)->data._##vtype; \
583 : break; \
584 : case GTT_OP_MUL: \
585 : (d)->data._##vtype *= (v)->data._##vtype; \
586 : break; \
587 : case GTT_OP_DIV: \
588 : if ((v)->data._##vtype == 0) { \
589 : GST_ERROR_OBJECT (filter, "Invalid state, denominator is 0."); \
590 : return FALSE; \
591 : } \
592 : (d)->data._##vtype /= (v)->data._##vtype; \
593 : break; \
594 : default: \
595 : GST_ERROR_OBJECT (filter, "Unknown operator %d", oper); \
596 : return FALSE; \
597 : } \
598 : } while (0)
599 :
600 : /**
601 : * @brief Handle operators for tensor value
602 : * @param filter "this" pointer
603 : * @param desc struct for tensor value
604 : * @param val struct for tensor value
605 : * @param op operator for given tensor value
606 : * @return TRUE if no error
607 : */
608 : static gboolean
609 61615 : gst_tensor_transform_do_operator (GstTensorTransform * filter,
610 : tensor_data_s * desc, const tensor_data_s * val,
611 : tensor_transform_operator op)
612 : {
613 61615 : g_return_val_if_fail (desc != NULL, FALSE);
614 61615 : g_return_val_if_fail (val != NULL, FALSE);
615 61615 : g_return_val_if_fail (desc->type == val->type, FALSE);
616 :
617 61615 : switch (desc->type) {
618 7755 : case _NNS_INT32:
619 7755 : handle_operator (desc, val, op, int32_t);
620 7755 : break;
621 7680 : case _NNS_UINT32:
622 7680 : handle_operator (desc, val, op, uint32_t);
623 7680 : break;
624 7680 : case _NNS_INT16:
625 7680 : handle_operator (desc, val, op, int16_t);
626 7680 : break;
627 7680 : case _NNS_UINT16:
628 7680 : handle_operator (desc, val, op, uint16_t);
629 7680 : break;
630 7680 : case _NNS_INT8:
631 7680 : handle_operator (desc, val, op, int8_t);
632 7680 : break;
633 7680 : case _NNS_UINT8:
634 7680 : handle_operator (desc, val, op, uint8_t);
635 7680 : break;
636 45 : case _NNS_FLOAT64:
637 45 : handle_operator (desc, val, op, double);
638 45 : break;
639 55 : case _NNS_FLOAT32:
640 55 : handle_operator (desc, val, op, float);
641 55 : break;
642 0 : case _NNS_FLOAT16:
643 : #ifdef FLOAT16_SUPPORT
644 : handle_operator (desc, val, op, float16);
645 : #else
646 0 : float16_not_supported ();
647 : #endif
648 0 : break;
649 7680 : case _NNS_INT64:
650 7680 : handle_operator (desc, val, op, int64_t);
651 7680 : break;
652 7680 : case _NNS_UINT64:
653 7680 : handle_operator (desc, val, op, uint64_t);
654 7680 : break;
655 0 : default:
656 0 : GST_ERROR_OBJECT (filter, "Unknown tensor type %d", desc->type);
657 0 : return FALSE;
658 : }
659 :
660 61615 : return TRUE;
661 : }
662 :
663 : /**
664 : * @brief Setup internal data (data_* in GstTensorTransform)
665 : * @param[in/out] filter "this" pointer. mode & option MUST BE set already.
666 : * @retval TRUE if OK or operation-skipped, FALSE if fatal-error.
667 : */
668 : static gboolean
669 541 : gst_tensor_transform_set_option_data (GstTensorTransform * filter)
670 : {
671 : gchar *filter_name;
672 541 : gboolean ret = FALSE;
673 :
674 541 : if (filter->mode == GTT_UNKNOWN || filter->option == NULL)
675 283 : return TRUE;
676 :
677 258 : filter_name = gst_object_get_name ((GstObject *) filter);
678 :
679 258 : switch (filter->mode) {
680 8 : case GTT_DIMCHG:
681 : {
682 8 : gchar **strv = NULL;
683 :
684 8 : if (!g_regex_match_simple (REGEX_DIMCHG_OPTION, filter->option,
685 : G_REGEX_CASELESS, 0)) {
686 3 : ml_loge
687 : ("%s: dimchg: \'%s\' is not valid option string: it should be in the form of IDX_DIM_FROM:IDX_DIM_TO: with a regex, "
688 : REGEX_DIMCHG_OPTION "\n", filter_name, filter->option);
689 3 : break;
690 : }
691 :
692 5 : strv = g_strsplit (filter->option, ":", 2);
693 :
694 5 : filter->data_dimchg.from = (int) g_ascii_strtoll (strv[0], NULL, 10);
695 5 : filter->data_dimchg.to = (int) g_ascii_strtoll (strv[1], NULL, 10);
696 5 : ret = filter->loaded = TRUE;
697 5 : g_strfreev (strv);
698 5 : break;
699 : }
700 117 : case GTT_TYPECAST:
701 : {
702 117 : if (g_regex_match_simple (REGEX_TYPECAST_OPTION, filter->option,
703 : G_REGEX_CASELESS, 0)) {
704 116 : filter->data_typecast.to = gst_tensor_get_type (filter->option);
705 116 : ret = filter->loaded = TRUE;
706 : } else {
707 1 : ml_loge
708 : ("%s: typecast: \'%s\' is not valid data type for tensor: data type of tensor should be one of %s\n",
709 : filter_name, filter->option, GST_TENSOR_TYPE_ALL);
710 : }
711 117 : break;
712 : }
713 90 : case GTT_ARITHMETIC:
714 : {
715 : gchar *str_option;
716 : gchar **str_operators;
717 : gchar **str_op;
718 : tensor_transform_operator_s *op_s;
719 : guint i, num_operators, num_op;
720 : GRegex *regex_option_tc;
721 :
722 90 : filter->data_arithmetic.out_type = _NNS_END;
723 90 : filter->data_arithmetic.per_channel_arith = FALSE;
724 :
725 90 : if (filter->operators) {
726 2 : GST_WARNING_OBJECT (filter,
727 : "There exists pre-defined operators (total %d), now reset these.",
728 : g_slist_length (filter->operators));
729 :
730 2 : g_slist_free_full (filter->operators, g_free);
731 2 : filter->operators = NULL;
732 : }
733 :
734 90 : regex_option_tc = g_regex_new (REGEX_ARITH_OPTION_TYPECAST,
735 : G_REGEX_CASELESS, 0, 0);
736 :
737 90 : if (!regex_option_tc) {
738 0 : GST_ERROR_OBJECT (filter,
739 : "arithmetic: failed to create a GRegex structure for %s\n",
740 : REGEX_ARITH_OPTION_TYPECAST);
741 0 : break;
742 : }
743 :
744 90 : if (g_regex_match_full (regex_option_tc, filter->option, -1,
745 : 1, 0, NULL, NULL)) {
746 2 : str_option = g_regex_replace (regex_option_tc, filter->option, -1, 1,
747 : "", 0, 0);
748 2 : ml_loge
749 : ("%s: arithmetic: [typecast:TYPE,] should be located at the first to prevent memory re-allocation: typecast(s) in the middle of \'%s\' will be ignored\n",
750 : filter_name, filter->option);
751 : } else {
752 176 : str_option = g_strdup (filter->option);
753 : }
754 90 : g_regex_unref (regex_option_tc);
755 :
756 90 : if (!g_regex_match_simple (REGEX_ARITH_OPTION, str_option,
757 : G_REGEX_CASELESS, 0)) {
758 13 : ml_loge
759 : ("%s: arithmetic: \'%s\' is not valid option string: it should be in the form of [typecast:TYPE,][per-channel:(false|true@DIM),]add|mul|div:NUMBER[@CH_IDX]..., ...\n",
760 : filter_name, str_option);
761 13 : g_free (str_option);
762 13 : break;
763 : }
764 77 : str_operators = g_strsplit (str_option, ",", -1);
765 77 : num_operators = g_strv_length (str_operators);
766 :
767 256 : for (i = 0; i < num_operators; ++i) {
768 179 : str_op = g_strsplit (str_operators[i], ":", -1);
769 179 : num_op = g_strv_length (str_op);
770 :
771 179 : if (str_op[0]) {
772 177 : gchar **values = g_strsplit (str_op[1], "@", -1);
773 177 : guint num_values = g_strv_length (values);
774 :
775 : /* check whether per-channel */
776 177 : if (g_ascii_strcasecmp (str_op[0], "per-channel") == 0) {
777 2 : if (num_values > 1 && g_ascii_strcasecmp (values[0], "true") == 0) {
778 2 : ml_logi
779 : ("Set per-channel for arithmetic and assume that %s-th dim is the channel",
780 : values[1]);
781 2 : filter->data_arithmetic.per_channel_arith = TRUE;
782 2 : filter->data_arithmetic.ch_dim =
783 2 : (guint) g_ascii_strtoull (values[1], NULL, 10);
784 : }
785 :
786 2 : g_strfreev (values);
787 2 : g_strfreev (str_op);
788 2 : continue;
789 : }
790 :
791 175 : op_s = g_new0 (tensor_transform_operator_s, 1);
792 175 : g_assert (op_s);
793 :
794 175 : op_s->op = gst_tensor_transform_get_operator (str_op[0]);
795 175 : op_s->applying_ch = -1; /* -1 means applying to all channels */
796 175 : switch (op_s->op) {
797 60 : case GTT_OP_TYPECAST:
798 60 : if (num_op > 1 && str_op[1]) {
799 60 : op_s->value.type = gst_tensor_get_type (values[0]);
800 60 : filter->data_arithmetic.out_type = op_s->value.type;
801 : } else {
802 0 : GST_WARNING_OBJECT (filter, "Invalid option for typecast %s",
803 : str_operators[i]);
804 0 : op_s->op = GTT_OP_UNKNOWN;
805 : }
806 60 : break;
807 115 : case GTT_OP_ADD:
808 : case GTT_OP_MUL:
809 : case GTT_OP_DIV:
810 115 : if (num_op > 1 && str_op[1]) {
811 : /* get operand */
812 115 : if (strchr (values[0], '.') || strchr (values[0], 'e') ||
813 115 : strchr (values[0], 'E')) {
814 : double val;
815 :
816 79 : val = g_ascii_strtod (values[0], NULL);
817 79 : gst_tensor_data_set (&op_s->value, _NNS_FLOAT64, &val);
818 : } else {
819 : int64_t val;
820 :
821 36 : val = g_ascii_strtoll (values[0], NULL, 10);
822 36 : gst_tensor_data_set (&op_s->value, _NNS_INT64, &val);
823 : }
824 :
825 115 : if (filter->data_arithmetic.per_channel_arith && num_values > 1) {
826 11 : op_s->applying_ch = g_ascii_strtoll (values[1], NULL, 10);
827 : }
828 :
829 : } else {
830 0 : GST_WARNING_OBJECT (filter,
831 : "Invalid option for arithmetic %s", str_operators[i]);
832 0 : op_s->op = GTT_OP_UNKNOWN;
833 : }
834 115 : break;
835 0 : default:
836 0 : GST_WARNING_OBJECT (filter, "Unknown operator %s", str_op[0]);
837 0 : break;
838 : }
839 :
840 : /* append operator */
841 175 : if (op_s->op != GTT_OP_UNKNOWN) {
842 175 : filter->operators = g_slist_append (filter->operators, op_s);
843 : } else {
844 0 : g_free (op_s);
845 : }
846 :
847 175 : g_strfreev (values);
848 : } else {
849 2 : GST_WARNING_OBJECT (filter, "Invalid option %s", str_operators[i]);
850 : }
851 :
852 177 : g_strfreev (str_op);
853 : }
854 :
855 77 : ret = filter->loaded = (filter->operators != NULL);
856 77 : g_strfreev (str_operators);
857 77 : g_free (str_option);
858 77 : break;
859 : }
860 20 : case GTT_TRANSPOSE:
861 : {
862 : int i;
863 20 : gchar **strv = NULL;
864 :
865 20 : if (!g_regex_match_simple (REGEX_TRANSPOSE_OPTION, filter->option,
866 : G_REGEX_CASELESS, 0)) {
867 4 : ml_loge
868 : ("%s: transpose: \'%s\' is not valid option string: it should be in the form of NEW_IDX_DIM0:NEW_IDX_DIM1:NEW_IDX_DIM2:3 (Now transpose mode's rank is fixed to 3. Note that the index of the last dim is always fixed to 3)\n",
869 : filter_name, filter->option);
870 4 : break;
871 : }
872 :
873 16 : strv = g_strsplit (filter->option, ":", NNS_TENSOR_TRANSPOSE_RANK_LIMIT);
874 80 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
875 64 : filter->data_transpose.trans_order[i] =
876 64 : (uint8_t) g_ascii_strtoull (strv[i], NULL, 10);
877 : }
878 :
879 16 : ret = filter->loaded = TRUE;
880 16 : g_strfreev (strv);
881 16 : break;
882 : }
883 9 : case GTT_STAND:
884 : {
885 9 : gchar **options = NULL;
886 : guint i, num_options;
887 :
888 9 : if (!g_regex_match_simple (REGEX_STAND_OPTION, filter->option,
889 : G_REGEX_CASELESS, 0)) {
890 5 : ml_loge
891 : ("%s: stand: \'%s\' is not a valid option string: it should be in the form of (default|dc-average)[:TYPE][,per-channel:(false|true)]\n",
892 : filter_name, filter->option);
893 5 : break;
894 : }
895 :
896 4 : filter->data_stand.out_type = _NNS_END;
897 4 : filter->data_stand.per_channel = FALSE;
898 :
899 4 : options = g_strsplit (filter->option, ",", -1);
900 4 : num_options = g_strv_length (options);
901 :
902 10 : for (i = 0; i < num_options; i++) {
903 6 : gchar **strv = g_strsplit (options[i], ":", -1);
904 :
905 6 : if (g_ascii_strcasecmp (strv[0], "default") == 0 ||
906 4 : g_ascii_strcasecmp (strv[0], "dc-average") == 0) {
907 4 : filter->data_stand.mode =
908 4 : gst_tensor_transform_get_stand_mode (strv[0]);
909 6 : if (g_strv_length (strv) > 1)
910 2 : filter->data_stand.out_type = gst_tensor_get_type (strv[1]);
911 2 : } else if (g_ascii_strcasecmp (strv[0], "per-channel") == 0) {
912 2 : if (g_strv_length (strv) > 1 &&
913 2 : g_ascii_strcasecmp (strv[1], "true") == 0)
914 2 : filter->data_stand.per_channel = TRUE;
915 : } else {
916 0 : filter->data_stand.mode = STAND_END;
917 0 : ml_logw ("Unknown option for stand mode: %s", strv[0]);
918 : }
919 :
920 6 : g_strfreev (strv);
921 : }
922 :
923 4 : g_strfreev (options);
924 4 : ret = filter->loaded = TRUE;
925 4 : break;
926 : }
927 10 : case GTT_CLAMP:
928 : {
929 10 : gchar **strv = NULL;
930 :
931 10 : if (!g_regex_match_simple (REGEX_CLAMP_OPTION, filter->option,
932 : G_REGEX_CASELESS, 0)) {
933 1 : ml_loge
934 : ("%s: clamp: \'%s\' is not valid option string: it should be in the form of [CLAMP_MIN:CLAMP_MAX]\n",
935 : filter_name, filter->option);
936 1 : break;
937 : }
938 :
939 9 : strv = g_strsplit (filter->option, ":", 2);
940 :
941 9 : filter->data_clamp.min = g_ascii_strtod (strv[0], NULL);
942 9 : if (errno == ERANGE) {
943 1 : ml_loge ("%s: clamp: CLAMP_MIN value has an invalid range\n",
944 : filter_name);
945 1 : g_strfreev (strv);
946 1 : break;
947 : }
948 8 : filter->data_clamp.max = g_ascii_strtod (strv[1], NULL);
949 8 : if (errno == ERANGE) {
950 1 : ml_loge ("%s: clamp: CLAMP_MAX value has an invalid range\n",
951 : filter_name);
952 1 : g_strfreev (strv);
953 1 : break;
954 : }
955 :
956 7 : g_strfreev (strv);
957 :
958 7 : if (filter->data_clamp.min > filter->data_clamp.max) {
959 2 : ml_loge ("%s: clamp: CLAMP_MIN is larger than CLAMP_MAX\n",
960 : filter_name);
961 2 : break;
962 : }
963 :
964 5 : ret = filter->loaded = TRUE;
965 5 : break;
966 : }
967 4 : case GTT_PADDING:
968 : {
969 4 : gchar **options = NULL;
970 : guint i, num_options;
971 :
972 4 : if (!g_regex_match_simple (REGEX_PADDING_OPTION, filter->option,
973 : G_REGEX_CASELESS, 0)) {
974 0 : ml_loge
975 : ("%s: padding: \'%s\' is not valid option string: it should be in the form of left|right|top|bottom|front|back:PADDING,[layout:(NCHW|NHWC)]\n",
976 : filter_name, filter->option);
977 0 : break;
978 : }
979 :
980 68 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++)
981 64 : filter->data_padding.pad[i] = 0;
982 4 : filter->data_padding.layout = _NNS_LAYOUT_ANY;
983 :
984 4 : options = g_strsplit (filter->option, ",", -1);
985 4 : num_options = g_strv_length (options);
986 :
987 23 : for (i = 0; i < num_options; i++) {
988 19 : gchar **strv = g_strsplit (options[i], ":", 2);
989 19 : if (g_ascii_strcasecmp (strv[0], "left") == 0) {
990 4 : filter->data_padding.pad[PADDING_LEFT] =
991 4 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
992 15 : } else if (g_ascii_strcasecmp (strv[0], "right") == 0) {
993 4 : filter->data_padding.pad[PADDING_RIGHT] =
994 4 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
995 11 : } else if (g_ascii_strcasecmp (strv[0], "top") == 0) {
996 3 : filter->data_padding.pad[PADDING_TOP] =
997 3 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
998 8 : } else if (g_ascii_strcasecmp (strv[0], "bottom") == 0) {
999 3 : filter->data_padding.pad[PADDING_BOTTOM] =
1000 3 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1001 5 : } else if (g_ascii_strcasecmp (strv[0], "front") == 0) {
1002 2 : filter->data_padding.pad[PADDING_FRONT] =
1003 2 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1004 3 : } else if (g_ascii_strcasecmp (strv[0], "back") == 0) {
1005 2 : filter->data_padding.pad[PADDING_BACK] =
1006 2 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1007 1 : } else if (g_ascii_strcasecmp (strv[0], "layout") == 0) {
1008 1 : if (g_ascii_strcasecmp (strv[1], "NHWC") == 0)
1009 1 : filter->data_padding.layout = _NNS_LAYOUT_NHWC;
1010 : else
1011 0 : filter->data_padding.layout = _NNS_LAYOUT_NCHW;
1012 : } else {
1013 0 : ml_logw ("Unknown option for padding mode: %s", strv[0]);
1014 : }
1015 19 : g_strfreev (strv);
1016 : }
1017 4 : g_strfreev (options);
1018 :
1019 4 : if (filter->data_padding.layout == _NNS_LAYOUT_NHWC) {
1020 1 : guint prev_left = filter->data_padding.pad[PADDING_LEFT],
1021 1 : prev_right = filter->data_padding.pad[PADDING_RIGHT];
1022 1 : filter->data_padding.pad[PADDING_LEFT] =
1023 1 : filter->data_padding.pad[PADDING_FRONT];
1024 1 : filter->data_padding.pad[PADDING_RIGHT] =
1025 1 : filter->data_padding.pad[PADDING_BACK];
1026 1 : filter->data_padding.pad[PADDING_FRONT] = prev_left;
1027 1 : filter->data_padding.pad[PADDING_BACK] = prev_right;
1028 : }
1029 :
1030 4 : ret = filter->loaded = TRUE;
1031 4 : break;
1032 : }
1033 0 : default:
1034 0 : GST_ERROR_OBJECT (filter, "Cannot identify mode\n");
1035 0 : ret = FALSE;
1036 : }
1037 :
1038 258 : g_free (filter_name);
1039 258 : return ret;
1040 : }
1041 :
1042 : /**
1043 : * @brief Set property (gst element vmethod)
1044 : */
1045 : static void
1046 572 : gst_tensor_transform_set_property (GObject * object, guint prop_id,
1047 : const GValue * value, GParamSpec * pspec)
1048 : {
1049 572 : GstTensorTransform *filter = GST_TENSOR_TRANSFORM (object);
1050 :
1051 572 : switch (prop_id) {
1052 1 : case PROP_SILENT:
1053 1 : filter->silent = g_value_get_boolean (value);
1054 1 : break;
1055 253 : case PROP_MODE:
1056 253 : filter->mode = g_value_get_enum (value);
1057 253 : gst_tensor_transform_set_option_data (filter);
1058 253 : break;
1059 257 : case PROP_OPTION:
1060 : {
1061 257 : gchar *backup_option = filter->option;
1062 257 : filter->option = g_value_dup_string (value);
1063 257 : if (gst_tensor_transform_set_option_data (filter)) {
1064 226 : silent_debug (filter, "Option = %s --> %s\n", backup_option,
1065 : filter->option);
1066 226 : g_free (backup_option);
1067 : } else {
1068 : /* ERROR! Revert the change! */
1069 31 : g_free (filter->option);
1070 31 : filter->option = backup_option;
1071 31 : gst_tensor_transform_set_option_data (filter);
1072 : }
1073 257 : break;
1074 : }
1075 58 : case PROP_ACCELERATION:
1076 : #ifdef HAVE_ORC
1077 58 : filter->acceleration = g_value_get_boolean (value);
1078 58 : silent_debug (filter, "acceleration = %d\n", filter->acceleration);
1079 : #else
1080 : GST_WARNING_OBJECT (filter, "Orc acceleration is not supported");
1081 : filter->acceleration = FALSE;
1082 : #endif
1083 58 : break;
1084 3 : case PROP_APPLY:
1085 : {
1086 : gint64 val;
1087 3 : const gchar *param = g_value_get_string (value);
1088 3 : gchar **strv = g_strsplit_set (param, ",", -1);
1089 3 : guint i, num = g_strv_length (strv);
1090 3 : gchar *endptr = NULL;
1091 :
1092 10 : for (i = 0; i < num; i++) {
1093 7 : errno = 0;
1094 7 : val = g_ascii_strtoll (strv[i], &endptr, 10);
1095 7 : if (errno == ERANGE || errno == EINVAL || (endptr == strv[i])) {
1096 0 : ml_loge ("Cannot convert string %s to a gint64 value", strv[i]);
1097 : }
1098 7 : filter->apply = g_list_append (filter->apply, GINT_TO_POINTER (val));
1099 : }
1100 3 : g_strfreev (strv);
1101 3 : break;
1102 : }
1103 0 : default:
1104 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1105 0 : break;
1106 : }
1107 572 : }
1108 :
1109 : /**
1110 : * @brief Get property (gst element vmethod)
1111 : */
1112 : static void
1113 41 : gst_tensor_transform_get_property (GObject * object, guint prop_id,
1114 : GValue * value, GParamSpec * pspec)
1115 : {
1116 41 : GstTensorTransform *filter = GST_TENSOR_TRANSFORM (object);
1117 :
1118 41 : switch (prop_id) {
1119 2 : case PROP_SILENT:
1120 2 : g_value_set_boolean (value, filter->silent);
1121 2 : break;
1122 1 : case PROP_MODE:
1123 1 : g_value_set_enum (value, filter->mode);
1124 1 : break;
1125 36 : case PROP_OPTION:
1126 36 : g_value_set_string (value, filter->option);
1127 36 : break;
1128 2 : case PROP_ACCELERATION:
1129 2 : g_value_set_boolean (value, filter->acceleration);
1130 2 : break;
1131 0 : case PROP_APPLY:
1132 : {
1133 : GList *list;
1134 : gchar *p;
1135 : GPtrArray *arr;
1136 : gchar **strings;
1137 :
1138 0 : if (filter->apply == NULL) {
1139 0 : g_value_set_string (value, "");
1140 0 : return;
1141 : }
1142 :
1143 0 : arr = g_ptr_array_new ();
1144 0 : for (list = filter->apply; list != NULL; list = list->next) {
1145 0 : g_ptr_array_add (arr, g_strdup_printf ("%i",
1146 0 : GPOINTER_TO_INT (list->data)));
1147 : }
1148 0 : g_ptr_array_add (arr, NULL);
1149 0 : strings = (gchar **) g_ptr_array_free (arr, FALSE);
1150 0 : p = g_strjoinv (",", strings);
1151 :
1152 0 : g_strfreev (strings);
1153 0 : g_value_take_string (value, p);
1154 0 : break;
1155 : }
1156 0 : case PROP_TRANSPOSE_RANK_LIMIT:
1157 0 : g_value_set_uint (value, NNS_TENSOR_TRANSPOSE_RANK_LIMIT);
1158 0 : break;
1159 0 : default:
1160 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1161 0 : break;
1162 : }
1163 : }
1164 :
1165 : /**
1166 : * @brief Function to finalize instance (gst element vmethod)
1167 : */
1168 : static void
1169 239 : gst_tensor_transform_finalize (GObject * object)
1170 : {
1171 : GstTensorTransform *filter;
1172 :
1173 239 : filter = GST_TENSOR_TRANSFORM (object);
1174 :
1175 239 : if (filter->option) {
1176 200 : g_free (filter->option);
1177 200 : filter->option = NULL;
1178 : }
1179 :
1180 239 : if (filter->operators) {
1181 60 : g_slist_free_full (filter->operators, g_free);
1182 60 : filter->operators = NULL;
1183 : }
1184 :
1185 239 : if (filter->apply) {
1186 3 : g_list_free (filter->apply);
1187 3 : filter->apply = NULL;
1188 : }
1189 :
1190 239 : G_OBJECT_CLASS (parent_class)->finalize (object);
1191 239 : }
1192 :
1193 : /**
1194 : * @brief subrouting for tensor-transform, "dimchg" case.
1195 : * @param[in/out] filter "this" pointer
1196 : * @param[in] in_info input tensor info
1197 : * @param[in] out_info output tensor info
1198 : * @param[in] inptr input tensor
1199 : * @param[out] outptr output tensor
1200 : * @return Gst flow status
1201 : */
1202 : static GstFlowReturn
1203 52 : gst_tensor_transform_dimchg (GstTensorTransform * filter,
1204 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1205 : const uint8_t * inptr, uint8_t * outptr)
1206 : {
1207 52 : uint32_t *fromDim = in_info->dimension;
1208 52 : uint32_t *toDim = out_info->dimension;
1209 52 : unsigned int from = filter->data_dimchg.from;
1210 52 : unsigned int to = filter->data_dimchg.to;
1211 : unsigned int i, j, k;
1212 52 : unsigned int loopLimit = 1;
1213 : gsize loopBlockSize, copyblocksize, copyblocklimit;
1214 :
1215 52 : if (from == to) {
1216 : /** Useless memcpy. Do not call this or @todo do "IP" operation */
1217 0 : nns_memcpy (outptr, inptr, gst_tensor_info_get_size (in_info));
1218 0 : GST_WARNING_OBJECT (filter,
1219 : "Calling tensor_transform with high memcpy overhead WITHOUT any effects! Check your stream whether you really need tensor_transform.\n");
1220 0 : return GST_FLOW_OK;
1221 : }
1222 :
1223 52 : g_assert (from < NNS_TENSOR_RANK_LIMIT);
1224 52 : g_assert (to < NNS_TENSOR_RANK_LIMIT);
1225 52 : g_assert (fromDim[from] == toDim[to]);
1226 :
1227 52 : loopBlockSize = copyblocksize = gst_tensor_get_element_size (in_info->type);
1228 52 : copyblocklimit = 1;
1229 :
1230 52 : if (from < to) {
1231 : /**
1232 : * Smaller-loop-ed a to larger-loop-ed b
1233 : * E.g., [N][H][W][c] (c:W:H:N) --> [N][c][H][W] (W:H:c:N)
1234 : *
1235 : * @todo CRITICAL-TODO: Optimize the performance!
1236 : */
1237 726 : for (i = NNS_TENSOR_RANK_LIMIT - 1; i > to; i--) {
1238 674 : if (toDim[i] == 0)
1239 620 : continue;
1240 54 : loopLimit *= toDim[i];
1241 : }
1242 :
1243 158 : for (i = 0; i < to; i++) {
1244 106 : if (toDim[i] == 0)
1245 0 : break;
1246 106 : loopBlockSize *= toDim[i];
1247 : }
1248 :
1249 52 : for (i = 0; i < from; i++) {
1250 0 : if (fromDim[i] == 0)
1251 0 : break;
1252 0 : copyblocksize *= fromDim[i];
1253 : }
1254 158 : for (i = 0; i < to; i++) {
1255 106 : if (toDim[i] == 0)
1256 0 : break;
1257 106 : copyblocklimit *= toDim[i];
1258 : }
1259 :
1260 228 : for (i = 0; i < loopLimit; i++) {
1261 : /* [i1][i2][...][iN][b][...] i = i1 x i2 x ... x iN */
1262 176 : uint8_t *destptr = outptr + loopBlockSize * toDim[to] * i;
1263 176 : const uint8_t *srcptr = inptr + loopBlockSize * toDim[to] * i;
1264 :
1265 744 : for (j = 0; j < toDim[to]; j++) {
1266 568 : uint8_t *j_destptr = destptr + loopBlockSize * j;
1267 121208 : for (k = 0; k < copyblocklimit; k++) {
1268 120640 : nns_memcpy (j_destptr + copyblocksize * k,
1269 : srcptr + k * copyblocksize * toDim[to] + j * copyblocksize,
1270 : copyblocksize);
1271 : }
1272 : }
1273 : }
1274 : } else {
1275 : /**
1276 : * Larger-loop-ed a to smaller-loop-ed b
1277 : * E.g., [N][c][H][W] (W:H:c:N) --> [N][H][W][c] (c:W:H:N)
1278 : * @todo NYI
1279 : */
1280 0 : ml_loge
1281 : ("tensor-transform/dimchg operation is not permitted if from >= to.\n");
1282 0 : return GST_FLOW_ERROR;
1283 : }
1284 :
1285 52 : return GST_FLOW_OK;
1286 : }
1287 :
1288 : /**
1289 : * @brief subrouting for tensor-transform, "typecast" case.
1290 : * @param[in/out] filter "this" pointer
1291 : * @param[in] in_info input tensor info
1292 : * @param[in] out_info output tensor info
1293 : * @param[in] inptr input tensor
1294 : * @param[out] outptr output tensor
1295 : * @return Gst flow status
1296 : */
1297 : static GstFlowReturn
1298 579 : gst_tensor_transform_typecast (GstTensorTransform * filter,
1299 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1300 : const uint8_t * inptr, uint8_t * outptr)
1301 : {
1302 : gulong i, num;
1303 : gsize in_element_size, out_element_size;
1304 :
1305 579 : num = gst_tensor_get_element_count (in_info->dimension);
1306 :
1307 : #ifdef HAVE_ORC
1308 579 : if (filter->acceleration) {
1309 457 : orc_typecast (inptr, outptr, num, in_info->type, out_info->type);
1310 457 : return GST_FLOW_OK;
1311 : }
1312 : #endif
1313 :
1314 122 : in_element_size = gst_tensor_get_element_size (in_info->type);
1315 122 : out_element_size = gst_tensor_get_element_size (out_info->type);
1316 :
1317 61772 : for (i = 0; i < num; ++i) {
1318 61650 : gst_tensor_data_raw_typecast (
1319 61650 : (gpointer) (inptr + in_element_size * i), in_info->type,
1320 61650 : (gpointer) (outptr + out_element_size * i), out_info->type);
1321 : }
1322 :
1323 122 : return GST_FLOW_OK;
1324 : }
1325 :
1326 : /**
1327 : * @brief subrouting for tensor-transform, "arithmetic" case.
1328 : * @param[in/out] filter "this" pointer
1329 : * @param[in] in_info input tensor info
1330 : * @param[in] out_info output tensor info
1331 : * @param[in] inptr input tensor
1332 : * @param[out] outptr output tensor
1333 : * @return Gst flow status
1334 : */
1335 : static GstFlowReturn
1336 21047 : gst_tensor_transform_arithmetic (GstTensorTransform * filter,
1337 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1338 : const uint8_t * inptr, uint8_t * outptr)
1339 : {
1340 : gulong i, num, j, ch;
1341 : gsize in_element_size, out_element_size;
1342 :
1343 : GSList *walk;
1344 : tensor_transform_operator_s *op_s;
1345 : tensor_data_s value;
1346 :
1347 21047 : num = gst_tensor_get_element_count (in_info->dimension);
1348 :
1349 : #ifdef HAVE_ORC
1350 21047 : if (filter->acceleration) {
1351 20947 : walk = filter->operators;
1352 : /**
1353 : * Typecast should be called at the first.
1354 : * Do the typecast. If in/out type is same, this will copy the input array to output.
1355 : */
1356 20947 : orc_typecast (inptr, outptr, num, in_info->type, out_info->type);
1357 :
1358 20947 : if (!filter->data_arithmetic.per_channel_arith) {
1359 64021 : while (walk) {
1360 43075 : op_s = (tensor_transform_operator_s *) walk->data;
1361 :
1362 43075 : if (op_s->op != GTT_OP_TYPECAST) {
1363 22207 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1364 23235 : orc_operator (outptr, num, &op_s->value, op_s->op);
1365 : }
1366 :
1367 43075 : walk = g_slist_next (walk);
1368 : }
1369 : } else {
1370 1 : gsize typesize = 0;
1371 1 : guint ch_dim = filter->data_arithmetic.ch_dim;
1372 1 : gsize ch_offset, ch_size = 1;
1373 1 : uint8_t *tmp_outptr = NULL;
1374 :
1375 1 : for (i = 0; i < ch_dim; ++i) {
1376 0 : ch_size *= in_info->dimension[i];
1377 : }
1378 1 : ch_offset = ch_size * in_info->dimension[ch_dim];
1379 1 : orc_typesize (typesize, out_info->type);
1380 :
1381 2 : while (walk) {
1382 1 : op_s = (tensor_transform_operator_s *) walk->data;
1383 1 : if (op_s->op == GTT_OP_TYPECAST) {
1384 0 : walk = g_slist_next (walk);
1385 0 : continue;
1386 : }
1387 :
1388 1 : if (op_s->applying_ch == -1) {
1389 0 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1390 0 : orc_operator (outptr, num, &op_s->value, op_s->op);
1391 : } else {
1392 33 : for (i = 0; i < num / ch_offset; ++i) {
1393 32 : tmp_outptr =
1394 32 : outptr + (ch_size * op_s->applying_ch +
1395 32 : ch_offset * i) * typesize;
1396 32 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1397 32 : orc_operator (tmp_outptr, ch_size, &op_s->value, op_s->op);
1398 : }
1399 : }
1400 1 : walk = g_slist_next (walk);
1401 : }
1402 : }
1403 21047 : return GST_FLOW_OK;
1404 : }
1405 : #endif
1406 :
1407 100 : in_element_size = gst_tensor_get_element_size (in_info->type);
1408 100 : out_element_size = gst_tensor_get_element_size (out_info->type);
1409 :
1410 : /* per-channel */
1411 100 : if (filter->data_arithmetic.per_channel_arith) {
1412 3 : guint ch_dim = filter->data_arithmetic.ch_dim;
1413 3 : gsize ch_offset, ch_size = 1;
1414 3 : for (i = 0; i < ch_dim; ++i) {
1415 0 : ch_size *= in_info->dimension[i];
1416 : }
1417 3 : ch_offset = ch_size * in_info->dimension[ch_dim];
1418 :
1419 : /** In case of 3:4:4:1,
1420 : * ch_dim:0 -> #ch: 3, ch_size: 1, ch_offset: 3
1421 : * ch_dim:1 -> #ch: 4, ch_size: 3, ch_offset: 12
1422 : * ch_dim:2 -> #ch: 4, ch_size: 12, ch_offset: 48
1423 : * ch_dim:3 -> #ch: 1, ch_size: 48, ch_offset: 48 * 4
1424 : */
1425 :
1426 6 : for (i = 0; i < num / ch_offset; ++i) {
1427 18 : for (ch = 0; ch < in_info->dimension[ch_dim]; ++ch) {
1428 30 : for (j = 0; j < ch_size; ++j) {
1429 15 : gulong data_idx = (i * ch_offset) + (ch * ch_size) + j;
1430 15 : gst_tensor_data_set (&value, in_info->type,
1431 15 : (gpointer) (inptr + in_element_size * data_idx));
1432 :
1433 15 : walk = filter->operators;
1434 180 : while (walk) {
1435 165 : op_s = (tensor_transform_operator_s *) walk->data;
1436 165 : switch (op_s->op) {
1437 15 : case GTT_OP_TYPECAST:
1438 15 : gst_tensor_data_typecast (&value, op_s->value.type);
1439 15 : break;
1440 150 : case GTT_OP_ADD:
1441 : case GTT_OP_MUL:
1442 : case GTT_OP_DIV:
1443 : {
1444 150 : gst_tensor_data_typecast (&op_s->value, value.type);
1445 :
1446 150 : if (op_s->applying_ch == (int) ch || op_s->applying_ch == -1) {
1447 30 : gst_tensor_transform_do_operator (filter, &value,
1448 30 : &op_s->value, op_s->op);
1449 : }
1450 150 : break;
1451 : }
1452 0 : default:
1453 0 : g_assert (0);
1454 : return GST_FLOW_ERROR;
1455 : }
1456 :
1457 165 : walk = g_slist_next (walk);
1458 : }
1459 :
1460 : /* set output value */
1461 15 : g_assert (out_info->type == value.type);
1462 15 : gst_tensor_data_get (&value, outptr + out_element_size * data_idx);
1463 : }
1464 : }
1465 : }
1466 :
1467 3 : return GST_FLOW_OK;
1468 : }
1469 :
1470 61622 : for (i = 0; i < num; ++i) {
1471 : /* init value with input tensor type */
1472 61525 : gst_tensor_data_set (&value, in_info->type,
1473 61525 : (gpointer) (inptr + in_element_size * i));
1474 :
1475 61525 : walk = filter->operators;
1476 184595 : while (walk) {
1477 123070 : op_s = (tensor_transform_operator_s *) walk->data;
1478 :
1479 : /**
1480 : * @todo add more options
1481 : */
1482 123070 : switch (op_s->op) {
1483 61485 : case GTT_OP_TYPECAST:
1484 61485 : gst_tensor_data_typecast (&value, op_s->value.type);
1485 61485 : break;
1486 61585 : case GTT_OP_ADD:
1487 : case GTT_OP_MUL:
1488 : case GTT_OP_DIV:
1489 61585 : gst_tensor_data_typecast (&op_s->value, value.type);
1490 61585 : gst_tensor_transform_do_operator (filter, &value, &op_s->value,
1491 : op_s->op);
1492 61585 : break;
1493 0 : default:
1494 0 : g_assert (0);
1495 : return GST_FLOW_ERROR;
1496 : }
1497 :
1498 123070 : walk = g_slist_next (walk);
1499 : }
1500 :
1501 : /* set output value */
1502 61525 : g_assert (out_info->type == value.type);
1503 61525 : gst_tensor_data_get (&value, outptr + out_element_size * i);
1504 : }
1505 :
1506 97 : return GST_FLOW_OK;
1507 : }
1508 :
1509 : /**
1510 : * Macro to run loop for various data types with transpose
1511 : */
1512 : #define transposeloop(cl,ck,cj,ci,sl,sk,sj,si,typesize) do { \
1513 : size_t i, j, k, l; \
1514 : int inidx = 0, outidx=0; \
1515 : for(cl=0;cl<sl;cl++) \
1516 : for(ci=0;ci<si;ci++) \
1517 : for(cj=0;cj<sj;cj++) \
1518 : for(ck=0;ck<sk;ck++){ \
1519 : const uint8_t *_in; \
1520 : uint8_t *_out; \
1521 : outidx = si*sj*sk*cl + sj*sk*ci + sk*cj + ck; \
1522 : inidx = SK*SJ*SI*l + SJ*SI*k + SI*j + i; \
1523 : _in = inptr + inidx * typesize; \
1524 : _out = outptr + outidx * typesize; \
1525 : nns_memcpy(_out, _in, typesize); \
1526 : } \
1527 : } while(0);
1528 :
1529 : /**
1530 : * @brief subrouting for tensor-transform, "transpose" case.
1531 : * @param[in/out] filter "this" pointer
1532 : * @param[in] in_info input tensor info
1533 : * @param[in] out_info output tensor info
1534 : * @param[in] inptr input tensor
1535 : * @param[out] outptr output tensor
1536 : * @return Gst flow status
1537 : */
1538 : static GstFlowReturn
1539 58 : gst_tensor_transform_transpose (GstTensorTransform * filter,
1540 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1541 : const uint8_t * inptr, uint8_t * outptr)
1542 : {
1543 : int i, from, to;
1544 58 : gboolean checkdim = FALSE;
1545 58 : uint32_t *fromDim = in_info->dimension;
1546 58 : gsize type_size = gst_tensor_get_element_size (in_info->type);
1547 : gsize indexI, indexJ, SL, SI, SJ, SK;
1548 : UNUSED (out_info);
1549 :
1550 58 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
1551 58 : from = i;
1552 58 : to = filter->data_transpose.trans_order[i];
1553 58 : if (from != to) {
1554 58 : checkdim = TRUE;
1555 58 : break;
1556 : }
1557 : }
1558 :
1559 58 : if (!checkdim) {
1560 0 : nns_memcpy (outptr, inptr, gst_tensor_info_get_size (in_info));
1561 0 : GST_WARNING_OBJECT (filter,
1562 : "Calling tensor_transform with high memcpy overhead WITHOUT any effects!");
1563 0 : return GST_FLOW_OK;
1564 : }
1565 :
1566 58 : indexI = filter->data_transpose.trans_order[0];
1567 58 : indexJ = filter->data_transpose.trans_order[1];
1568 58 : SL = fromDim[3] > 0 ? fromDim[3] : 1;
1569 58 : SI = fromDim[0] > 0 ? fromDim[0] : 1;
1570 58 : SJ = fromDim[1] > 0 ? fromDim[1] : 1;
1571 58 : SK = fromDim[2] > 0 ? fromDim[2] : 1;
1572 :
1573 58 : switch (indexI) {
1574 0 : case 0:
1575 0 : if (indexJ == 1) {
1576 0 : transposeloop (l, i, j, k, SL, SI, SJ, SK, type_size);
1577 : } else {
1578 0 : transposeloop (l, i, k, j, SL, SI, SK, SJ, type_size);
1579 : }
1580 0 : break;
1581 54 : case 1:
1582 54 : if (indexJ == 0) {
1583 242420 : transposeloop (l, j, i, k, SL, SJ, SI, SK, type_size);
1584 : } else {
1585 1231478 : transposeloop (l, j, k, i, SL, SJ, SK, SI, type_size);
1586 : }
1587 54 : break;
1588 4 : case 2:
1589 4 : if (indexJ == 0) {
1590 140258 : transposeloop (l, k, i, j, SL, SK, SI, SJ, type_size);
1591 : } else {
1592 0 : transposeloop (l, k, j, i, SL, SK, SJ, SI, type_size);
1593 : }
1594 4 : break;
1595 : }
1596 :
1597 58 : return GST_FLOW_OK;
1598 : }
1599 :
1600 : /**
1601 : * @brief subrouting for tensor-transform, "stand" case.
1602 : * : pixel = abs((pixel - average(tensor))/(std(tensor) + val))
1603 : * @param[in/out] filter "this" pointer
1604 : * @param[in] in_info input tensor info
1605 : * @param[in] out_info output tensor info
1606 : * @param[in] inptr input tensor
1607 : * @param[out] outptr output tensor
1608 : * @return Gst flow status
1609 : */
1610 : static GstFlowReturn
1611 16 : gst_tensor_transform_stand (GstTensorTransform * filter,
1612 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1613 : const uint8_t * inptr, uint8_t * outptr)
1614 : {
1615 16 : GstFlowReturn ret = GST_FLOW_OK;
1616 : gsize in_element_size, out_element_size, data_size, ch_size;
1617 : gulong i, num, data_idx, ch;
1618 : gdouble tmp, *average, *std;
1619 :
1620 16 : in_element_size = gst_tensor_get_element_size (in_info->type);
1621 16 : out_element_size = gst_tensor_get_element_size (out_info->type);
1622 16 : num = gst_tensor_get_element_count (in_info->dimension);
1623 :
1624 16 : data_size = gst_tensor_info_get_size (in_info);
1625 16 : ch_size = in_info->dimension[0];
1626 :
1627 : /* calc average and std */
1628 16 : average = std = NULL;
1629 16 : if (filter->data_stand.per_channel) {
1630 8 : gst_tensor_data_raw_average_per_channel ((gpointer) inptr, data_size,
1631 8 : in_info->type, in_info->dimension, &average);
1632 : /* calculate std only for default mode */
1633 8 : if (filter->data_stand.mode == STAND_DEFAULT)
1634 4 : gst_tensor_data_raw_std_per_channel ((gpointer) inptr, data_size,
1635 4 : in_info->type, in_info->dimension, average, &std);
1636 : } else {
1637 8 : gst_tensor_data_raw_average ((gpointer) inptr, data_size,
1638 : in_info->type, &average);
1639 : /* calculate std only for default mode */
1640 8 : if (filter->data_stand.mode == STAND_DEFAULT)
1641 4 : gst_tensor_data_raw_std ((gpointer) inptr, data_size, in_info->type,
1642 : average, &std);
1643 : }
1644 :
1645 16 : switch (filter->data_stand.mode) {
1646 8 : case STAND_DEFAULT:
1647 : {
1648 8 : if (!filter->data_stand.per_channel) {
1649 20004 : for (i = 0; i < num; i++) {
1650 20000 : data_idx = in_element_size * i;
1651 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1652 : in_info->type, &tmp, _NNS_FLOAT64);
1653 :
1654 20000 : tmp = fabs ((tmp - *average) / *std);
1655 :
1656 20000 : data_idx = out_element_size * i;
1657 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1658 20000 : (gpointer) (outptr + data_idx), out_info->type);
1659 : }
1660 : } else {
1661 204 : for (ch = 0; ch < ch_size; ++ch) {
1662 20200 : for (i = 0; i < num / ch_size; i++) {
1663 20000 : data_idx = in_element_size * ((i * ch_size) + ch);
1664 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1665 : in_info->type, &tmp, _NNS_FLOAT64);
1666 :
1667 20000 : tmp = fabs ((tmp - average[ch]) / std[ch]);
1668 :
1669 20000 : data_idx = out_element_size * ((i * ch_size) + ch);
1670 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1671 20000 : (gpointer) (outptr + data_idx), out_info->type);
1672 : }
1673 : }
1674 : }
1675 8 : break;
1676 : }
1677 8 : case STAND_DC_AVERAGE:
1678 : {
1679 8 : if (!filter->data_stand.per_channel) {
1680 20004 : for (i = 0; i < num; i++) {
1681 20000 : data_idx = in_element_size * i;
1682 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1683 : in_info->type, &tmp, _NNS_FLOAT64);
1684 :
1685 20000 : tmp -= *average;
1686 :
1687 20000 : data_idx = out_element_size * i;
1688 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1689 20000 : (gpointer) (outptr + data_idx), out_info->type);
1690 : }
1691 : } else {
1692 204 : for (ch = 0; ch < ch_size; ++ch) {
1693 20200 : for (i = 0; i < num / ch_size; i++) {
1694 20000 : data_idx = in_element_size * ((i * ch_size) + ch);
1695 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1696 : in_info->type, &tmp, _NNS_FLOAT64);
1697 :
1698 20000 : tmp -= average[ch];
1699 :
1700 20000 : data_idx = out_element_size * ((i * ch_size) + ch);
1701 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1702 20000 : (gpointer) (outptr + data_idx), out_info->type);
1703 : }
1704 : }
1705 : }
1706 8 : break;
1707 : }
1708 0 : default:
1709 0 : GST_ERROR_OBJECT (filter, "Cannot identify mode\n");
1710 0 : ret = GST_FLOW_ERROR;
1711 : }
1712 :
1713 16 : g_free (average);
1714 16 : g_free (std);
1715 :
1716 16 : return ret;
1717 : }
1718 :
1719 : /**
1720 : * @brief subrouting for tensor-transform, "clamp" case.
1721 : * : pixel = if (pixel > max) ? max :
1722 : * if (pixel < min) ? min : pixel
1723 : * @param[in/out] filter "this" pointer
1724 : * @param[in] in_info input tensor info
1725 : * @param[in] out_info output tensor info
1726 : * @param[in] inptr input tensor
1727 : * @param[out] outptr output tensor
1728 : * @return Gst flow status
1729 : */
1730 : static GstFlowReturn
1731 16 : gst_tensor_transform_clamp (GstTensorTransform * filter,
1732 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1733 : const uint8_t * inptr, uint8_t * outptr)
1734 : {
1735 : gsize in_element_size, out_element_size;
1736 : gulong i, num, data_idx;
1737 : gdouble tmp;
1738 :
1739 16 : in_element_size = gst_tensor_get_element_size (in_info->type);
1740 16 : out_element_size = gst_tensor_get_element_size (out_info->type);
1741 16 : num = gst_tensor_get_element_count (in_info->dimension);
1742 :
1743 606016 : for (i = 0; i < num; ++i) {
1744 606000 : data_idx = in_element_size * i;
1745 606000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), in_info->type,
1746 : &tmp, _NNS_FLOAT64);
1747 :
1748 606000 : tmp = CLAMP (tmp, filter->data_clamp.min, filter->data_clamp.max);
1749 :
1750 606000 : data_idx = out_element_size * i;
1751 606000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, outptr + data_idx,
1752 : out_info->type);
1753 : }
1754 :
1755 16 : return GST_FLOW_OK;
1756 : }
1757 :
1758 : /**
1759 : * @brief subrouting for tensor-transform, "padding" case.
1760 : * @param[in/out] filter "this" pointer
1761 : * @param[in] in_info input tensor info
1762 : * @param[in] out_info output tensor info
1763 : * @param[in] inptr input tensor
1764 : * @param[out] outptr output tensor
1765 : * @return Gst flow status
1766 : */
1767 : static GstFlowReturn
1768 6 : gst_tensor_transform_padding (GstTensorTransform * filter,
1769 : GstTensorInfo * in_info, GstTensorInfo * out_info, const uint8_t * inptr,
1770 : uint8_t * outptr)
1771 : {
1772 : gsize element_size, in_loop_size, out_loop_size, copy_block_size;
1773 6 : guint i, j, k, left, top, front, loop_limit = 1;
1774 6 : element_size = gst_tensor_get_element_size (in_info->type);
1775 :
1776 6 : in_loop_size = (gsize) in_info->dimension[2] * in_info->dimension[1]
1777 6 : * in_info->dimension[0] * element_size;
1778 6 : out_loop_size =(gsize) out_info->dimension[2] * out_info->dimension[1]
1779 6 : * out_info->dimension[0] * element_size;
1780 6 : copy_block_size = in_info->dimension[0] * element_size;
1781 :
1782 9 : for (i = NNS_TENSOR_PADDING_RANK_LIMIT; i < NNS_TENSOR_RANK_LIMIT; i++) {
1783 9 : if (in_info->dimension[i] == 0)
1784 6 : break;
1785 3 : loop_limit *= in_info->dimension[i];
1786 : }
1787 :
1788 6 : left = filter->data_padding.pad[PADDING_LEFT];
1789 6 : top = filter->data_padding.pad[PADDING_TOP];
1790 6 : front = filter->data_padding.pad[PADDING_FRONT];
1791 :
1792 : /** @todo Add constant option instead of using zero padding value */
1793 6 : memset (outptr, 0, out_loop_size * loop_limit);
1794 :
1795 20 : for (i = 0; i < loop_limit; i++)
1796 56 : for (j = 0; j < in_info->dimension[2]; j++)
1797 2142 : for (k = 0; k < in_info->dimension[1]; k++) {
1798 2100 : guint in_idx = j * in_info->dimension[1] * in_info->dimension[0]
1799 2100 : + k * in_info->dimension[0];
1800 2100 : guint out_idx = j * out_info->dimension[1] * out_info->dimension[0]
1801 2100 : + k * out_info->dimension[0];
1802 :
1803 2100 : out_idx += left + top * out_info->dimension[0]
1804 2100 : + front * out_info->dimension[1] * out_info->dimension[0];
1805 :
1806 2100 : memcpy (outptr + out_idx * element_size + out_loop_size * i,
1807 2100 : inptr + in_idx * element_size + in_loop_size * i, copy_block_size);
1808 : }
1809 :
1810 6 : return GST_FLOW_OK;
1811 : }
1812 :
1813 : /**
1814 : * @brief non-ip transform. required vmethod for BaseTransform class.
1815 : * @param[in/out] trans "super" pointer
1816 : * @param[in] inbuf The input gst buffer
1817 : * @param[out] outbuf The output gst buffer
1818 : * @return Gst Flow Status
1819 : */
1820 : static GstFlowReturn
1821 21707 : gst_tensor_transform_transform (GstBaseTransform * trans,
1822 : GstBuffer * inbuf, GstBuffer * outbuf)
1823 : {
1824 : GstTensorTransform *filter;
1825 : GstTensorInfo *in_info, *out_info;
1826 21707 : GstFlowReturn res = GST_FLOW_ERROR;
1827 21707 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT] = { 0, };
1828 21707 : GstMemory *out_mem[NNS_TENSOR_SIZE_LIMIT] = { 0, };
1829 : GstMapInfo in_map[NNS_TENSOR_SIZE_LIMIT];
1830 : GstMapInfo out_map[NNS_TENSOR_SIZE_LIMIT];
1831 : uint8_t *inptr, *outptr;
1832 : guint i, num_tensors, num_mems;
1833 : gsize buf_size, hsize;
1834 : GstTensorMetaInfo meta;
1835 : GstTensorInfo in_flex_info, out_flex_info;
1836 : gboolean in_flexible, out_flexible;
1837 :
1838 21707 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
1839 :
1840 43410 : g_return_val_if_fail (filter->loaded, GST_FLOW_ERROR);
1841 21703 : inbuf = gst_tensor_buffer_from_config (inbuf, &filter->in_config);
1842 :
1843 21703 : in_flexible =
1844 21703 : gst_tensor_pad_caps_is_flexible (GST_BASE_TRANSFORM_SINK_PAD (trans));
1845 21703 : out_flexible =
1846 21703 : gst_tensor_pad_caps_is_flexible (GST_BASE_TRANSFORM_SRC_PAD (trans));
1847 :
1848 21703 : num_mems = gst_tensor_buffer_get_count (inbuf);
1849 21703 : if (in_flexible) {
1850 9 : num_tensors = num_mems;
1851 9 : g_return_val_if_fail (out_flexible, GST_FLOW_ERROR);
1852 : } else {
1853 21694 : num_tensors = filter->in_config.info.num_tensors;
1854 21694 : g_return_val_if_fail (num_mems == num_tensors, GST_FLOW_ERROR);
1855 : }
1856 :
1857 43492 : for (i = 0; i < num_tensors; i++) {
1858 21789 : in_info = gst_tensors_info_get_nth_info (&filter->in_config.info, i);
1859 21789 : out_info = gst_tensors_info_get_nth_info (&filter->out_config.info, i);
1860 :
1861 21789 : if (filter->apply && !g_list_find (filter->apply, GINT_TO_POINTER (i))) {
1862 15 : GstMemory *mem = gst_tensor_buffer_get_nth_memory (inbuf, i);
1863 :
1864 15 : if (!in_flexible && out_flexible) {
1865 0 : GstMemory *old = mem;
1866 :
1867 : /* append meta */
1868 0 : gst_tensor_info_convert_to_meta (out_info, &meta);
1869 0 : mem = gst_tensor_meta_info_append_header (&meta, old);
1870 0 : gst_memory_unref (old);
1871 : }
1872 :
1873 15 : gst_tensor_buffer_append_memory (outbuf, mem, out_info);
1874 15 : continue;
1875 : }
1876 :
1877 : /* parse input buffer */
1878 21774 : in_mem[i] = gst_tensor_buffer_get_nth_memory (inbuf, i);
1879 21774 : if (!gst_memory_map (in_mem[i], &in_map[i], GST_MAP_READ)) {
1880 0 : ml_loge ("Cannot map input buffer to gst-buf at tensor-transform.\n");
1881 0 : res = GST_FLOW_ERROR;
1882 0 : goto done;
1883 : }
1884 21774 : inptr = in_map[i].data;
1885 :
1886 21774 : if (in_flexible) {
1887 9 : in_info = &in_flex_info;
1888 9 : out_info = &out_flex_info;
1889 :
1890 9 : gst_tensor_meta_info_parse_header (&meta, inptr);
1891 : /** @todo max rank supported in tensor-transform is 4 */
1892 9 : if (!gst_tensor_meta_info_convert (&meta, in_info)) {
1893 0 : res = GST_FLOW_ERROR;
1894 0 : goto done;
1895 : }
1896 :
1897 9 : gst_tensor_transform_convert_dimension (filter, GST_PAD_SINK,
1898 : i, in_info, out_info);
1899 :
1900 9 : hsize = gst_tensor_meta_info_get_header_size (&meta);
1901 9 : inptr += hsize;
1902 : }
1903 :
1904 : /* prepare output buffer */
1905 21774 : buf_size = gst_tensor_info_get_size (out_info);
1906 21774 : if (out_flexible) {
1907 9 : gst_tensor_info_convert_to_meta (out_info, &meta);
1908 9 : hsize = gst_tensor_meta_info_get_header_size (&meta);
1909 9 : buf_size += hsize;
1910 : }
1911 :
1912 21774 : out_mem[i] = gst_allocator_alloc (NULL, buf_size, NULL);
1913 21774 : gst_tensor_buffer_append_memory (outbuf, out_mem[i], out_info);
1914 :
1915 21774 : if (!gst_memory_map (out_mem[i], &out_map[i], GST_MAP_WRITE)) {
1916 0 : ml_loge ("Cannot map output buffer to gst-buf at tensor-transform.\n");
1917 0 : res = GST_FLOW_ERROR;
1918 0 : goto done;
1919 : }
1920 21774 : outptr = out_map[i].data;
1921 :
1922 21774 : if (out_flexible) {
1923 9 : gst_tensor_meta_info_update_header (&meta, outptr);
1924 9 : outptr += hsize;
1925 : }
1926 :
1927 21774 : switch (filter->mode) {
1928 52 : case GTT_DIMCHG:
1929 52 : res = gst_tensor_transform_dimchg (filter, in_info, out_info,
1930 : inptr, outptr);
1931 52 : break;
1932 579 : case GTT_TYPECAST:
1933 579 : res = gst_tensor_transform_typecast (filter, in_info, out_info,
1934 : inptr, outptr);
1935 579 : break;
1936 21047 : case GTT_ARITHMETIC:
1937 21047 : res = gst_tensor_transform_arithmetic (filter, in_info, out_info,
1938 : inptr, outptr);
1939 21047 : break;
1940 58 : case GTT_TRANSPOSE:
1941 58 : res = gst_tensor_transform_transpose (filter, in_info, out_info,
1942 : inptr, outptr);
1943 58 : break;
1944 16 : case GTT_STAND:
1945 16 : res = gst_tensor_transform_stand (filter, in_info, out_info,
1946 : inptr, outptr);
1947 16 : break;
1948 16 : case GTT_CLAMP:
1949 16 : res = gst_tensor_transform_clamp (filter, in_info, out_info,
1950 : inptr, outptr);
1951 16 : break;
1952 6 : case GTT_PADDING:
1953 6 : res = gst_tensor_transform_padding (filter, in_info, out_info,
1954 : inptr, outptr);
1955 6 : break;
1956 0 : default:
1957 0 : ml_loge ("Not supported tensor transform mode");
1958 0 : res = GST_FLOW_NOT_SUPPORTED;
1959 0 : goto done;
1960 : }
1961 : }
1962 :
1963 21703 : done:
1964 43492 : for (i = 0; i < num_tensors; i++) {
1965 21789 : if (in_mem[i]) {
1966 21774 : gst_memory_unmap (in_mem[i], &in_map[i]);
1967 21774 : gst_memory_unref (in_mem[i]);
1968 : }
1969 21789 : if (out_mem[i])
1970 21774 : gst_memory_unmap (out_mem[i], &out_map[i]);
1971 : }
1972 :
1973 21703 : return res;
1974 : }
1975 :
1976 : /**
1977 : * @brief Dimension conversion calculation
1978 : * @param[in] filter "this" pointer
1979 : * @param[in] direction GST_PAD_SINK if input->output conv
1980 : * @param[in] idx index of the input tensors
1981 : * @param[in] in_info tensor info structure of source tensor (input if direction is SINK)
1982 : * @param[out] out_info tensor info structure of destination tensor (output if direction is SINK)
1983 : * @return TRUE if success
1984 : */
1985 : static gboolean
1986 3779 : gst_tensor_transform_convert_dimension (GstTensorTransform * filter,
1987 : GstPadDirection direction, guint idx, const GstTensorInfo * in_info,
1988 : GstTensorInfo * out_info)
1989 : {
1990 : guint i;
1991 :
1992 : /* copy input info first, then update output info */
1993 3779 : gst_tensor_info_copy (out_info, in_info);
1994 :
1995 3779 : if (filter->apply && !g_list_find (filter->apply, GINT_TO_POINTER (idx)))
1996 10 : return TRUE;
1997 :
1998 3769 : switch (filter->mode) {
1999 70 : case GTT_DIMCHG:
2000 : {
2001 70 : unsigned int from = filter->data_dimchg.from;
2002 70 : unsigned int to = filter->data_dimchg.to;
2003 :
2004 70 : if (direction == GST_PAD_SINK) {
2005 850 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) {
2006 800 : if ((i < from && i < to) || (i > from && i > to) || from == to) {
2007 634 : out_info->dimension[i] = in_info->dimension[i];
2008 166 : } else if (i == to) {
2009 50 : out_info->dimension[i] = in_info->dimension[from];
2010 116 : } else if (from > to) {
2011 0 : g_assert (i > 0 && i > to);
2012 0 : out_info->dimension[i] = in_info->dimension[i - 1];
2013 : } else {
2014 116 : g_assert (i < to && i < (NNS_TENSOR_RANK_LIMIT - 1));
2015 116 : out_info->dimension[i] = in_info->dimension[i + 1];
2016 : }
2017 : }
2018 : } else {
2019 340 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) {
2020 320 : if ((i < from && i < to) || (i > from && i > to) || from == to) {
2021 252 : out_info->dimension[i] = in_info->dimension[i];
2022 68 : } else if (i == from) {
2023 20 : out_info->dimension[i] = in_info->dimension[to];
2024 48 : } else if (from > to) {
2025 0 : g_assert (i < from && i < (NNS_TENSOR_RANK_LIMIT - 1));
2026 0 : out_info->dimension[i] = in_info->dimension[i + 1];
2027 : } else {
2028 48 : g_assert (i > 0 && i > from);
2029 48 : out_info->dimension[i] = in_info->dimension[i - 1];
2030 : }
2031 : }
2032 : }
2033 70 : break;
2034 : }
2035 2016 : case GTT_TYPECAST:
2036 : /** For both directions, dimension does not change */
2037 2016 : if (direction == GST_PAD_SINK) {
2038 : /** src = SINKPAD / dest = SRCPAD */
2039 1142 : out_info->type = filter->data_typecast.to;
2040 : } else {
2041 : /* cannot get the incoming data type on sink pad */
2042 874 : out_info->type = _NNS_END;
2043 : }
2044 2016 : break;
2045 :
2046 1030 : case GTT_ARITHMETIC:
2047 : /* check arith mode option has typecast operator */
2048 1030 : if (filter->data_arithmetic.out_type != _NNS_END) {
2049 788 : if (direction == GST_PAD_SINK) {
2050 419 : out_info->type = filter->data_arithmetic.out_type;
2051 : } else {
2052 : /* cannot get the incoming data type on sink pad */
2053 369 : out_info->type = _NNS_END;
2054 : }
2055 : }
2056 1030 : break;
2057 :
2058 281 : case GTT_TRANSPOSE:
2059 281 : if (direction == GST_PAD_SINK) {
2060 715 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
2061 572 : out_info->dimension[i] =
2062 572 : in_info->dimension[filter->data_transpose.trans_order[i]];
2063 : }
2064 : } else {
2065 690 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
2066 552 : g_assert (filter->data_transpose.trans_order[i] <
2067 : NNS_TENSOR_RANK_LIMIT);
2068 552 : out_info->dimension[filter->data_transpose.trans_order[i]] =
2069 552 : in_info->dimension[i];
2070 : }
2071 : }
2072 281 : break;
2073 :
2074 40 : case GTT_STAND:
2075 : /** For both directions, dimension does not change */
2076 40 : if (direction == GST_PAD_SINK) {
2077 24 : if (filter->data_stand.out_type != _NNS_END)
2078 12 : out_info->type = filter->data_stand.out_type;
2079 : } else {
2080 : /* cannot get the incoming data type on sink pad */
2081 16 : out_info->type = _NNS_END;
2082 : }
2083 40 : break;
2084 :
2085 136 : case GTT_CLAMP:
2086 : /* same tensors info, do nothing. */
2087 136 : break;
2088 :
2089 44 : case GTT_PADDING:
2090 44 : if (direction == GST_PAD_SINK) {
2091 30 : out_info->dimension[0] +=
2092 30 : filter->data_padding.pad[PADDING_LEFT] +
2093 30 : filter->data_padding.pad[PADDING_RIGHT];
2094 30 : out_info->dimension[1] +=
2095 30 : filter->data_padding.pad[PADDING_TOP] +
2096 30 : filter->data_padding.pad[PADDING_BOTTOM];
2097 30 : out_info->dimension[2] +=
2098 30 : filter->data_padding.pad[PADDING_FRONT] +
2099 30 : filter->data_padding.pad[PADDING_BACK];
2100 : }
2101 44 : break;
2102 152 : default:
2103 152 : return FALSE;
2104 : }
2105 :
2106 3617 : return TRUE;
2107 : }
2108 :
2109 : /**
2110 : * @brief configure srcpad cap from "proposed" cap. (required vmethod for BaseTransform)
2111 : *
2112 : * @param trans ("this" pointer)
2113 : * @param direction (why do we need this?)
2114 : * @param caps sinkpad cap
2115 : * @param filtercap this element's cap (don't know specifically.)
2116 : *
2117 : * @todo Get to know what the heck is @filtercap and use it!
2118 : */
2119 : static GstCaps *
2120 3172 : gst_tensor_transform_transform_caps (GstBaseTransform * trans,
2121 : GstPadDirection direction, GstCaps * caps, GstCaps * filtercap)
2122 : {
2123 : GstTensorTransform *filter;
2124 3172 : GstCaps *result = NULL;
2125 : GstStructure *structure;
2126 : guint i, j;
2127 :
2128 3172 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2129 :
2130 3172 : silent_debug (filter, "Calling TransformCaps, direction = %d\n", direction);
2131 3172 : silent_debug_caps (filter, caps, "from");
2132 3172 : silent_debug_caps (filter, filtercap, "filter");
2133 :
2134 3172 : result = gst_caps_new_empty ();
2135 9453 : for (i = 0; i < gst_caps_get_size (caps); i++) {
2136 : GstTensorsConfig in_config, out_config;
2137 : GstTensorInfo *in_info, *out_info;
2138 6281 : gboolean is_types_not_fixed = FALSE;
2139 6281 : GstCaps *result_aux = gst_caps_new_empty ();
2140 :
2141 6281 : gst_tensors_config_init (&out_config);
2142 :
2143 6281 : structure = gst_caps_get_structure (caps, i);
2144 6281 : gst_tensors_config_from_structure (&in_config, structure);
2145 :
2146 6281 : if (gst_tensors_config_is_flexible (&in_config)) {
2147 : /* output caps is also flexible */
2148 714 : out_config.info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
2149 : } else {
2150 9109 : for (j = 0; j < in_config.info.num_tensors; j++) {
2151 3542 : in_info = gst_tensors_info_get_nth_info (&in_config.info, j);
2152 3542 : out_info = gst_tensors_info_get_nth_info (&out_config.info, j);
2153 :
2154 3542 : gst_tensor_transform_convert_dimension (filter, direction,
2155 : j, in_info, out_info);
2156 3542 : if (out_info->type == _NNS_END) {
2157 : /* types cannot be specified */
2158 1819 : is_types_not_fixed = TRUE;
2159 : }
2160 : }
2161 : }
2162 :
2163 6281 : out_config.rate_d = in_config.rate_d;
2164 6281 : out_config.rate_n = in_config.rate_n;
2165 6281 : out_config.info.num_tensors = in_config.info.num_tensors;
2166 :
2167 6281 : if (gst_structure_has_name (structure, NNS_MIMETYPE_TENSOR)) {
2168 2960 : gst_caps_append (result_aux, gst_tensor_caps_from_config (&out_config));
2169 : } else {
2170 3321 : gst_caps_append (result_aux, gst_tensors_caps_from_config (&out_config));
2171 :
2172 : /* remove `types` field from caps */
2173 3321 : if (is_types_not_fixed) {
2174 154 : GstStructure *s = gst_caps_get_structure (result_aux, 0);
2175 154 : gst_structure_remove_field (s, "types");
2176 : }
2177 : }
2178 :
2179 6281 : gst_caps_append (result, result_aux);
2180 :
2181 6281 : gst_tensors_config_free (&in_config);
2182 6281 : gst_tensors_config_free (&out_config);
2183 : }
2184 :
2185 3172 : if (filtercap && gst_caps_get_size (filtercap) > 0) {
2186 : GstCaps *intersection;
2187 :
2188 : intersection =
2189 640 : gst_caps_intersect_full (result, filtercap, GST_CAPS_INTERSECT_FIRST);
2190 :
2191 640 : gst_caps_unref (result);
2192 640 : result = intersection;
2193 : }
2194 :
2195 3172 : silent_debug_caps (filter, result, "to");
2196 3172 : return result;
2197 : }
2198 :
2199 : /**
2200 : * @brief fixate caps. required vmethod of BaseTransform
2201 : */
2202 : static GstCaps *
2203 243 : gst_tensor_transform_fixate_caps (GstBaseTransform * trans,
2204 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
2205 : {
2206 : GstTensorTransform *filter;
2207 : GstCaps *result;
2208 :
2209 243 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2210 :
2211 243 : silent_debug (filter, "Calling FixateCaps, direction = %d\n", direction);
2212 243 : silent_debug_caps (filter, caps, "caps");
2213 243 : silent_debug_caps (filter, othercaps, "othercaps");
2214 :
2215 : result =
2216 243 : gst_tensor_transform_transform_caps (trans, direction, caps, othercaps);
2217 243 : gst_caps_unref (othercaps);
2218 :
2219 243 : result = gst_caps_make_writable (result);
2220 243 : result = gst_caps_fixate (result);
2221 :
2222 243 : silent_debug_caps (filter, result, "result");
2223 243 : return result;
2224 : }
2225 :
2226 : /**
2227 : * @brief set caps. required vmethod of BaseTransform
2228 : */
2229 : static gboolean
2230 221 : gst_tensor_transform_set_caps (GstBaseTransform * trans,
2231 : GstCaps * incaps, GstCaps * outcaps)
2232 : {
2233 : GstTensorTransform *filter;
2234 : GstTensorsConfig in_config, out_config;
2235 : GstTensorsConfig config;
2236 : GstTensorInfo *in_info, *out_info;
2237 : gboolean in_flexible, out_flexible;
2238 221 : gboolean allowed = FALSE;
2239 : guint i;
2240 :
2241 221 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2242 :
2243 221 : silent_debug (filter, "Calling SetCaps\n");
2244 221 : silent_debug_caps (filter, incaps, "incaps");
2245 221 : silent_debug_caps (filter, outcaps, "outcaps");
2246 :
2247 221 : if (!gst_tensors_config_from_caps (&in_config, incaps, TRUE)) {
2248 0 : GST_ERROR_OBJECT (filter, "Cannot read cap of incaps\n");
2249 0 : goto error;
2250 : }
2251 :
2252 221 : if (!gst_tensors_config_from_caps (&out_config, outcaps, TRUE)) {
2253 0 : GST_ERROR_OBJECT (filter, "Cannot read cap of outcaps\n");
2254 0 : goto error;
2255 : }
2256 :
2257 221 : in_flexible = gst_tensors_config_is_flexible (&in_config);
2258 221 : out_flexible = gst_tensors_config_is_flexible (&out_config);
2259 :
2260 : /* compare type and dimension */
2261 221 : gst_tensors_config_init (&config);
2262 221 : config.info.format = out_config.info.format;
2263 :
2264 221 : config.rate_n = in_config.rate_n;
2265 221 : config.rate_d = in_config.rate_d;
2266 221 : config.info.num_tensors = in_config.info.num_tensors;
2267 :
2268 221 : if (!in_flexible) {
2269 440 : for (i = 0; i < in_config.info.num_tensors; i++) {
2270 228 : in_info = gst_tensors_info_get_nth_info (&in_config.info, i);
2271 228 : out_info = gst_tensors_info_get_nth_info (&config.info, i);
2272 :
2273 228 : if (!gst_tensor_transform_convert_dimension (filter, GST_PAD_SINK,
2274 : i, in_info, out_info)) {
2275 0 : GST_ERROR_OBJECT (filter,
2276 : "Tensor info is not matched with given properties.");
2277 0 : goto error;
2278 : }
2279 : }
2280 : }
2281 :
2282 221 : if (out_flexible) {
2283 9 : GST_INFO_OBJECT (filter, "Output tensor is flexible.");
2284 :
2285 : /* set output configuration if input is static */
2286 9 : if (!in_flexible)
2287 0 : out_config = config;
2288 212 : } else if (!gst_tensors_config_is_equal (&out_config, &config)) {
2289 0 : GST_ERROR_OBJECT (filter,
2290 : "Tensor info is not matched with given properties.\n");
2291 0 : goto error;
2292 : }
2293 :
2294 : /* set in/out tensor info */
2295 221 : filter->in_config = in_config;
2296 221 : filter->out_config = out_config;
2297 221 : allowed = TRUE;
2298 :
2299 221 : error:
2300 221 : if (!allowed)
2301 0 : GST_ERROR_OBJECT (filter, "Set Caps Failed!\n");
2302 :
2303 221 : return allowed;
2304 : }
2305 :
2306 : /**
2307 : * @brief Tell the framework the required size of buffer based on the info of the other side pad. Note that this is always the same with the input. optional vmethod of BaseTransform
2308 : */
2309 : static gboolean
2310 21707 : gst_tensor_transform_transform_size (GstBaseTransform * trans,
2311 : GstPadDirection direction, GstCaps * caps, gsize size, GstCaps * othercaps,
2312 : gsize * othersize)
2313 : {
2314 : UNUSED (trans);
2315 : UNUSED (direction);
2316 : UNUSED (caps);
2317 : UNUSED (size);
2318 : UNUSED (othercaps);
2319 : /**
2320 : * Consider multi-tensors.
2321 : * Set each memory block in transform()
2322 : */
2323 21707 : *othersize = 0;
2324 :
2325 21707 : return TRUE;
2326 : }
|
