Line data Source code
1 : /**
2 : * GStreamer
3 : * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
4 : * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
5 : * Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>
6 : *
7 : * This library is free software; you can redistribute it and/or
8 : * modify it under the terms of the GNU Library General Public
9 : * License as published by the Free Software Foundation;
10 : * version 2.1 of the License.
11 : *
12 : * This library is distributed in the hope that it will be useful,
13 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : * Library General Public License for more details.
16 : *
17 : */
18 : /**
19 : * @file gsttensor_transform.c
20 : * @date 10 Jul 2018
21 : * @brief GStreamer plugin to transform tensor dimension or type
22 : * @see https://github.com/nnstreamer/nnstreamer
23 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
24 : * @bug This is NYI.
25 : *
26 : */
27 :
28 : /**
29 : * SECTION:element-tensor_transform
30 : *
31 : * A filter that transforms tensors dimension or type.
32 : * The input and output is always in the format of other/tensor or other/tensors.
33 : *
34 : * <refsect2>
35 : * <title>Example launch line</title>
36 : * |[
37 : * gst-launch -v -m fakesrc ! tensor_converter ! tensor_transform mode=dimchg option=0:2 ! fakesink silent=TRUE
38 : * ]|
39 : * <title>How to use dimchg</title>
40 : * |[
41 : * option=0:2 # Move 0th dim to 2nd dim. I.e., [a][H][W][C] ==> [a][C][H][W]
42 : * ]|
43 : * </refsect2>
44 : */
45 :
46 : #ifdef HAVE_CONFIG_H
47 : #include <config.h>
48 : #endif
49 :
50 : #include <string.h>
51 : #include <math.h>
52 : #include <nnstreamer_log.h>
53 : #include <nnstreamer_util.h>
54 : #include "gsttensor_transform.h"
55 :
56 : #ifdef HAVE_ORC
57 : #include "nnstreamer-orc.h"
58 : #endif
59 :
60 : /**
61 : * @brief Macro for debug mode.
62 : */
63 : #ifndef DBG
64 : #define DBG (!filter->silent)
65 : #endif
66 :
67 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_transform_debug);
68 : #define GST_CAT_DEFAULT gst_tensor_transform_debug
69 : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_MAKE ("{ static, flexible }")
70 : #define REGEX_DIMCHG_OPTION "^([0-9]|1[0-5]):([0-9]|1[0-5])$"
71 : #define REGEX_TYPECAST_OPTION "(^[u]?int(8|16|32|64)$|^float(16|32|64)$)"
72 : #define REGEX_TRANSPOSE_OPTION "^(?:([0-2]):(?!.*\\1)){3}3$"
73 : #define REGEX_STAND_OPTION "^(default|dc-average)(:([u]?int(8|16|32|64)|float(16|32|64)))?(,per-channel:(true|false))?$"
74 : #define REGEX_CLAMP_OPTION "^((([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?))):"\
75 : "((([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?)))$"
76 : #define REGEX_PADDING_OPTION "^((left|right|top|bottom|front|back):(\\d)(,)?)+(layout:(NCHW|NHWC))?$"
77 : #define REGEX_ARITH_OPTION "^(typecast:([u]?int(8|16|32|64)|float(16|32|64)),)?"\
78 : "(per-channel:(false|true@[0-9]+),)?"\
79 : "(((add|mul|div)(:([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?))+(@[0-9]+)?)(,|))+$"
80 :
81 : #define REGEX_ARITH_OPTION_TYPECAST "(typecast:([u]?int(8|16|32|64)|float(16|32|64)))"
82 :
83 : /**
84 : * @brief The transpose rank is fixed to 4.
85 : * This RANK does not affect other/tensors(s)'s NNS_TENSOR_RANK_LIMIT.
86 : */
87 : #define NNS_TENSOR_TRANSPOSE_RANK_LIMIT (4)
88 :
89 : /**
90 : * @brief The padding rank is fixed to 3.
91 : * This RANK does not affect other/tensors(s)'s NNS_TENSOR_RANK_LIMIT.
92 : */
93 : #define NNS_TENSOR_PADDING_RANK_LIMIT (3)
94 :
95 : /**
96 : * @brief tensor_transform properties
97 : */
98 : enum
99 : {
100 : PROP_0,
101 : PROP_SILENT,
102 : PROP_MODE,
103 : PROP_OPTION,
104 : PROP_ACCELERATION,
105 : PROP_APPLY,
106 : PROP_TRANSPOSE_RANK_LIMIT
107 : };
108 :
109 : /**
110 : * @brief Flag to set orc acceleration.
111 : */
112 : #ifdef HAVE_ORC
113 : #define DEFAULT_ACCELERATION TRUE
114 : #else
115 : #define DEFAULT_ACCELERATION FALSE
116 : #endif
117 :
118 : static const gchar *gst_tensor_transform_stand_string[] = {
119 : [STAND_DEFAULT] = "default",
120 : [STAND_DC_AVERAGE] = "dc-average",
121 : [STAND_END] = NULL
122 : };
123 :
124 : static const gchar *gst_tensor_transform_operator_string[] = {
125 : [GTT_OP_TYPECAST] = "typecast",
126 : [GTT_OP_ADD] = "add",
127 : [GTT_OP_MUL] = "mul",
128 : [GTT_OP_DIV] = "div",
129 : [GTT_OP_UNKNOWN] = NULL
130 : };
131 :
132 : /**
133 : * @brief The capabilities of the inputs
134 : */
135 : static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
136 : GST_PAD_SINK,
137 : GST_PAD_ALWAYS,
138 : GST_STATIC_CAPS (CAPS_STRING));
139 :
140 : /**
141 : * @brief The capabilities of the outputs
142 : */
143 : static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
144 : GST_PAD_SRC,
145 : GST_PAD_ALWAYS,
146 : GST_STATIC_CAPS (CAPS_STRING));
147 :
148 : #define gst_tensor_transform_parent_class parent_class
149 1811 : G_DEFINE_TYPE (GstTensorTransform, gst_tensor_transform,
150 : GST_TYPE_BASE_TRANSFORM);
151 :
152 : /* GObject vmethod implementations */
153 : static void gst_tensor_transform_set_property (GObject * object, guint prop_id,
154 : const GValue * value, GParamSpec * pspec);
155 : static void gst_tensor_transform_get_property (GObject * object, guint prop_id,
156 : GValue * value, GParamSpec * pspec);
157 : static void gst_tensor_transform_finalize (GObject * object);
158 :
159 : /* GstBaseTransformer vmethod implementations */
160 : static GstFlowReturn gst_tensor_transform_transform (GstBaseTransform * trans,
161 : GstBuffer * inbuf, GstBuffer * outbuf);
162 : static GstCaps *gst_tensor_transform_transform_caps (GstBaseTransform * trans,
163 : GstPadDirection direction, GstCaps * caps, GstCaps * filter);
164 : static GstCaps *gst_tensor_transform_fixate_caps (GstBaseTransform * trans,
165 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
166 : static gboolean gst_tensor_transform_set_caps (GstBaseTransform * trans,
167 : GstCaps * incaps, GstCaps * outcaps);
168 : static gboolean gst_tensor_transform_transform_size (GstBaseTransform * trans,
169 : GstPadDirection direction, GstCaps * caps, gsize size,
170 : GstCaps * othercaps, gsize * othersize);
171 :
172 : static gboolean gst_tensor_transform_convert_dimension (GstTensorTransform *
173 : filter, GstPadDirection direction, guint idx, const GstTensorInfo * in_info,
174 : GstTensorInfo * out_info);
175 :
176 : #define GST_TYPE_TENSOR_TRANSFORM_MODE (gst_tensor_transform_mode_get_type ())
177 : /**
178 : * @brief A private function to register GEnumValue array for the 'mode' property
179 : * to a GType and return it
180 : */
181 : static GType
182 95 : gst_tensor_transform_mode_get_type (void)
183 : {
184 : static GType mode_type = 0;
185 :
186 95 : if (mode_type == 0) {
187 : static GEnumValue mode_types[] = {
188 : {GTT_DIMCHG, "Mode for changing tensor dimensions, "
189 : "option=FROM_DIM:TO_DIM (with a regex, " REGEX_DIMCHG_OPTION
190 : ", where NNS_TENSOR_RANK_LIMIT is 16)",
191 : "dimchg"},
192 : {GTT_TYPECAST, "Mode for casting type of tensor, "
193 : "option=" REGEX_TYPECAST_OPTION, "typecast"},
194 : {GTT_ARITHMETIC, "Mode for arithmetic operations with tensor, "
195 : "option=[typecast:TYPE,][per-channel:(false|true@DIM),]add|mul|div:NUMBER[@CH_IDX], ...",
196 : "arithmetic"},
197 : {GTT_TRANSPOSE, "Mode for transposing shape of tensor, "
198 : "option=D1\':D2\':D3\':D4 (fixed to 3)",
199 : "transpose"},
200 : {GTT_STAND, "Mode for statistical standardization of tensor, "
201 : "option=(default|dc-average)[:TYPE][,per-channel:(false|true)]",
202 : "stand"},
203 : {GTT_CLAMP, "Mode for clamping all elements of tensor into the range, "
204 : "option=CLAMP_MIN:CLAMP_MAX",
205 : "clamp"},
206 : {GTT_PADDING, "Mode for padding of tensor, "
207 : "option=left|right|top|bottom|front|back:NUMBER[,layout:(NCHW|NHWC)]",
208 : "padding"},
209 : {GTT_UNKNOWN, "Unknown or not-implemented-yet mode",
210 : "unknown"},
211 : {0, NULL, NULL},
212 : };
213 :
214 95 : mode_type = g_enum_register_static ("gtt_mode_type", mode_types);
215 : }
216 :
217 95 : return mode_type;
218 : }
219 :
220 : /**
221 : * @brief initialize the tensor_transform's class
222 : */
223 : static void
224 95 : gst_tensor_transform_class_init (GstTensorTransformClass * klass)
225 : {
226 : GObjectClass *gobject_class;
227 : GstElementClass *gstelement_class;
228 : GstBaseTransformClass *trans_class;
229 :
230 95 : GST_DEBUG_CATEGORY_INIT (gst_tensor_transform_debug, "tensor_transform", 0,
231 : "Element to transforms tensor dimension or type");
232 :
233 95 : trans_class = (GstBaseTransformClass *) klass;
234 95 : gstelement_class = (GstElementClass *) trans_class;
235 95 : gobject_class = (GObjectClass *) gstelement_class;
236 :
237 95 : gobject_class->set_property = gst_tensor_transform_set_property;
238 95 : gobject_class->get_property = gst_tensor_transform_get_property;
239 95 : gobject_class->finalize = gst_tensor_transform_finalize;
240 :
241 95 : g_object_class_install_property (gobject_class, PROP_SILENT,
242 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output ?",
243 : FALSE, G_PARAM_READWRITE));
244 95 : g_object_class_install_property (gobject_class, PROP_MODE,
245 : g_param_spec_enum ("mode", "Mode", "Mode used for transforming tensor",
246 : GST_TYPE_TENSOR_TRANSFORM_MODE, GTT_UNKNOWN,
247 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
248 95 : g_object_class_install_property (gobject_class, PROP_OPTION,
249 : g_param_spec_string ("option", "Option",
250 : "Option for the tensor transform mode ?", "", G_PARAM_READWRITE));
251 95 : g_object_class_install_property (gobject_class, PROP_ACCELERATION,
252 : g_param_spec_boolean ("acceleration", "Acceleration", "Orc acceleration",
253 : DEFAULT_ACCELERATION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
254 95 : g_object_class_install_property (gobject_class, PROP_APPLY,
255 : g_param_spec_string ("apply", "Apply", "Select tensors to apply, "
256 : "separated with ',' in case of multiple tensors. Default to apply all tensors.",
257 : "", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
258 95 : g_object_class_install_property (gobject_class, PROP_TRANSPOSE_RANK_LIMIT,
259 : g_param_spec_uint ("transpose-rank-limit", "Transpose rank limit",
260 : "The rank limit of transpose, which varies per version of nnstreamer and may be lower than the global rank limit if it is over 4.",
261 : 0, NNS_TENSOR_RANK_LIMIT, NNS_TENSOR_TRANSPOSE_RANK_LIMIT,
262 : G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
263 :
264 95 : gst_element_class_set_details_simple (gstelement_class,
265 : "TensorTransform",
266 : "Filter/Tensor",
267 : "Transforms other/tensor dimensions for different models or frameworks",
268 : "MyungJoo Ham <myungjoo.ham@samsung.com>");
269 :
270 95 : gst_element_class_add_pad_template (gstelement_class,
271 : gst_static_pad_template_get (&src_factory));
272 95 : gst_element_class_add_pad_template (gstelement_class,
273 : gst_static_pad_template_get (&sink_factory));
274 : /* Refer: https://gstreamer.freedesktop.org/documentation/design/element-transform.html */
275 95 : trans_class->passthrough_on_same_caps = FALSE;
276 :
277 : /* Processing units */
278 95 : trans_class->transform = GST_DEBUG_FUNCPTR (gst_tensor_transform_transform);
279 :
280 : /* Negotiation units */
281 95 : trans_class->transform_caps =
282 95 : GST_DEBUG_FUNCPTR (gst_tensor_transform_transform_caps);
283 95 : trans_class->fixate_caps =
284 95 : GST_DEBUG_FUNCPTR (gst_tensor_transform_fixate_caps);
285 95 : trans_class->set_caps = GST_DEBUG_FUNCPTR (gst_tensor_transform_set_caps);
286 :
287 : /* Allocation units */
288 95 : trans_class->transform_size =
289 95 : GST_DEBUG_FUNCPTR (gst_tensor_transform_transform_size);
290 95 : }
291 :
292 : /**
293 : * @brief initialize the new element (G_DEFINE_TYPE requires this)
294 : * instantiate pads and add them to element
295 : * set pad callback functions
296 : * initialize instance structure
297 : */
298 : static void
299 254 : gst_tensor_transform_init (GstTensorTransform * filter)
300 : {
301 254 : filter->silent = TRUE;
302 254 : filter->mode = GTT_UNKNOWN;
303 254 : filter->option = NULL;
304 254 : filter->loaded = FALSE;
305 254 : filter->operators = NULL;
306 254 : filter->acceleration = DEFAULT_ACCELERATION;
307 254 : filter->apply = NULL;
308 :
309 254 : gst_tensors_config_init (&filter->in_config);
310 254 : gst_tensors_config_init (&filter->out_config);
311 254 : }
312 :
313 : /**
314 : * @brief Get the corresponding operator from the string value
315 : * @param[in] str The string value for the operator
316 : * @return corresponding operator for the string (GTT_OP_UNKNOWN for errors)
317 : */
318 : static tensor_transform_operator
319 175 : gst_tensor_transform_get_operator (const gchar * str)
320 : {
321 : int index;
322 :
323 175 : index = find_key_strv (gst_tensor_transform_operator_string, str);
324 :
325 175 : return (index < 0) ? GTT_OP_UNKNOWN : index;
326 : }
327 :
328 : /**
329 : * @brief Get the corresponding mode from the string value
330 : * @param[in] str The string value for the mode
331 : * @return corresponding mode for the string. STAND_END for errors
332 : */
333 : static tensor_transform_stand_mode
334 4 : gst_tensor_transform_get_stand_mode (const gchar * str)
335 : {
336 : int index;
337 :
338 4 : index = find_key_strv (gst_tensor_transform_stand_string, str);
339 :
340 4 : return (index < 0) ? STAND_END : index;
341 : }
342 :
343 : #ifndef FLOAT16_SUPPORT
344 : /**
345 : * @brief Generate error if float16 is required.
346 : */
347 : static void
348 0 : float16_not_supported (void)
349 : {
350 0 : ml_loge
351 : ("Tensor_transform does not support float16 operators. Apply -Denable-float16=true for meson build option if your architecture support float16. Note that tensor-transform's float16 is adhoc and does NOT perform good (slow!).\n");
352 0 : g_assert (0);
353 : }
354 : #endif
355 :
356 : #ifdef FLOAT16_SUPPORT
357 : /**
358 : * @brief Refrain from heavy operations on float16
359 : * @todo Remove this after applying SIMD or ORC
360 : */
361 : static void
362 : refrain_from_heavy_op_on_float16 (gulong n)
363 : {
364 : static int warned = 0;
365 : /* 1 million */
366 : if (n > 1000000) {
367 : if (warned)
368 : return;
369 : ml_logw
370 : ("Tensor_transform implementation for float16 does not support SIMD. Heavy tensor-transform operations of float16 is not recommended. Try to apply heavy ops with other types (e.g., float32) and convert it to float16 at the time when it's really needed.\n");
371 : warned = 1;
372 : }
373 : }
374 :
375 : /** @todo Make this use SIMD or ORC */
376 : #define _conv_to_f16(intype, o, i, n) \
377 : do { \
378 : float16 *op = (gpointer) (o); \
379 : intype *ip = (gpointer) (i); \
380 : gulong idx; \
381 : refrain_from_heavy_op_on_float16 (n); \
382 : for (idx = 0; idx < n; idx++) \
383 : *(op + idx) = (float16) *(ip + idx); \
384 : } while (0)
385 :
386 : /** @todo Make this use SIMD or ORC */
387 : #define _conv_from_f16_action(n, op, ip, otypename) \
388 : do { \
389 : gulong idx; \
390 : for (idx = 0; idx < n; idx++) \
391 : *(op + idx) = (otypename) *(ip + idx); \
392 : } while (0)
393 :
394 : /** @todo Make this use SIMD or ORC */
395 : #define _conv_from_f16(otype, o, i, n) \
396 : do { \
397 : float16 *ip = (gpointer) (i); \
398 : refrain_from_heavy_op_on_float16 (n); \
399 : switch (otype) { \
400 : case _NNS_INT32: { \
401 : int32_t *op = (gpointer) (o); \
402 : _conv_from_f16_action (n, op, ip, int32_t); \
403 : break; } \
404 : case _NNS_UINT32: { \
405 : uint32_t *op = (gpointer) (o); \
406 : _conv_from_f16_action (n, op, ip, uint32_t); \
407 : break; } \
408 : case _NNS_INT16: { \
409 : int16_t *op = (gpointer) (o); \
410 : _conv_from_f16_action (n, op, ip, int16_t); \
411 : break; } \
412 : case _NNS_UINT16: { \
413 : uint16_t *op = (gpointer) (o); \
414 : _conv_from_f16_action (n, op, ip, uint16_t); \
415 : break; } \
416 : case _NNS_INT8: { \
417 : int8_t *op = (gpointer) (o); \
418 : _conv_from_f16_action (n, op, ip, int8_t); \
419 : break; } \
420 : case _NNS_UINT8: { \
421 : uint8_t *op = (gpointer) (o); \
422 : _conv_from_f16_action (n, op, ip, uint8_t); \
423 : break; } \
424 : case _NNS_FLOAT64: { \
425 : double *op = (gpointer) (o); \
426 : _conv_from_f16_action (n, op, ip, double); \
427 : break; } \
428 : case _NNS_FLOAT32: { \
429 : float *op = (gpointer) (o); \
430 : _conv_from_f16_action (n, op, ip, float); \
431 : break; } \
432 : case _NNS_FLOAT16: { \
433 : float16 *op = (gpointer) (o); \
434 : _conv_from_f16_action (n, op, ip, float16); \
435 : break; } \
436 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (otype)); g_assert (0); \
437 : } \
438 : } while (0)
439 :
440 : /** @todo Make this use SIMD or ORC */
441 : #define _op_float16(i, n, v, op) \
442 : do { \
443 : gulong idx; \
444 : float16 *data_in = (float16 *) (i); \
445 : refrain_from_heavy_op_on_float16 (n); \
446 : switch (op) { \
447 : case GTT_OP_ADD: \
448 : for (idx = 0; idx < n; idx++) \
449 : data_in[idx] = data_in[idx] + (v); \
450 : break; \
451 : case GTT_OP_MUL: \
452 : for (idx = 0; idx < n; idx++) \
453 : data_in[idx] = data_in[idx] * (v); \
454 : break; \
455 : case GTT_OP_DIV: \
456 : for (idx = 0; idx < n; idx++) \
457 : data_in[idx] = data_in[idx] / (v); \
458 : break; \
459 : default: GST_ERROR_OBJECT (filter, "Unknown operator for float16: %d", op); break; \
460 : } \
461 : } while (0)
462 :
463 : #else /* ! FLOAT16_SUPPORT */
464 : #define _conv_to_f16(intype, o, i, n) do { float16_not_supported (); } while (0)
465 : #define _conv_from_f16(otype, o, i, n) do { float16_not_supported (); } while (0)
466 : #define _op_float16(i, n, v, op) do { float16_not_supported (); } while (0)
467 : #endif /* FLOAT16_SUPPORT */
468 :
469 : #ifdef HAVE_ORC
470 : /* define macros for orc */
471 : #define orc_func_conv(intype,outtype) nns_orc_conv_ ## intype ## _to_ ## outtype
472 : #define orc_func_add(intype) nns_orc_add_c_ ## intype
473 : #define orc_func_mul(intype) nns_orc_mul_c_ ## intype
474 : #define orc_func_div(intype) nns_orc_div_c_ ## intype
475 :
476 : #define orc_typecast_to(i,o,n,intype,otype,intypename) do { \
477 : switch (otype) { \
478 : case _NNS_INT32: orc_func_conv (intype, s32) ((gpointer) o, (gpointer) i, n); break; \
479 : case _NNS_UINT32: orc_func_conv (intype, u32) ((gpointer) o, (gpointer) i, n); break; \
480 : case _NNS_INT16: orc_func_conv (intype, s16) ((gpointer) o, (gpointer) i, n); break; \
481 : case _NNS_UINT16: orc_func_conv (intype, u16) ((gpointer) o, (gpointer) i, n); break; \
482 : case _NNS_INT8: orc_func_conv (intype, s8) ((gpointer) o, (gpointer) i, n); break; \
483 : case _NNS_UINT8: orc_func_conv (intype, u8) ((gpointer) o, (gpointer) i, n); break; \
484 : case _NNS_FLOAT64: orc_func_conv (intype, f64) ((gpointer) o, (gpointer) i, n); break; \
485 : case _NNS_FLOAT32: orc_func_conv (intype, f32) ((gpointer) o, (gpointer) i, n); break; \
486 : case _NNS_INT64: orc_func_conv (intype, s64) ((gpointer) o, (gpointer) i, n); break; \
487 : case _NNS_UINT64: orc_func_conv (intype, u64) ((gpointer) o, (gpointer) i, n); break; \
488 : case _NNS_FLOAT16: _conv_to_f16 (intypename, o, i, n); break; \
489 : default: GST_ERROR_OBJECT (filter, "Unsupported output type %d", otype); g_assert (0); break; \
490 : } \
491 : } while (0)
492 :
493 : #define orc_typecast(i,o,n,itype,otype) do { \
494 : switch (itype) { \
495 : case _NNS_INT32: orc_typecast_to (i, o, n, s32, otype, int32_t); break; \
496 : case _NNS_UINT32: orc_typecast_to (i, o, n, u32, otype, uint32_t); break; \
497 : case _NNS_INT16: orc_typecast_to (i, o, n, s16, otype, int16_t); break; \
498 : case _NNS_UINT16: orc_typecast_to (i, o, n, u16, otype, uint16_t); break; \
499 : case _NNS_INT8: orc_typecast_to (i, o, n, s8, otype, int8_t); break; \
500 : case _NNS_UINT8: orc_typecast_to (i, o, n, u8, otype, uint8_t); break; \
501 : case _NNS_FLOAT64: orc_typecast_to (i, o, n, f64, otype, double); break; \
502 : case _NNS_FLOAT32: orc_typecast_to (i, o, n, f32, otype, float); break; \
503 : case _NNS_INT64: orc_typecast_to (i, o, n, s64, otype, int64_t); break; \
504 : case _NNS_UINT64: orc_typecast_to (i, o, n, u64, otype, uint64_t); break; \
505 : case _NNS_FLOAT16: _conv_from_f16 (otype, o, i, n); break; \
506 : default: GST_ERROR_OBJECT (filter, "Unsupported input type %d", itype); g_assert (0); break; \
507 : } \
508 : } while (0)
509 :
510 : #define orc_typesize(size, type) do { \
511 : switch (type) { \
512 : case _NNS_INT32: size = sizeof(int32_t); break; \
513 : case _NNS_UINT32: size = sizeof(uint32_t); break; \
514 : case _NNS_INT16: size = sizeof(int16_t); break; \
515 : case _NNS_UINT16: size = sizeof(uint16_t); break; \
516 : case _NNS_INT8: size = sizeof(int8_t); break; \
517 : case _NNS_UINT8: size = sizeof(uint8_t); break; \
518 : case _NNS_FLOAT64: size = sizeof(double); break; \
519 : case _NNS_FLOAT32: size = sizeof(float); break; \
520 : case _NNS_INT64: size = sizeof(int64_t); break; \
521 : case _NNS_UINT64: size = sizeof(uint64_t); break; \
522 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", type); g_assert (0); break; \
523 : } \
524 : } while (0)
525 :
526 : #define orc_operator_func(i,n,v,opfunc,op) do { \
527 : switch ((v)->type) { \
528 : case _NNS_INT32: opfunc (s32) ((gpointer) i, (v)->data._int32_t, n); break; \
529 : case _NNS_UINT32: opfunc (u32) ((gpointer) i, (v)->data._uint32_t, n); break; \
530 : case _NNS_INT16: opfunc (s16) ((gpointer) i, (v)->data._int16_t, n); break; \
531 : case _NNS_UINT16: opfunc (u16) ((gpointer) i, (v)->data._uint16_t, n); break; \
532 : case _NNS_INT8: opfunc (s8) ((gpointer) i, (v)->data._int8_t, n); break; \
533 : case _NNS_UINT8: opfunc (u8) ((gpointer) i, (v)->data._uint8_t, n); break; \
534 : case _NNS_FLOAT64: opfunc (f64) ((gpointer) i, (v)->data._double, n); break; \
535 : case _NNS_FLOAT32: opfunc (f32) ((gpointer) i, (v)->data._float, n); break; \
536 : case _NNS_INT64: opfunc (s64) ((gpointer) i, (v)->data._int64_t, n); break; \
537 : case _NNS_UINT64: opfunc (u64) ((gpointer) i, (v)->data._uint64_t, n); break; \
538 : case _NNS_FLOAT16: _op_float16 (i, n, (v)->data._float16, op); break; \
539 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (v)->type); g_assert (0); break; \
540 : } \
541 : } while (0)
542 :
543 : #define orc_operator_div_loop(i,n,val,typename) do { \
544 : gsize idx_div; \
545 : typename *data_in = (typename *) (i); \
546 : for (idx_div = 0; idx_div < (n); ++idx_div) { \
547 : data_in[idx_div] = data_in[idx_div] / (val); \
548 : } \
549 : } while (0)
550 :
551 : #define orc_operator(i,n,v,op) do { \
552 : switch (op) { \
553 : case GTT_OP_ADD: orc_operator_func (i, n, v, orc_func_add, op); break; \
554 : case GTT_OP_MUL: orc_operator_func (i, n, v, orc_func_mul, op); break; \
555 : case GTT_OP_DIV: \
556 : switch ((v)->type) { \
557 : case _NNS_INT32: orc_operator_div_loop (i, n, (v)->data._int32_t, int32_t); break; \
558 : case _NNS_UINT32: orc_operator_div_loop (i, n, (v)->data._uint32_t, uint32_t); break; \
559 : case _NNS_INT16: orc_operator_div_loop (i, n, (v)->data._int16_t, int16_t); break; \
560 : case _NNS_UINT16: orc_operator_div_loop (i, n, (v)->data._uint16_t, uint16_t); break; \
561 : case _NNS_INT8: orc_operator_div_loop (i, n, (v)->data._int8_t, int8_t); break; \
562 : case _NNS_UINT8: orc_operator_div_loop (i, n, (v)->data._uint8_t, uint8_t); break; \
563 : case _NNS_FLOAT64: orc_func_div (f64) ((gpointer) i, (v)->data._double, n); break; \
564 : case _NNS_FLOAT32: orc_func_div (f32) ((gpointer) i, (v)->data._float, n); break; \
565 : case _NNS_INT64: orc_operator_div_loop (i, n, (v)->data._int64_t, int64_t); break; \
566 : case _NNS_UINT64: orc_operator_div_loop (i, n, (v)->data._uint64_t, uint64_t); break; \
567 : case _NNS_FLOAT16: _op_float16 (i, n, (v)->data._float16, op); break; \
568 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (v)->type); g_assert (0); break; \
569 : } \
570 : break; \
571 : default: GST_ERROR_OBJECT (filter, "Unknown operator %d", op); break; \
572 : } \
573 : } while (0)
574 : #endif /* HAVE_ORC */
575 :
576 : /**
577 : * @brief Macro for operator
578 : */
579 : #define handle_operator(d,v,oper,vtype) do { \
580 : switch (oper) { \
581 : case GTT_OP_ADD: \
582 : (d)->data._##vtype += (v)->data._##vtype; \
583 : break; \
584 : case GTT_OP_MUL: \
585 : (d)->data._##vtype *= (v)->data._##vtype; \
586 : break; \
587 : case GTT_OP_DIV: \
588 : if ((v)->data._##vtype == 0) { \
589 : GST_ERROR_OBJECT (filter, "Invalid state, denominator is 0."); \
590 : return FALSE; \
591 : } \
592 : (d)->data._##vtype /= (v)->data._##vtype; \
593 : break; \
594 : default: \
595 : GST_ERROR_OBJECT (filter, "Unknown operator %d", oper); \
596 : return FALSE; \
597 : } \
598 : } while (0)
599 :
600 : /**
601 : * @brief Handle operators for tensor value
602 : * @param filter "this" pointer
603 : * @param desc struct for tensor value
604 : * @param val struct for tensor value
605 : * @param op operator for given tensor value
606 : * @return TRUE if no error
607 : */
608 : static gboolean
609 61615 : gst_tensor_transform_do_operator (GstTensorTransform * filter,
610 : tensor_data_s * desc, const tensor_data_s * val,
611 : tensor_transform_operator op)
612 : {
613 61615 : g_return_val_if_fail (desc != NULL, FALSE);
614 61615 : g_return_val_if_fail (val != NULL, FALSE);
615 61615 : g_return_val_if_fail (desc->type == val->type, FALSE);
616 :
617 61615 : switch (desc->type) {
618 7755 : case _NNS_INT32:
619 7755 : handle_operator (desc, val, op, int32_t);
620 7755 : break;
621 7680 : case _NNS_UINT32:
622 7680 : handle_operator (desc, val, op, uint32_t);
623 7680 : break;
624 7680 : case _NNS_INT16:
625 7680 : handle_operator (desc, val, op, int16_t);
626 7680 : break;
627 7680 : case _NNS_UINT16:
628 7680 : handle_operator (desc, val, op, uint16_t);
629 7680 : break;
630 7680 : case _NNS_INT8:
631 7680 : handle_operator (desc, val, op, int8_t);
632 7680 : break;
633 7680 : case _NNS_UINT8:
634 7680 : handle_operator (desc, val, op, uint8_t);
635 7680 : break;
636 45 : case _NNS_FLOAT64:
637 45 : handle_operator (desc, val, op, double);
638 45 : break;
639 55 : case _NNS_FLOAT32:
640 55 : handle_operator (desc, val, op, float);
641 55 : break;
642 0 : case _NNS_FLOAT16:
643 : #ifdef FLOAT16_SUPPORT
644 : handle_operator (desc, val, op, float16);
645 : #else
646 0 : float16_not_supported ();
647 : #endif
648 0 : break;
649 7680 : case _NNS_INT64:
650 7680 : handle_operator (desc, val, op, int64_t);
651 7680 : break;
652 7680 : case _NNS_UINT64:
653 7680 : handle_operator (desc, val, op, uint64_t);
654 7680 : break;
655 0 : default:
656 0 : GST_ERROR_OBJECT (filter, "Unknown tensor type %d", desc->type);
657 0 : return FALSE;
658 : }
659 :
660 61615 : return TRUE;
661 : }
662 :
663 : /**
664 : * @brief Setup internal data (data_* in GstTensorTransform)
665 : * @param[in/out] filter "this" pointer. mode & option MUST BE set already.
666 : * @retval TRUE if OK or operation-skipped, FALSE if fatal-error.
667 : */
668 : static gboolean
669 541 : gst_tensor_transform_set_option_data (GstTensorTransform * filter)
670 : {
671 : gchar *filter_name;
672 541 : gboolean ret = FALSE;
673 :
674 541 : if (filter->mode == GTT_UNKNOWN || filter->option == NULL)
675 283 : return TRUE;
676 :
677 258 : filter_name = gst_object_get_name ((GstObject *) filter);
678 :
679 258 : switch (filter->mode) {
680 8 : case GTT_DIMCHG:
681 : {
682 8 : gchar **strv = NULL;
683 :
684 8 : if (!g_regex_match_simple (REGEX_DIMCHG_OPTION, filter->option,
685 : G_REGEX_CASELESS, 0)) {
686 3 : ml_loge
687 : ("%s: dimchg: \'%s\' is not valid option string: it should be in the form of IDX_DIM_FROM:IDX_DIM_TO: with a regex, "
688 : REGEX_DIMCHG_OPTION "\n", filter_name, filter->option);
689 3 : break;
690 : }
691 :
692 5 : strv = g_strsplit (filter->option, ":", 2);
693 :
694 5 : filter->data_dimchg.from = (int) g_ascii_strtoll (strv[0], NULL, 10);
695 5 : filter->data_dimchg.to = (int) g_ascii_strtoll (strv[1], NULL, 10);
696 5 : ret = filter->loaded = TRUE;
697 5 : g_strfreev (strv);
698 5 : break;
699 : }
700 117 : case GTT_TYPECAST:
701 : {
702 117 : if (g_regex_match_simple (REGEX_TYPECAST_OPTION, filter->option,
703 : G_REGEX_CASELESS, 0)) {
704 116 : filter->data_typecast.to = gst_tensor_get_type (filter->option);
705 116 : ret = filter->loaded = TRUE;
706 : } else {
707 1 : ml_loge
708 : ("%s: typecast: \'%s\' is not valid data type for tensor: data type of tensor should be one of %s\n",
709 : filter_name, filter->option, GST_TENSOR_TYPE_ALL);
710 : }
711 117 : break;
712 : }
713 90 : case GTT_ARITHMETIC:
714 : {
715 : gchar *str_option;
716 : gchar **str_operators;
717 : gchar **str_op;
718 : tensor_transform_operator_s *op_s;
719 : guint i, num_operators, num_op;
720 : GRegex *regex_option_tc;
721 :
722 90 : filter->data_arithmetic.out_type = _NNS_END;
723 90 : filter->data_arithmetic.per_channel_arith = FALSE;
724 :
725 90 : if (filter->operators) {
726 2 : GST_WARNING_OBJECT (filter,
727 : "There exists pre-defined operators (total %d), now reset these.",
728 : g_slist_length (filter->operators));
729 :
730 2 : g_slist_free_full (filter->operators, g_free);
731 2 : filter->operators = NULL;
732 : }
733 :
734 90 : regex_option_tc = g_regex_new (REGEX_ARITH_OPTION_TYPECAST,
735 : G_REGEX_CASELESS, 0, 0);
736 :
737 90 : if (!regex_option_tc) {
738 0 : GST_ERROR_OBJECT (filter,
739 : "arithmetic: failed to create a GRegex structure for %s\n",
740 : REGEX_ARITH_OPTION_TYPECAST);
741 0 : break;
742 : }
743 :
744 90 : if (g_regex_match_full (regex_option_tc, filter->option, -1,
745 : 1, 0, NULL, NULL)) {
746 2 : str_option = g_regex_replace (regex_option_tc, filter->option, -1, 1,
747 : "", 0, 0);
748 2 : ml_loge
749 : ("%s: arithmetic: [typecast:TYPE,] should be located at the first to prevent memory re-allocation: typecast(s) in the middle of \'%s\' will be ignored\n",
750 : filter_name, filter->option);
751 : } else {
752 176 : str_option = g_strdup (filter->option);
753 : }
754 90 : g_regex_unref (regex_option_tc);
755 :
756 90 : if (!g_regex_match_simple (REGEX_ARITH_OPTION, str_option,
757 : G_REGEX_CASELESS, 0)) {
758 13 : ml_loge
759 : ("%s: arithmetic: \'%s\' is not valid option string: it should be in the form of [typecast:TYPE,][per-channel:(false|true@DIM),]add|mul|div:NUMBER[@CH_IDX]..., ...\n",
760 : filter_name, str_option);
761 13 : g_free (str_option);
762 13 : break;
763 : }
764 77 : str_operators = g_strsplit (str_option, ",", -1);
765 77 : num_operators = g_strv_length (str_operators);
766 :
767 256 : for (i = 0; i < num_operators; ++i) {
768 179 : str_op = g_strsplit (str_operators[i], ":", -1);
769 179 : num_op = g_strv_length (str_op);
770 :
771 179 : if (str_op[0]) {
772 177 : gchar **values = g_strsplit (str_op[1], "@", -1);
773 177 : guint num_values = g_strv_length (values);
774 :
775 : /* check whether per-channel */
776 177 : if (g_ascii_strcasecmp (str_op[0], "per-channel") == 0) {
777 2 : if (num_values > 1 && g_ascii_strcasecmp (values[0], "true") == 0) {
778 2 : ml_logi
779 : ("Set per-channel for arithmetic and assume that %s-th dim is the channel",
780 : values[1]);
781 2 : filter->data_arithmetic.per_channel_arith = TRUE;
782 2 : filter->data_arithmetic.ch_dim =
783 2 : (guint) g_ascii_strtoull (values[1], NULL, 10);
784 : }
785 :
786 2 : g_strfreev (values);
787 2 : g_strfreev (str_op);
788 2 : continue;
789 : }
790 :
791 175 : op_s = g_new0 (tensor_transform_operator_s, 1);
792 175 : g_assert (op_s);
793 :
794 175 : op_s->op = gst_tensor_transform_get_operator (str_op[0]);
795 175 : op_s->applying_ch = -1; /* -1 means applying to all channels */
796 175 : switch (op_s->op) {
797 60 : case GTT_OP_TYPECAST:
798 60 : if (num_op > 1 && str_op[1]) {
799 60 : op_s->value.type = gst_tensor_get_type (values[0]);
800 60 : filter->data_arithmetic.out_type = op_s->value.type;
801 : } else {
802 0 : GST_WARNING_OBJECT (filter, "Invalid option for typecast %s",
803 : str_operators[i]);
804 0 : op_s->op = GTT_OP_UNKNOWN;
805 : }
806 60 : break;
807 115 : case GTT_OP_ADD:
808 : case GTT_OP_MUL:
809 : case GTT_OP_DIV:
810 115 : if (num_op > 1 && str_op[1]) {
811 : /* get operand */
812 115 : if (strchr (values[0], '.') || strchr (values[0], 'e') ||
813 115 : strchr (values[0], 'E')) {
814 : double val;
815 :
816 79 : val = g_ascii_strtod (values[0], NULL);
817 79 : gst_tensor_data_set (&op_s->value, _NNS_FLOAT64, &val);
818 : } else {
819 : int64_t val;
820 :
821 36 : val = g_ascii_strtoll (values[0], NULL, 10);
822 36 : gst_tensor_data_set (&op_s->value, _NNS_INT64, &val);
823 : }
824 :
825 115 : if (filter->data_arithmetic.per_channel_arith && num_values > 1) {
826 11 : op_s->applying_ch = g_ascii_strtoll (values[1], NULL, 10);
827 : }
828 :
829 : } else {
830 0 : GST_WARNING_OBJECT (filter,
831 : "Invalid option for arithmetic %s", str_operators[i]);
832 0 : op_s->op = GTT_OP_UNKNOWN;
833 : }
834 115 : break;
835 0 : default:
836 0 : GST_WARNING_OBJECT (filter, "Unknown operator %s", str_op[0]);
837 0 : break;
838 : }
839 :
840 : /* append operator */
841 175 : if (op_s->op != GTT_OP_UNKNOWN) {
842 175 : filter->operators = g_slist_append (filter->operators, op_s);
843 : } else {
844 0 : g_free (op_s);
845 : }
846 :
847 175 : g_strfreev (values);
848 : } else {
849 2 : GST_WARNING_OBJECT (filter, "Invalid option %s", str_operators[i]);
850 : }
851 :
852 177 : g_strfreev (str_op);
853 : }
854 :
855 77 : ret = filter->loaded = (filter->operators != NULL);
856 77 : g_strfreev (str_operators);
857 77 : g_free (str_option);
858 77 : break;
859 : }
860 20 : case GTT_TRANSPOSE:
861 : {
862 : int i;
863 20 : gchar **strv = NULL;
864 :
865 20 : if (!g_regex_match_simple (REGEX_TRANSPOSE_OPTION, filter->option,
866 : G_REGEX_CASELESS, 0)) {
867 4 : ml_loge
868 : ("%s: transpose: \'%s\' is not valid option string: it should be in the form of NEW_IDX_DIM0:NEW_IDX_DIM1:NEW_IDX_DIM2:3 (Now transpose mode's rank is fixed to 3. Note that the index of the last dim is always fixed to 3)\n",
869 : filter_name, filter->option);
870 4 : break;
871 : }
872 :
873 16 : strv = g_strsplit (filter->option, ":", NNS_TENSOR_TRANSPOSE_RANK_LIMIT);
874 80 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
875 64 : filter->data_transpose.trans_order[i] =
876 64 : (uint8_t) g_ascii_strtoull (strv[i], NULL, 10);
877 : }
878 :
879 16 : ret = filter->loaded = TRUE;
880 16 : g_strfreev (strv);
881 16 : break;
882 : }
883 9 : case GTT_STAND:
884 : {
885 9 : gchar **options = NULL;
886 : guint i, num_options;
887 :
888 9 : if (!g_regex_match_simple (REGEX_STAND_OPTION, filter->option,
889 : G_REGEX_CASELESS, 0)) {
890 5 : ml_loge
891 : ("%s: stand: \'%s\' is not a valid option string: it should be in the form of (default|dc-average)[:TYPE][,per-channel:(false|true)]\n",
892 : filter_name, filter->option);
893 5 : break;
894 : }
895 :
896 4 : filter->data_stand.out_type = _NNS_END;
897 4 : filter->data_stand.per_channel = FALSE;
898 :
899 4 : options = g_strsplit (filter->option, ",", -1);
900 4 : num_options = g_strv_length (options);
901 :
902 10 : for (i = 0; i < num_options; i++) {
903 6 : gchar **strv = g_strsplit (options[i], ":", -1);
904 :
905 6 : if (g_ascii_strcasecmp (strv[0], "default") == 0 ||
906 4 : g_ascii_strcasecmp (strv[0], "dc-average") == 0) {
907 4 : filter->data_stand.mode =
908 4 : gst_tensor_transform_get_stand_mode (strv[0]);
909 6 : if (g_strv_length (strv) > 1)
910 2 : filter->data_stand.out_type = gst_tensor_get_type (strv[1]);
911 2 : } else if (g_ascii_strcasecmp (strv[0], "per-channel") == 0) {
912 2 : if (g_strv_length (strv) > 1 &&
913 2 : g_ascii_strcasecmp (strv[1], "true") == 0)
914 2 : filter->data_stand.per_channel = TRUE;
915 : } else {
916 0 : filter->data_stand.mode = STAND_END;
917 0 : ml_logw ("Unknown option for stand mode: %s", strv[0]);
918 : }
919 :
920 6 : g_strfreev (strv);
921 : }
922 :
923 4 : g_strfreev (options);
924 4 : ret = filter->loaded = TRUE;
925 4 : break;
926 : }
927 10 : case GTT_CLAMP:
928 : {
929 10 : gchar **strv = NULL;
930 :
931 10 : if (!g_regex_match_simple (REGEX_CLAMP_OPTION, filter->option,
932 : G_REGEX_CASELESS, 0)) {
933 1 : ml_loge
934 : ("%s: clamp: \'%s\' is not valid option string: it should be in the form of [CLAMP_MIN:CLAMP_MAX]\n",
935 : filter_name, filter->option);
936 1 : break;
937 : }
938 :
939 9 : strv = g_strsplit (filter->option, ":", 2);
940 :
941 9 : filter->data_clamp.min = g_ascii_strtod (strv[0], NULL);
942 9 : if (errno == ERANGE) {
943 1 : ml_loge ("%s: clamp: CLAMP_MIN value has an invalid range\n",
944 : filter_name);
945 1 : g_strfreev (strv);
946 1 : break;
947 : }
948 8 : filter->data_clamp.max = g_ascii_strtod (strv[1], NULL);
949 8 : if (errno == ERANGE) {
950 1 : ml_loge ("%s: clamp: CLAMP_MAX value has an invalid range\n",
951 : filter_name);
952 1 : g_strfreev (strv);
953 1 : break;
954 : }
955 :
956 7 : g_strfreev (strv);
957 :
958 7 : if (filter->data_clamp.min > filter->data_clamp.max) {
959 2 : ml_loge ("%s: clamp: CLAMP_MIN is larger than CLAMP_MAX\n",
960 : filter_name);
961 2 : break;
962 : }
963 :
964 5 : ret = filter->loaded = TRUE;
965 5 : break;
966 : }
967 4 : case GTT_PADDING:
968 : {
969 4 : gchar **options = NULL;
970 : guint i, num_options;
971 :
972 4 : if (!g_regex_match_simple (REGEX_PADDING_OPTION, filter->option,
973 : G_REGEX_CASELESS, 0)) {
974 0 : ml_loge
975 : ("%s: padding: \'%s\' is not valid option string: it should be in the form of left|right|top|bottom|front|back:PADDING,[layout:(NCHW|NHWC)]\n",
976 : filter_name, filter->option);
977 0 : break;
978 : }
979 :
980 68 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++)
981 64 : filter->data_padding.pad[i] = 0;
982 4 : filter->data_padding.layout = _NNS_LAYOUT_ANY;
983 :
984 4 : options = g_strsplit (filter->option, ",", -1);
985 4 : num_options = g_strv_length (options);
986 :
987 23 : for (i = 0; i < num_options; i++) {
988 19 : gchar **strv = g_strsplit (options[i], ":", 2);
989 19 : if (g_ascii_strcasecmp (strv[0], "left") == 0) {
990 4 : filter->data_padding.pad[PADDING_LEFT] =
991 4 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
992 15 : } else if (g_ascii_strcasecmp (strv[0], "right") == 0) {
993 4 : filter->data_padding.pad[PADDING_RIGHT] =
994 4 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
995 11 : } else if (g_ascii_strcasecmp (strv[0], "top") == 0) {
996 3 : filter->data_padding.pad[PADDING_TOP] =
997 3 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
998 8 : } else if (g_ascii_strcasecmp (strv[0], "bottom") == 0) {
999 3 : filter->data_padding.pad[PADDING_BOTTOM] =
1000 3 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1001 5 : } else if (g_ascii_strcasecmp (strv[0], "front") == 0) {
1002 2 : filter->data_padding.pad[PADDING_FRONT] =
1003 2 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1004 3 : } else if (g_ascii_strcasecmp (strv[0], "back") == 0) {
1005 2 : filter->data_padding.pad[PADDING_BACK] =
1006 2 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1007 1 : } else if (g_ascii_strcasecmp (strv[0], "layout") == 0) {
1008 1 : if (g_ascii_strcasecmp (strv[1], "NHWC") == 0)
1009 1 : filter->data_padding.layout = _NNS_LAYOUT_NHWC;
1010 : else
1011 0 : filter->data_padding.layout = _NNS_LAYOUT_NCHW;
1012 : } else {
1013 0 : ml_logw ("Unknown option for padding mode: %s", strv[0]);
1014 : }
1015 19 : g_strfreev (strv);
1016 : }
1017 4 : g_strfreev (options);
1018 :
1019 4 : if (filter->data_padding.layout == _NNS_LAYOUT_NHWC) {
1020 1 : guint prev_left = filter->data_padding.pad[PADDING_LEFT],
1021 1 : prev_right = filter->data_padding.pad[PADDING_RIGHT];
1022 1 : filter->data_padding.pad[PADDING_LEFT] =
1023 1 : filter->data_padding.pad[PADDING_FRONT];
1024 1 : filter->data_padding.pad[PADDING_RIGHT] =
1025 1 : filter->data_padding.pad[PADDING_BACK];
1026 1 : filter->data_padding.pad[PADDING_FRONT] = prev_left;
1027 1 : filter->data_padding.pad[PADDING_BACK] = prev_right;
1028 : }
1029 :
1030 4 : ret = filter->loaded = TRUE;
1031 4 : break;
1032 : }
1033 0 : default:
1034 0 : GST_ERROR_OBJECT (filter, "Cannot identify mode\n");
1035 0 : ret = FALSE;
1036 : }
1037 :
1038 258 : g_free (filter_name);
1039 258 : return ret;
1040 : }
1041 :
1042 : /**
1043 : * @brief Set property (gst element vmethod)
1044 : */
1045 : static void
1046 572 : gst_tensor_transform_set_property (GObject * object, guint prop_id,
1047 : const GValue * value, GParamSpec * pspec)
1048 : {
1049 572 : GstTensorTransform *filter = GST_TENSOR_TRANSFORM (object);
1050 :
1051 572 : switch (prop_id) {
1052 1 : case PROP_SILENT:
1053 1 : filter->silent = g_value_get_boolean (value);
1054 1 : break;
1055 253 : case PROP_MODE:
1056 253 : filter->mode = g_value_get_enum (value);
1057 253 : gst_tensor_transform_set_option_data (filter);
1058 253 : break;
1059 257 : case PROP_OPTION:
1060 : {
1061 257 : gchar *backup_option = filter->option;
1062 257 : filter->option = g_value_dup_string (value);
1063 257 : if (gst_tensor_transform_set_option_data (filter)) {
1064 226 : silent_debug (filter, "Option = %s --> %s\n", backup_option,
1065 : filter->option);
1066 226 : g_free (backup_option);
1067 : } else {
1068 : /* ERROR! Revert the change! */
1069 31 : g_free (filter->option);
1070 31 : filter->option = backup_option;
1071 31 : gst_tensor_transform_set_option_data (filter);
1072 : }
1073 257 : break;
1074 : }
1075 58 : case PROP_ACCELERATION:
1076 : #ifdef HAVE_ORC
1077 58 : filter->acceleration = g_value_get_boolean (value);
1078 58 : silent_debug (filter, "acceleration = %d\n", filter->acceleration);
1079 : #else
1080 : GST_WARNING_OBJECT (filter, "Orc acceleration is not supported");
1081 : filter->acceleration = FALSE;
1082 : #endif
1083 58 : break;
1084 3 : case PROP_APPLY:
1085 : {
1086 : gint64 val;
1087 3 : const gchar *param = g_value_get_string (value);
1088 3 : gchar **strv = g_strsplit_set (param, ",", -1);
1089 3 : guint i, num = g_strv_length (strv);
1090 3 : gchar *endptr = NULL;
1091 :
1092 10 : for (i = 0; i < num; i++) {
1093 7 : errno = 0;
1094 7 : val = g_ascii_strtoll (strv[i], &endptr, 10);
1095 7 : if (errno == ERANGE || errno == EINVAL || (endptr == strv[i])) {
1096 0 : ml_loge ("Cannot convert string %s to a gint64 value", strv[i]);
1097 : }
1098 7 : filter->apply = g_list_append (filter->apply, GINT_TO_POINTER (val));
1099 : }
1100 3 : g_strfreev (strv);
1101 3 : break;
1102 : }
1103 0 : default:
1104 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1105 0 : break;
1106 : }
1107 572 : }
1108 :
1109 : /**
1110 : * @brief Get property (gst element vmethod)
1111 : */
1112 : static void
1113 41 : gst_tensor_transform_get_property (GObject * object, guint prop_id,
1114 : GValue * value, GParamSpec * pspec)
1115 : {
1116 41 : GstTensorTransform *filter = GST_TENSOR_TRANSFORM (object);
1117 :
1118 41 : switch (prop_id) {
1119 2 : case PROP_SILENT:
1120 2 : g_value_set_boolean (value, filter->silent);
1121 2 : break;
1122 1 : case PROP_MODE:
1123 1 : g_value_set_enum (value, filter->mode);
1124 1 : break;
1125 36 : case PROP_OPTION:
1126 36 : g_value_set_string (value, filter->option);
1127 36 : break;
1128 2 : case PROP_ACCELERATION:
1129 2 : g_value_set_boolean (value, filter->acceleration);
1130 2 : break;
1131 0 : case PROP_APPLY:
1132 : {
1133 : GList *list;
1134 : gchar *p;
1135 : GPtrArray *arr;
1136 : gchar **strings;
1137 :
1138 0 : if (filter->apply == NULL) {
1139 0 : g_value_set_string (value, "");
1140 0 : return;
1141 : }
1142 :
1143 0 : arr = g_ptr_array_new ();
1144 0 : for (list = filter->apply; list != NULL; list = list->next) {
1145 0 : g_ptr_array_add (arr, g_strdup_printf ("%i",
1146 0 : GPOINTER_TO_INT (list->data)));
1147 : }
1148 0 : g_ptr_array_add (arr, NULL);
1149 0 : strings = (gchar **) g_ptr_array_free (arr, FALSE);
1150 0 : p = g_strjoinv (",", strings);
1151 :
1152 0 : g_strfreev (strings);
1153 0 : g_value_take_string (value, p);
1154 0 : break;
1155 : }
1156 0 : case PROP_TRANSPOSE_RANK_LIMIT:
1157 0 : g_value_set_uint (value, NNS_TENSOR_TRANSPOSE_RANK_LIMIT);
1158 0 : break;
1159 0 : default:
1160 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1161 0 : break;
1162 : }
1163 : }
1164 :
1165 : /**
1166 : * @brief Function to finalize instance (gst element vmethod)
1167 : */
1168 : static void
1169 239 : gst_tensor_transform_finalize (GObject * object)
1170 : {
1171 : GstTensorTransform *filter;
1172 :
1173 239 : filter = GST_TENSOR_TRANSFORM (object);
1174 :
1175 239 : if (filter->option) {
1176 200 : g_free (filter->option);
1177 200 : filter->option = NULL;
1178 : }
1179 :
1180 239 : if (filter->operators) {
1181 60 : g_slist_free_full (filter->operators, g_free);
1182 60 : filter->operators = NULL;
1183 : }
1184 :
1185 239 : if (filter->apply) {
1186 3 : g_list_free (filter->apply);
1187 3 : filter->apply = NULL;
1188 : }
1189 :
1190 239 : G_OBJECT_CLASS (parent_class)->finalize (object);
1191 239 : }
1192 :
1193 : /**
1194 : * @brief subrouting for tensor-transform, "dimchg" case.
1195 : * @param[in/out] filter "this" pointer
1196 : * @param[in] in_info input tensor info
1197 : * @param[in] out_info output tensor info
1198 : * @param[in] inptr input tensor
1199 : * @param[out] outptr output tensor
1200 : * @return Gst flow status
1201 : */
1202 : static GstFlowReturn
1203 52 : gst_tensor_transform_dimchg (GstTensorTransform * filter,
1204 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1205 : const uint8_t * inptr, uint8_t * outptr)
1206 : {
1207 52 : uint32_t *fromDim = in_info->dimension;
1208 52 : uint32_t *toDim = out_info->dimension;
1209 52 : unsigned int from = filter->data_dimchg.from;
1210 52 : unsigned int to = filter->data_dimchg.to;
1211 : unsigned int i, j, k;
1212 52 : unsigned int loopLimit = 1;
1213 : gsize loopBlockSize, copyblocksize, copyblocklimit;
1214 :
1215 52 : if (from == to) {
1216 : /** Useless memcpy. Do not call this or @todo do "IP" operation */
1217 0 : nns_memcpy (outptr, inptr, gst_tensor_info_get_size (in_info));
1218 0 : GST_WARNING_OBJECT (filter,
1219 : "Calling tensor_transform with high memcpy overhead WITHOUT any effects! Check your stream whether you really need tensor_transform.\n");
1220 0 : return GST_FLOW_OK;
1221 : }
1222 :
1223 52 : g_assert (from < NNS_TENSOR_RANK_LIMIT);
1224 52 : g_assert (to < NNS_TENSOR_RANK_LIMIT);
1225 52 : g_assert (fromDim[from] == toDim[to]);
1226 :
1227 52 : loopBlockSize = copyblocksize = gst_tensor_get_element_size (in_info->type);
1228 52 : copyblocklimit = 1;
1229 :
1230 52 : if (from < to) {
1231 : /**
1232 : * Smaller-loop-ed a to larger-loop-ed b
1233 : * E.g., [N][H][W][c] (c:W:H:N) --> [N][c][H][W] (W:H:c:N)
1234 : *
1235 : * @todo CRITICAL-TODO: Optimize the performance!
1236 : */
1237 726 : for (i = NNS_TENSOR_RANK_LIMIT - 1; i > to; i--) {
1238 674 : if (toDim[i] == 0)
1239 620 : continue;
1240 54 : loopLimit *= toDim[i];
1241 : }
1242 :
1243 158 : for (i = 0; i < to; i++) {
1244 106 : if (toDim[i] == 0)
1245 0 : break;
1246 106 : loopBlockSize *= toDim[i];
1247 : }
1248 :
1249 52 : for (i = 0; i < from; i++) {
1250 0 : if (fromDim[i] == 0)
1251 0 : break;
1252 0 : copyblocksize *= fromDim[i];
1253 : }
1254 158 : for (i = 0; i < to; i++) {
1255 106 : if (toDim[i] == 0)
1256 0 : break;
1257 106 : copyblocklimit *= toDim[i];
1258 : }
1259 :
1260 228 : for (i = 0; i < loopLimit; i++) {
1261 : /* [i1][i2][...][iN][b][...] i = i1 x i2 x ... x iN */
1262 176 : uint8_t *destptr = outptr + loopBlockSize * toDim[to] * i;
1263 176 : const uint8_t *srcptr = inptr + loopBlockSize * toDim[to] * i;
1264 :
1265 744 : for (j = 0; j < toDim[to]; j++) {
1266 568 : uint8_t *j_destptr = destptr + loopBlockSize * j;
1267 121208 : for (k = 0; k < copyblocklimit; k++) {
1268 120640 : nns_memcpy (j_destptr + copyblocksize * k,
1269 : srcptr + k * copyblocksize * toDim[to] + j * copyblocksize,
1270 : copyblocksize);
1271 : }
1272 : }
1273 : }
1274 : } else {
1275 : /**
1276 : * Larger-loop-ed a to smaller-loop-ed b
1277 : * E.g., [N][c][H][W] (W:H:c:N) --> [N][H][W][c] (c:W:H:N)
1278 : * @todo NYI
1279 : */
1280 0 : ml_loge
1281 : ("tensor-transform/dimchg operation is not permitted if from >= to.\n");
1282 0 : return GST_FLOW_ERROR;
1283 : }
1284 :
1285 52 : return GST_FLOW_OK;
1286 : }
1287 :
1288 : /**
1289 : * @brief subrouting for tensor-transform, "typecast" case.
1290 : * @param[in/out] filter "this" pointer
1291 : * @param[in] in_info input tensor info
1292 : * @param[in] out_info output tensor info
1293 : * @param[in] inptr input tensor
1294 : * @param[out] outptr output tensor
1295 : * @return Gst flow status
1296 : */
1297 : static GstFlowReturn
1298 579 : gst_tensor_transform_typecast (GstTensorTransform * filter,
1299 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1300 : const uint8_t * inptr, uint8_t * outptr)
1301 : {
1302 : gulong i, num;
1303 : gsize in_element_size, out_element_size;
1304 :
1305 579 : num = gst_tensor_get_element_count (in_info->dimension);
1306 :
1307 : #ifdef HAVE_ORC
1308 579 : if (filter->acceleration) {
1309 457 : orc_typecast (inptr, outptr, num, in_info->type, out_info->type);
1310 457 : return GST_FLOW_OK;
1311 : }
1312 : #endif
1313 :
1314 122 : in_element_size = gst_tensor_get_element_size (in_info->type);
1315 122 : out_element_size = gst_tensor_get_element_size (out_info->type);
1316 :
1317 61772 : for (i = 0; i < num; ++i) {
1318 61650 : gst_tensor_data_raw_typecast (
1319 61650 : (gpointer) (inptr + in_element_size * i), in_info->type,
1320 61650 : (gpointer) (outptr + out_element_size * i), out_info->type);
1321 : }
1322 :
1323 122 : return GST_FLOW_OK;
1324 : }
1325 :
1326 : /**
1327 : * @brief subrouting for tensor-transform, "arithmetic" case.
1328 : * @param[in/out] filter "this" pointer
1329 : * @param[in] in_info input tensor info
1330 : * @param[in] out_info output tensor info
1331 : * @param[in] inptr input tensor
1332 : * @param[out] outptr output tensor
1333 : * @return Gst flow status
1334 : */
1335 : static GstFlowReturn
1336 21847 : gst_tensor_transform_arithmetic (GstTensorTransform * filter,
1337 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1338 : const uint8_t * inptr, uint8_t * outptr)
1339 : {
1340 : gulong i, num, j, ch;
1341 : gsize in_element_size, out_element_size;
1342 :
1343 : GSList *walk;
1344 : tensor_transform_operator_s *op_s;
1345 : tensor_data_s value;
1346 :
1347 21847 : num = gst_tensor_get_element_count (in_info->dimension);
1348 :
1349 : #ifdef HAVE_ORC
1350 21847 : if (filter->acceleration) {
1351 21747 : walk = filter->operators;
1352 : /**
1353 : * Typecast should be called at the first.
1354 : * Do the typecast. If in/out type is same, this will copy the input array to output.
1355 : */
1356 21747 : orc_typecast (inptr, outptr, num, in_info->type, out_info->type);
1357 :
1358 21747 : if (!filter->data_arithmetic.per_channel_arith) {
1359 66411 : while (walk) {
1360 44665 : op_s = (tensor_transform_operator_s *) walk->data;
1361 :
1362 44665 : if (op_s->op != GTT_OP_TYPECAST) {
1363 22997 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1364 24025 : orc_operator (outptr, num, &op_s->value, op_s->op);
1365 : }
1366 :
1367 44665 : walk = g_slist_next (walk);
1368 : }
1369 : } else {
1370 1 : gsize typesize = 0;
1371 1 : guint ch_dim = filter->data_arithmetic.ch_dim;
1372 1 : gsize ch_offset, ch_size = 1;
1373 1 : uint8_t *tmp_outptr = NULL;
1374 :
1375 1 : for (i = 0; i < ch_dim; ++i) {
1376 0 : ch_size *= in_info->dimension[i];
1377 : }
1378 1 : ch_offset = ch_size * in_info->dimension[ch_dim];
1379 1 : orc_typesize (typesize, out_info->type);
1380 :
1381 2 : while (walk) {
1382 1 : op_s = (tensor_transform_operator_s *) walk->data;
1383 1 : if (op_s->op == GTT_OP_TYPECAST) {
1384 0 : walk = g_slist_next (walk);
1385 0 : continue;
1386 : }
1387 :
1388 1 : if (op_s->applying_ch == -1) {
1389 0 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1390 0 : orc_operator (outptr, num, &op_s->value, op_s->op);
1391 : } else {
1392 33 : for (i = 0; i < num / ch_offset; ++i) {
1393 32 : tmp_outptr =
1394 32 : outptr + (ch_size * op_s->applying_ch +
1395 32 : ch_offset * i) * typesize;
1396 32 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1397 32 : orc_operator (tmp_outptr, ch_size, &op_s->value, op_s->op);
1398 : }
1399 : }
1400 1 : walk = g_slist_next (walk);
1401 : }
1402 : }
1403 21847 : return GST_FLOW_OK;
1404 : }
1405 : #endif
1406 :
1407 100 : in_element_size = gst_tensor_get_element_size (in_info->type);
1408 100 : out_element_size = gst_tensor_get_element_size (out_info->type);
1409 :
1410 : /* per-channel */
1411 100 : if (filter->data_arithmetic.per_channel_arith) {
1412 3 : guint ch_dim = filter->data_arithmetic.ch_dim;
1413 3 : gsize ch_offset, ch_size = 1;
1414 3 : for (i = 0; i < ch_dim; ++i) {
1415 0 : ch_size *= in_info->dimension[i];
1416 : }
1417 3 : ch_offset = ch_size * in_info->dimension[ch_dim];
1418 :
1419 : /** In case of 3:4:4:1,
1420 : * ch_dim:0 -> #ch: 3, ch_size: 1, ch_offset: 3
1421 : * ch_dim:1 -> #ch: 4, ch_size: 3, ch_offset: 12
1422 : * ch_dim:2 -> #ch: 4, ch_size: 12, ch_offset: 48
1423 : * ch_dim:3 -> #ch: 1, ch_size: 48, ch_offset: 48 * 4
1424 : */
1425 :
1426 6 : for (i = 0; i < num / ch_offset; ++i) {
1427 18 : for (ch = 0; ch < in_info->dimension[ch_dim]; ++ch) {
1428 30 : for (j = 0; j < ch_size; ++j) {
1429 15 : gulong data_idx = (i * ch_offset) + (ch * ch_size) + j;
1430 15 : gst_tensor_data_set (&value, in_info->type,
1431 15 : (gpointer) (inptr + in_element_size * data_idx));
1432 :
1433 15 : walk = filter->operators;
1434 180 : while (walk) {
1435 165 : op_s = (tensor_transform_operator_s *) walk->data;
1436 165 : switch (op_s->op) {
1437 15 : case GTT_OP_TYPECAST:
1438 15 : gst_tensor_data_typecast (&value, op_s->value.type);
1439 15 : break;
1440 150 : case GTT_OP_ADD:
1441 : case GTT_OP_MUL:
1442 : case GTT_OP_DIV:
1443 : {
1444 150 : gst_tensor_data_typecast (&op_s->value, value.type);
1445 :
1446 150 : if (op_s->applying_ch == (int) ch || op_s->applying_ch == -1) {
1447 30 : gst_tensor_transform_do_operator (filter, &value,
1448 30 : &op_s->value, op_s->op);
1449 : }
1450 150 : break;
1451 : }
1452 0 : default:
1453 0 : g_assert (0);
1454 : return GST_FLOW_ERROR;
1455 : }
1456 :
1457 165 : walk = g_slist_next (walk);
1458 : }
1459 :
1460 : /* set output value */
1461 15 : g_assert (out_info->type == value.type);
1462 15 : gst_tensor_data_get (&value, outptr + out_element_size * data_idx);
1463 : }
1464 : }
1465 : }
1466 :
1467 3 : return GST_FLOW_OK;
1468 : }
1469 :
1470 61622 : for (i = 0; i < num; ++i) {
1471 : /* init value with input tensor type */
1472 61525 : gst_tensor_data_set (&value, in_info->type,
1473 61525 : (gpointer) (inptr + in_element_size * i));
1474 :
1475 61525 : walk = filter->operators;
1476 184595 : while (walk) {
1477 123070 : op_s = (tensor_transform_operator_s *) walk->data;
1478 :
1479 : /**
1480 : * @todo add more options
1481 : */
1482 123070 : switch (op_s->op) {
1483 61485 : case GTT_OP_TYPECAST:
1484 61485 : gst_tensor_data_typecast (&value, op_s->value.type);
1485 61485 : break;
1486 61585 : case GTT_OP_ADD:
1487 : case GTT_OP_MUL:
1488 : case GTT_OP_DIV:
1489 61585 : gst_tensor_data_typecast (&op_s->value, value.type);
1490 61585 : gst_tensor_transform_do_operator (filter, &value, &op_s->value,
1491 : op_s->op);
1492 61585 : break;
1493 0 : default:
1494 0 : g_assert (0);
1495 : return GST_FLOW_ERROR;
1496 : }
1497 :
1498 123070 : walk = g_slist_next (walk);
1499 : }
1500 :
1501 : /* set output value */
1502 61525 : g_assert (out_info->type == value.type);
1503 61525 : gst_tensor_data_get (&value, outptr + out_element_size * i);
1504 : }
1505 :
1506 97 : return GST_FLOW_OK;
1507 : }
1508 :
1509 : /**
1510 : * Macro to run loop for various data types with transpose
1511 : */
1512 : #define transposeloop(cl,ck,cj,ci,sl,sk,sj,si,typesize) do { \
1513 : size_t i, j, k, l; \
1514 : int inidx = 0, outidx=0; \
1515 : for(cl=0;cl<sl;cl++) \
1516 : for(ci=0;ci<si;ci++) \
1517 : for(cj=0;cj<sj;cj++) \
1518 : for(ck=0;ck<sk;ck++){ \
1519 : const uint8_t *_in; \
1520 : uint8_t *_out; \
1521 : outidx = si*sj*sk*cl + sj*sk*ci + sk*cj + ck; \
1522 : inidx = SK*SJ*SI*l + SJ*SI*k + SI*j + i; \
1523 : _in = inptr + inidx * typesize; \
1524 : _out = outptr + outidx * typesize; \
1525 : nns_memcpy(_out, _in, typesize); \
1526 : } \
1527 : } while(0);
1528 :
1529 : /**
1530 : * @brief subrouting for tensor-transform, "transpose" case.
1531 : * @param[in/out] filter "this" pointer
1532 : * @param[in] in_info input tensor info
1533 : * @param[in] out_info output tensor info
1534 : * @param[in] inptr input tensor
1535 : * @param[out] outptr output tensor
1536 : * @return Gst flow status
1537 : */
1538 : static GstFlowReturn
1539 58 : gst_tensor_transform_transpose (GstTensorTransform * filter,
1540 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1541 : const uint8_t * inptr, uint8_t * outptr)
1542 : {
1543 : int i, from, to;
1544 58 : gboolean checkdim = FALSE;
1545 58 : uint32_t *fromDim = in_info->dimension;
1546 58 : gsize type_size = gst_tensor_get_element_size (in_info->type);
1547 : gsize indexI, indexJ, SL, SI, SJ, SK;
1548 : UNUSED (out_info);
1549 :
1550 58 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
1551 58 : from = i;
1552 58 : to = filter->data_transpose.trans_order[i];
1553 58 : if (from != to) {
1554 58 : checkdim = TRUE;
1555 58 : break;
1556 : }
1557 : }
1558 :
1559 58 : if (!checkdim) {
1560 0 : nns_memcpy (outptr, inptr, gst_tensor_info_get_size (in_info));
1561 0 : GST_WARNING_OBJECT (filter,
1562 : "Calling tensor_transform with high memcpy overhead WITHOUT any effects!");
1563 0 : return GST_FLOW_OK;
1564 : }
1565 :
1566 58 : indexI = filter->data_transpose.trans_order[0];
1567 58 : indexJ = filter->data_transpose.trans_order[1];
1568 58 : SL = fromDim[3] > 0 ? fromDim[3] : 1;
1569 58 : SI = fromDim[0] > 0 ? fromDim[0] : 1;
1570 58 : SJ = fromDim[1] > 0 ? fromDim[1] : 1;
1571 58 : SK = fromDim[2] > 0 ? fromDim[2] : 1;
1572 :
1573 58 : switch (indexI) {
1574 0 : case 0:
1575 0 : if (indexJ == 1) {
1576 0 : transposeloop (l, i, j, k, SL, SI, SJ, SK, type_size);
1577 : } else {
1578 0 : transposeloop (l, i, k, j, SL, SI, SK, SJ, type_size);
1579 : }
1580 0 : break;
1581 54 : case 1:
1582 54 : if (indexJ == 0) {
1583 242420 : transposeloop (l, j, i, k, SL, SJ, SI, SK, type_size);
1584 : } else {
1585 1231478 : transposeloop (l, j, k, i, SL, SJ, SK, SI, type_size);
1586 : }
1587 54 : break;
1588 4 : case 2:
1589 4 : if (indexJ == 0) {
1590 140258 : transposeloop (l, k, i, j, SL, SK, SI, SJ, type_size);
1591 : } else {
1592 0 : transposeloop (l, k, j, i, SL, SK, SJ, SI, type_size);
1593 : }
1594 4 : break;
1595 : }
1596 :
1597 58 : return GST_FLOW_OK;
1598 : }
1599 :
1600 : /**
1601 : * @brief subrouting for tensor-transform, "stand" case.
1602 : * : pixel = abs((pixel - average(tensor))/(std(tensor) + val))
1603 : * @param[in/out] filter "this" pointer
1604 : * @param[in] in_info input tensor info
1605 : * @param[in] out_info output tensor info
1606 : * @param[in] inptr input tensor
1607 : * @param[out] outptr output tensor
1608 : * @return Gst flow status
1609 : */
1610 : static GstFlowReturn
1611 16 : gst_tensor_transform_stand (GstTensorTransform * filter,
1612 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1613 : const uint8_t * inptr, uint8_t * outptr)
1614 : {
1615 16 : GstFlowReturn ret = GST_FLOW_OK;
1616 : gsize in_element_size, out_element_size, data_size, ch_size;
1617 : gulong i, num, data_idx, ch;
1618 : gdouble tmp, *average, *std;
1619 :
1620 16 : in_element_size = gst_tensor_get_element_size (in_info->type);
1621 16 : out_element_size = gst_tensor_get_element_size (out_info->type);
1622 16 : num = gst_tensor_get_element_count (in_info->dimension);
1623 :
1624 16 : data_size = gst_tensor_info_get_size (in_info);
1625 16 : ch_size = in_info->dimension[0];
1626 :
1627 : /* calc average and std */
1628 16 : average = std = NULL;
1629 16 : if (filter->data_stand.per_channel) {
1630 8 : gst_tensor_data_raw_average_per_channel ((gpointer) inptr, data_size,
1631 8 : in_info->type, in_info->dimension, &average);
1632 : /* calculate std only for default mode */
1633 8 : if (filter->data_stand.mode == STAND_DEFAULT)
1634 4 : gst_tensor_data_raw_std_per_channel ((gpointer) inptr, data_size,
1635 4 : in_info->type, in_info->dimension, average, &std);
1636 : } else {
1637 8 : gst_tensor_data_raw_average ((gpointer) inptr, data_size,
1638 : in_info->type, &average);
1639 : /* calculate std only for default mode */
1640 8 : if (filter->data_stand.mode == STAND_DEFAULT)
1641 4 : gst_tensor_data_raw_std ((gpointer) inptr, data_size, in_info->type,
1642 : average, &std);
1643 : }
1644 :
1645 16 : switch (filter->data_stand.mode) {
1646 8 : case STAND_DEFAULT:
1647 : {
1648 8 : if (!filter->data_stand.per_channel) {
1649 20004 : for (i = 0; i < num; i++) {
1650 20000 : data_idx = in_element_size * i;
1651 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1652 : in_info->type, &tmp, _NNS_FLOAT64);
1653 :
1654 20000 : tmp = fabs ((tmp - *average) / *std);
1655 :
1656 20000 : data_idx = out_element_size * i;
1657 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1658 20000 : (gpointer) (outptr + data_idx), out_info->type);
1659 : }
1660 : } else {
1661 204 : for (ch = 0; ch < ch_size; ++ch) {
1662 20200 : for (i = 0; i < num / ch_size; i++) {
1663 20000 : data_idx = in_element_size * ((i * ch_size) + ch);
1664 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1665 : in_info->type, &tmp, _NNS_FLOAT64);
1666 :
1667 20000 : tmp = fabs ((tmp - average[ch]) / std[ch]);
1668 :
1669 20000 : data_idx = out_element_size * ((i * ch_size) + ch);
1670 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1671 20000 : (gpointer) (outptr + data_idx), out_info->type);
1672 : }
1673 : }
1674 : }
1675 8 : break;
1676 : }
1677 8 : case STAND_DC_AVERAGE:
1678 : {
1679 8 : if (!filter->data_stand.per_channel) {
1680 20004 : for (i = 0; i < num; i++) {
1681 20000 : data_idx = in_element_size * i;
1682 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1683 : in_info->type, &tmp, _NNS_FLOAT64);
1684 :
1685 20000 : tmp -= *average;
1686 :
1687 20000 : data_idx = out_element_size * i;
1688 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1689 20000 : (gpointer) (outptr + data_idx), out_info->type);
1690 : }
1691 : } else {
1692 204 : for (ch = 0; ch < ch_size; ++ch) {
1693 20200 : for (i = 0; i < num / ch_size; i++) {
1694 20000 : data_idx = in_element_size * ((i * ch_size) + ch);
1695 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1696 : in_info->type, &tmp, _NNS_FLOAT64);
1697 :
1698 20000 : tmp -= average[ch];
1699 :
1700 20000 : data_idx = out_element_size * ((i * ch_size) + ch);
1701 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1702 20000 : (gpointer) (outptr + data_idx), out_info->type);
1703 : }
1704 : }
1705 : }
1706 8 : break;
1707 : }
1708 0 : default:
1709 0 : GST_ERROR_OBJECT (filter, "Cannot identify mode\n");
1710 0 : ret = GST_FLOW_ERROR;
1711 : }
1712 :
1713 16 : g_free (average);
1714 16 : g_free (std);
1715 :
1716 16 : return ret;
1717 : }
1718 :
1719 : /**
1720 : * @brief subrouting for tensor-transform, "clamp" case.
1721 : * : pixel = if (pixel > max) ? max :
1722 : * if (pixel < min) ? min : pixel
1723 : * @param[in/out] filter "this" pointer
1724 : * @param[in] in_info input tensor info
1725 : * @param[in] out_info output tensor info
1726 : * @param[in] inptr input tensor
1727 : * @param[out] outptr output tensor
1728 : * @return Gst flow status
1729 : */
1730 : static GstFlowReturn
1731 16 : gst_tensor_transform_clamp (GstTensorTransform * filter,
1732 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1733 : const uint8_t * inptr, uint8_t * outptr)
1734 : {
1735 : gsize in_element_size, out_element_size;
1736 : gulong i, num, data_idx;
1737 : gdouble tmp;
1738 :
1739 16 : in_element_size = gst_tensor_get_element_size (in_info->type);
1740 16 : out_element_size = gst_tensor_get_element_size (out_info->type);
1741 16 : num = gst_tensor_get_element_count (in_info->dimension);
1742 :
1743 606016 : for (i = 0; i < num; ++i) {
1744 606000 : data_idx = in_element_size * i;
1745 606000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), in_info->type,
1746 : &tmp, _NNS_FLOAT64);
1747 :
1748 606000 : tmp = CLAMP (tmp, filter->data_clamp.min, filter->data_clamp.max);
1749 :
1750 606000 : data_idx = out_element_size * i;
1751 606000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, outptr + data_idx,
1752 : out_info->type);
1753 : }
1754 :
1755 16 : return GST_FLOW_OK;
1756 : }
1757 :
1758 : /**
1759 : * @brief subrouting for tensor-transform, "padding" case.
1760 : * @param[in/out] filter "this" pointer
1761 : * @param[in] in_info input tensor info
1762 : * @param[in] out_info output tensor info
1763 : * @param[in] inptr input tensor
1764 : * @param[out] outptr output tensor
1765 : * @return Gst flow status
1766 : */
1767 : static GstFlowReturn
1768 6 : gst_tensor_transform_padding (GstTensorTransform * filter,
1769 : GstTensorInfo * in_info, GstTensorInfo * out_info, const uint8_t * inptr,
1770 : uint8_t * outptr)
1771 : {
1772 : gsize element_size, in_loop_size, out_loop_size, copy_block_size;
1773 6 : guint i, j, k, left, top, front, loop_limit = 1;
1774 6 : element_size = gst_tensor_get_element_size (in_info->type);
1775 :
1776 6 : in_loop_size = (gsize) in_info->dimension[2] * in_info->dimension[1]
1777 6 : * in_info->dimension[0] * element_size;
1778 6 : out_loop_size =(gsize) out_info->dimension[2] * out_info->dimension[1]
1779 6 : * out_info->dimension[0] * element_size;
1780 6 : copy_block_size = in_info->dimension[0] * element_size;
1781 :
1782 9 : for (i = NNS_TENSOR_PADDING_RANK_LIMIT; i < NNS_TENSOR_RANK_LIMIT; i++) {
1783 9 : if (in_info->dimension[i] == 0)
1784 6 : break;
1785 3 : loop_limit *= in_info->dimension[i];
1786 : }
1787 :
1788 6 : left = filter->data_padding.pad[PADDING_LEFT];
1789 6 : top = filter->data_padding.pad[PADDING_TOP];
1790 6 : front = filter->data_padding.pad[PADDING_FRONT];
1791 :
1792 : /** @todo Add constant option instead of using zero padding value */
1793 6 : memset (outptr, 0, out_loop_size * loop_limit);
1794 :
1795 20 : for (i = 0; i < loop_limit; i++)
1796 56 : for (j = 0; j < in_info->dimension[2]; j++)
1797 2142 : for (k = 0; k < in_info->dimension[1]; k++) {
1798 2100 : guint in_idx = j * in_info->dimension[1] * in_info->dimension[0]
1799 2100 : + k * in_info->dimension[0];
1800 2100 : guint out_idx = j * out_info->dimension[1] * out_info->dimension[0]
1801 2100 : + k * out_info->dimension[0];
1802 :
1803 2100 : out_idx += left + top * out_info->dimension[0]
1804 2100 : + front * out_info->dimension[1] * out_info->dimension[0];
1805 :
1806 2100 : memcpy (outptr + out_idx * element_size + out_loop_size * i,
1807 2100 : inptr + in_idx * element_size + in_loop_size * i, copy_block_size);
1808 : }
1809 :
1810 6 : return GST_FLOW_OK;
1811 : }
1812 :
1813 : /**
1814 : * @brief non-ip transform. required vmethod for BaseTransform class.
1815 : * @param[in/out] trans "super" pointer
1816 : * @param[in] inbuf The input gst buffer
1817 : * @param[out] outbuf The output gst buffer
1818 : * @return Gst Flow Status
1819 : */
1820 : static GstFlowReturn
1821 22507 : gst_tensor_transform_transform (GstBaseTransform * trans,
1822 : GstBuffer * inbuf, GstBuffer * outbuf)
1823 : {
1824 : GstTensorTransform *filter;
1825 : GstTensorInfo *in_info, *out_info;
1826 22507 : GstFlowReturn res = GST_FLOW_ERROR;
1827 22507 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT] = { 0, };
1828 22507 : GstMemory *out_mem[NNS_TENSOR_SIZE_LIMIT] = { 0, };
1829 : GstMapInfo in_map[NNS_TENSOR_SIZE_LIMIT];
1830 : GstMapInfo out_map[NNS_TENSOR_SIZE_LIMIT];
1831 : uint8_t *inptr, *outptr;
1832 : guint i, num_tensors, num_mems;
1833 : gsize buf_size, hsize;
1834 : GstTensorMetaInfo meta;
1835 : GstTensorInfo in_flex_info, out_flex_info;
1836 : gboolean in_flexible, out_flexible;
1837 :
1838 22507 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
1839 :
1840 45010 : g_return_val_if_fail (filter->loaded, GST_FLOW_ERROR);
1841 22503 : inbuf = gst_tensor_buffer_from_config (inbuf, &filter->in_config);
1842 :
1843 22503 : in_flexible =
1844 22503 : gst_tensor_pad_caps_is_flexible (GST_BASE_TRANSFORM_SINK_PAD (trans));
1845 22503 : out_flexible =
1846 22503 : gst_tensor_pad_caps_is_flexible (GST_BASE_TRANSFORM_SRC_PAD (trans));
1847 :
1848 22503 : num_mems = gst_tensor_buffer_get_count (inbuf);
1849 22503 : if (in_flexible) {
1850 8 : num_tensors = num_mems;
1851 8 : g_return_val_if_fail (out_flexible, GST_FLOW_ERROR);
1852 : } else {
1853 22495 : num_tensors = filter->in_config.info.num_tensors;
1854 22495 : g_return_val_if_fail (num_mems == num_tensors, GST_FLOW_ERROR);
1855 : }
1856 :
1857 45092 : for (i = 0; i < num_tensors; i++) {
1858 22589 : in_info = gst_tensors_info_get_nth_info (&filter->in_config.info, i);
1859 22589 : out_info = gst_tensors_info_get_nth_info (&filter->out_config.info, i);
1860 :
1861 22589 : if (filter->apply && !g_list_find (filter->apply, GINT_TO_POINTER (i))) {
1862 15 : GstMemory *mem = gst_tensor_buffer_get_nth_memory (inbuf, i);
1863 :
1864 15 : if (!in_flexible && out_flexible) {
1865 0 : GstMemory *old = mem;
1866 :
1867 : /* append meta */
1868 0 : gst_tensor_info_convert_to_meta (out_info, &meta);
1869 0 : mem = gst_tensor_meta_info_append_header (&meta, old);
1870 0 : gst_memory_unref (old);
1871 : }
1872 :
1873 15 : gst_tensor_buffer_append_memory (outbuf, mem, out_info);
1874 15 : continue;
1875 : }
1876 :
1877 : /* parse input buffer */
1878 22574 : in_mem[i] = gst_tensor_buffer_get_nth_memory (inbuf, i);
1879 22574 : if (!gst_memory_map (in_mem[i], &in_map[i], GST_MAP_READ)) {
1880 0 : ml_loge ("Cannot map input buffer to gst-buf at tensor-transform.\n");
1881 0 : res = GST_FLOW_ERROR;
1882 0 : goto done;
1883 : }
1884 22574 : inptr = in_map[i].data;
1885 :
1886 22574 : if (in_flexible) {
1887 8 : in_info = &in_flex_info;
1888 8 : out_info = &out_flex_info;
1889 :
1890 8 : gst_tensor_meta_info_parse_header (&meta, inptr);
1891 : /** @todo max rank supported in tensor-transform is 4 */
1892 8 : if (!gst_tensor_meta_info_convert (&meta, in_info)) {
1893 0 : res = GST_FLOW_ERROR;
1894 0 : goto done;
1895 : }
1896 :
1897 8 : gst_tensor_transform_convert_dimension (filter, GST_PAD_SINK,
1898 : i, in_info, out_info);
1899 :
1900 8 : hsize = gst_tensor_meta_info_get_header_size (&meta);
1901 8 : inptr += hsize;
1902 : }
1903 :
1904 : /* prepare output buffer */
1905 22574 : buf_size = gst_tensor_info_get_size (out_info);
1906 22574 : if (out_flexible) {
1907 8 : gst_tensor_info_convert_to_meta (out_info, &meta);
1908 8 : hsize = gst_tensor_meta_info_get_header_size (&meta);
1909 8 : buf_size += hsize;
1910 : }
1911 :
1912 22574 : out_mem[i] = gst_allocator_alloc (NULL, buf_size, NULL);
1913 22574 : gst_tensor_buffer_append_memory (outbuf, out_mem[i], out_info);
1914 :
1915 22574 : if (!gst_memory_map (out_mem[i], &out_map[i], GST_MAP_WRITE)) {
1916 0 : ml_loge ("Cannot map output buffer to gst-buf at tensor-transform.\n");
1917 0 : res = GST_FLOW_ERROR;
1918 0 : goto done;
1919 : }
1920 22574 : outptr = out_map[i].data;
1921 :
1922 22574 : if (out_flexible) {
1923 8 : gst_tensor_meta_info_update_header (&meta, outptr);
1924 8 : outptr += hsize;
1925 : }
1926 :
1927 22574 : switch (filter->mode) {
1928 52 : case GTT_DIMCHG:
1929 52 : res = gst_tensor_transform_dimchg (filter, in_info, out_info,
1930 : inptr, outptr);
1931 52 : break;
1932 579 : case GTT_TYPECAST:
1933 579 : res = gst_tensor_transform_typecast (filter, in_info, out_info,
1934 : inptr, outptr);
1935 579 : break;
1936 21847 : case GTT_ARITHMETIC:
1937 21847 : res = gst_tensor_transform_arithmetic (filter, in_info, out_info,
1938 : inptr, outptr);
1939 21847 : break;
1940 58 : case GTT_TRANSPOSE:
1941 58 : res = gst_tensor_transform_transpose (filter, in_info, out_info,
1942 : inptr, outptr);
1943 58 : break;
1944 16 : case GTT_STAND:
1945 16 : res = gst_tensor_transform_stand (filter, in_info, out_info,
1946 : inptr, outptr);
1947 16 : break;
1948 16 : case GTT_CLAMP:
1949 16 : res = gst_tensor_transform_clamp (filter, in_info, out_info,
1950 : inptr, outptr);
1951 16 : break;
1952 6 : case GTT_PADDING:
1953 6 : res = gst_tensor_transform_padding (filter, in_info, out_info,
1954 : inptr, outptr);
1955 6 : break;
1956 0 : default:
1957 0 : ml_loge ("Not supported tensor transform mode");
1958 0 : res = GST_FLOW_NOT_SUPPORTED;
1959 0 : goto done;
1960 : }
1961 : }
1962 :
1963 22503 : done:
1964 45092 : for (i = 0; i < num_tensors; i++) {
1965 22589 : if (in_mem[i]) {
1966 22574 : gst_memory_unmap (in_mem[i], &in_map[i]);
1967 22574 : gst_memory_unref (in_mem[i]);
1968 : }
1969 22589 : if (out_mem[i])
1970 22574 : gst_memory_unmap (out_mem[i], &out_map[i]);
1971 : }
1972 :
1973 22503 : return res;
1974 : }
1975 :
1976 : /**
1977 : * @brief Read cap, parse tensor configuration (dim/type) from the cap.
1978 : * @param[in] filter "this" pointer
1979 : * @param[in] caps The input caps to be read
1980 : * @param[out] config configured tensor info
1981 : * @return TRUE if successful (both dim/type read). FALSE if not.
1982 : */
1983 : static gboolean
1984 440 : gst_tensor_transform_read_caps (GstTensorTransform * filter,
1985 : const GstCaps * caps, GstTensorsConfig * config)
1986 : {
1987 : GstStructure *structure;
1988 440 : g_return_val_if_fail (config != NULL, FALSE);
1989 :
1990 440 : structure = gst_caps_get_structure (caps, 0);
1991 :
1992 440 : if (!gst_tensors_config_from_structure (config, structure)) {
1993 0 : GST_WARNING_OBJECT (filter, "caps is not tensor %s\n",
1994 : gst_structure_get_name (structure));
1995 0 : return FALSE;
1996 : }
1997 :
1998 440 : return gst_tensors_config_validate (config);
1999 : }
2000 :
2001 : /**
2002 : * @brief Dimension conversion calculation
2003 : * @param[in] filter "this" pointer
2004 : * @param[in] direction GST_PAD_SINK if input->output conv
2005 : * @param[in] idx index of the input tensors
2006 : * @param[in] in_info tensor info structure of source tensor (input if direction is SINK)
2007 : * @param[out] out_info tensor info structure of destination tensor (output if direction is SINK)
2008 : * @return TRUE if success
2009 : */
2010 : static gboolean
2011 3640 : gst_tensor_transform_convert_dimension (GstTensorTransform * filter,
2012 : GstPadDirection direction, guint idx, const GstTensorInfo * in_info,
2013 : GstTensorInfo * out_info)
2014 : {
2015 : guint i;
2016 :
2017 : /* copy input info first, then update output info */
2018 3640 : gst_tensor_info_copy (out_info, in_info);
2019 :
2020 3640 : if (filter->apply && !g_list_find (filter->apply, GINT_TO_POINTER (idx)))
2021 10 : return TRUE;
2022 :
2023 3630 : switch (filter->mode) {
2024 70 : case GTT_DIMCHG:
2025 : {
2026 70 : unsigned int from = filter->data_dimchg.from;
2027 70 : unsigned int to = filter->data_dimchg.to;
2028 :
2029 70 : if (direction == GST_PAD_SINK) {
2030 850 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) {
2031 800 : if ((i < from && i < to) || (i > from && i > to) || from == to) {
2032 634 : out_info->dimension[i] = in_info->dimension[i];
2033 166 : } else if (i == to) {
2034 50 : out_info->dimension[i] = in_info->dimension[from];
2035 116 : } else if (from > to) {
2036 0 : g_assert (i > 0 && i > to);
2037 0 : out_info->dimension[i] = in_info->dimension[i - 1];
2038 : } else {
2039 116 : g_assert (i < to && i < (NNS_TENSOR_RANK_LIMIT - 1));
2040 116 : out_info->dimension[i] = in_info->dimension[i + 1];
2041 : }
2042 : }
2043 : } else {
2044 340 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) {
2045 320 : if ((i < from && i < to) || (i > from && i > to) || from == to) {
2046 252 : out_info->dimension[i] = in_info->dimension[i];
2047 68 : } else if (i == from) {
2048 20 : out_info->dimension[i] = in_info->dimension[to];
2049 48 : } else if (from > to) {
2050 0 : g_assert (i < from && i < (NNS_TENSOR_RANK_LIMIT - 1));
2051 0 : out_info->dimension[i] = in_info->dimension[i + 1];
2052 : } else {
2053 48 : g_assert (i > 0 && i > from);
2054 48 : out_info->dimension[i] = in_info->dimension[i - 1];
2055 : }
2056 : }
2057 : }
2058 70 : break;
2059 : }
2060 1896 : case GTT_TYPECAST:
2061 : /** For both directions, dimension does not change */
2062 1896 : if (direction == GST_PAD_SINK) {
2063 : /** src = SINKPAD / dest = SRCPAD */
2064 1042 : out_info->type = filter->data_typecast.to;
2065 : } else {
2066 : /* cannot get the incoming data type on sink pad */
2067 854 : out_info->type = _NNS_END;
2068 : }
2069 1896 : break;
2070 :
2071 1011 : case GTT_ARITHMETIC:
2072 : /* check arith mode option has typecast operator */
2073 1011 : if (filter->data_arithmetic.out_type != _NNS_END) {
2074 785 : if (direction == GST_PAD_SINK) {
2075 414 : out_info->type = filter->data_arithmetic.out_type;
2076 : } else {
2077 : /* cannot get the incoming data type on sink pad */
2078 371 : out_info->type = _NNS_END;
2079 : }
2080 : }
2081 1011 : break;
2082 :
2083 281 : case GTT_TRANSPOSE:
2084 281 : if (direction == GST_PAD_SINK) {
2085 715 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
2086 572 : out_info->dimension[i] =
2087 572 : in_info->dimension[filter->data_transpose.trans_order[i]];
2088 : }
2089 : } else {
2090 690 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
2091 552 : g_assert (filter->data_transpose.trans_order[i] <
2092 : NNS_TENSOR_RANK_LIMIT);
2093 552 : out_info->dimension[filter->data_transpose.trans_order[i]] =
2094 552 : in_info->dimension[i];
2095 : }
2096 : }
2097 281 : break;
2098 :
2099 40 : case GTT_STAND:
2100 : /** For both directions, dimension does not change */
2101 40 : if (direction == GST_PAD_SINK) {
2102 24 : if (filter->data_stand.out_type != _NNS_END)
2103 12 : out_info->type = filter->data_stand.out_type;
2104 : } else {
2105 : /* cannot get the incoming data type on sink pad */
2106 16 : out_info->type = _NNS_END;
2107 : }
2108 40 : break;
2109 :
2110 136 : case GTT_CLAMP:
2111 : /* same tensors info, do nothing. */
2112 136 : break;
2113 :
2114 44 : case GTT_PADDING:
2115 44 : if (direction == GST_PAD_SINK) {
2116 30 : out_info->dimension[0] +=
2117 30 : filter->data_padding.pad[PADDING_LEFT] +
2118 30 : filter->data_padding.pad[PADDING_RIGHT];
2119 30 : out_info->dimension[1] +=
2120 30 : filter->data_padding.pad[PADDING_TOP] +
2121 30 : filter->data_padding.pad[PADDING_BOTTOM];
2122 30 : out_info->dimension[2] +=
2123 30 : filter->data_padding.pad[PADDING_FRONT] +
2124 30 : filter->data_padding.pad[PADDING_BACK];
2125 : }
2126 44 : break;
2127 152 : default:
2128 152 : return FALSE;
2129 : }
2130 :
2131 3478 : return TRUE;
2132 : }
2133 :
2134 : /**
2135 : * @brief configure srcpad cap from "proposed" cap. (required vmethod for BaseTransform)
2136 : *
2137 : * @param trans ("this" pointer)
2138 : * @param direction (why do we need this?)
2139 : * @param caps sinkpad cap
2140 : * @param filtercap this element's cap (don't know specifically.)
2141 : *
2142 : * @todo Get to know what the heck is @filtercap and use it!
2143 : */
2144 : static GstCaps *
2145 3171 : gst_tensor_transform_transform_caps (GstBaseTransform * trans,
2146 : GstPadDirection direction, GstCaps * caps, GstCaps * filtercap)
2147 : {
2148 : GstTensorTransform *filter;
2149 3171 : GstCaps *result = NULL;
2150 : GstStructure *structure;
2151 : guint i, j;
2152 :
2153 3171 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2154 :
2155 3171 : silent_debug (filter, "Calling TransformCaps, direction = %d\n", direction);
2156 3171 : silent_debug_caps (filter, caps, "from");
2157 3171 : silent_debug_caps (filter, filtercap, "filter");
2158 :
2159 3171 : result = gst_caps_new_empty ();
2160 9317 : for (i = 0; i < gst_caps_get_size (caps); i++) {
2161 : GstTensorsConfig in_config, out_config;
2162 : GstTensorInfo *in_info, *out_info;
2163 6146 : gboolean is_types_not_fixed = FALSE;
2164 6146 : GstCaps *result_aux = gst_caps_new_empty ();
2165 :
2166 6146 : gst_tensors_config_init (&out_config);
2167 :
2168 6146 : structure = gst_caps_get_structure (caps, i);
2169 6146 : gst_tensors_config_from_structure (&in_config, structure);
2170 :
2171 6146 : if (gst_tensors_config_is_flexible (&in_config)) {
2172 : /* output caps is also flexible */
2173 714 : out_config.info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
2174 : } else {
2175 8837 : for (j = 0; j < in_config.info.num_tensors; j++) {
2176 3405 : in_info = gst_tensors_info_get_nth_info (&in_config.info, j);
2177 3405 : out_info = gst_tensors_info_get_nth_info (&out_config.info, j);
2178 :
2179 3405 : gst_tensor_transform_convert_dimension (filter, direction,
2180 : j, in_info, out_info);
2181 3405 : if (out_info->type == _NNS_END) {
2182 : /* types cannot be specified */
2183 1801 : is_types_not_fixed = TRUE;
2184 : }
2185 : }
2186 : }
2187 :
2188 6146 : out_config.rate_d = in_config.rate_d;
2189 6146 : out_config.rate_n = in_config.rate_n;
2190 6146 : out_config.info.num_tensors = in_config.info.num_tensors;
2191 :
2192 6146 : if (gst_structure_has_name (structure, NNS_MIMETYPE_TENSOR)) {
2193 2891 : gst_caps_append (result_aux, gst_tensor_caps_from_config (&out_config));
2194 : } else {
2195 3255 : gst_caps_append (result_aux, gst_tensors_caps_from_config (&out_config));
2196 :
2197 : /* remove `types` field from caps */
2198 3255 : if (is_types_not_fixed) {
2199 144 : GstStructure *s = gst_caps_get_structure (result_aux, 0);
2200 144 : gst_structure_remove_field (s, "types");
2201 : }
2202 : }
2203 :
2204 6146 : gst_caps_append (result, result_aux);
2205 :
2206 6146 : gst_tensors_config_free (&in_config);
2207 6146 : gst_tensors_config_free (&out_config);
2208 : }
2209 :
2210 3171 : if (filtercap && gst_caps_get_size (filtercap) > 0) {
2211 : GstCaps *intersection;
2212 :
2213 : intersection =
2214 638 : gst_caps_intersect_full (result, filtercap, GST_CAPS_INTERSECT_FIRST);
2215 :
2216 638 : gst_caps_unref (result);
2217 638 : result = intersection;
2218 : }
2219 :
2220 3171 : silent_debug_caps (filter, result, "to");
2221 3171 : return result;
2222 : }
2223 :
2224 : /**
2225 : * @brief fixate caps. required vmethod of BaseTransform
2226 : */
2227 : static GstCaps *
2228 242 : gst_tensor_transform_fixate_caps (GstBaseTransform * trans,
2229 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
2230 : {
2231 : GstTensorTransform *filter;
2232 : GstCaps *result;
2233 :
2234 242 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2235 :
2236 242 : silent_debug (filter, "Calling FixateCaps, direction = %d\n", direction);
2237 242 : silent_debug_caps (filter, caps, "caps");
2238 242 : silent_debug_caps (filter, othercaps, "othercaps");
2239 :
2240 : result =
2241 242 : gst_tensor_transform_transform_caps (trans, direction, caps, othercaps);
2242 242 : gst_caps_unref (othercaps);
2243 :
2244 242 : result = gst_caps_make_writable (result);
2245 242 : result = gst_caps_fixate (result);
2246 :
2247 242 : silent_debug_caps (filter, result, "result");
2248 242 : return result;
2249 : }
2250 :
2251 : /**
2252 : * @brief set caps. required vmethod of BaseTransform
2253 : */
2254 : static gboolean
2255 220 : gst_tensor_transform_set_caps (GstBaseTransform * trans,
2256 : GstCaps * incaps, GstCaps * outcaps)
2257 : {
2258 : GstTensorTransform *filter;
2259 : GstTensorsConfig in_config, out_config;
2260 : GstTensorsConfig config;
2261 : GstTensorInfo *in_info, *out_info;
2262 : gboolean in_flexible, out_flexible;
2263 220 : gboolean allowed = FALSE;
2264 : guint i;
2265 :
2266 220 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2267 :
2268 220 : silent_debug (filter, "Calling SetCaps\n");
2269 220 : silent_debug_caps (filter, incaps, "incaps");
2270 220 : silent_debug_caps (filter, outcaps, "outcaps");
2271 :
2272 220 : if (!gst_tensor_transform_read_caps (filter, incaps, &in_config)) {
2273 0 : GST_ERROR_OBJECT (filter, "Cannot read cap of incaps\n");
2274 0 : goto error;
2275 : }
2276 :
2277 220 : if (!gst_tensor_transform_read_caps (filter, outcaps, &out_config)) {
2278 0 : GST_ERROR_OBJECT (filter, "Cannot read cap of outcaps\n");
2279 0 : goto error;
2280 : }
2281 :
2282 220 : in_flexible = gst_tensors_config_is_flexible (&in_config);
2283 220 : out_flexible = gst_tensors_config_is_flexible (&out_config);
2284 :
2285 : /* compare type and dimension */
2286 220 : gst_tensors_config_init (&config);
2287 220 : config.info.format = out_config.info.format;
2288 :
2289 220 : config.rate_n = in_config.rate_n;
2290 220 : config.rate_d = in_config.rate_d;
2291 220 : config.info.num_tensors = in_config.info.num_tensors;
2292 :
2293 220 : if (!in_flexible) {
2294 438 : for (i = 0; i < in_config.info.num_tensors; i++) {
2295 227 : in_info = gst_tensors_info_get_nth_info (&in_config.info, i);
2296 227 : out_info = gst_tensors_info_get_nth_info (&config.info, i);
2297 :
2298 227 : if (!gst_tensor_transform_convert_dimension (filter, GST_PAD_SINK,
2299 : i, in_info, out_info)) {
2300 0 : GST_ERROR_OBJECT (filter,
2301 : "Tensor info is not matched with given properties.");
2302 0 : goto error;
2303 : }
2304 : }
2305 : }
2306 :
2307 220 : if (out_flexible) {
2308 9 : GST_INFO_OBJECT (filter, "Output tensor is flexible.");
2309 :
2310 : /* set output configuration if input is static */
2311 9 : if (!in_flexible)
2312 0 : out_config = config;
2313 211 : } else if (!gst_tensors_config_is_equal (&out_config, &config)) {
2314 0 : GST_ERROR_OBJECT (filter,
2315 : "Tensor info is not matched with given properties.\n");
2316 0 : goto error;
2317 : }
2318 :
2319 : /* set in/out tensor info */
2320 220 : filter->in_config = in_config;
2321 220 : filter->out_config = out_config;
2322 220 : allowed = TRUE;
2323 :
2324 220 : error:
2325 220 : if (!allowed)
2326 0 : GST_ERROR_OBJECT (filter, "Set Caps Failed!\n");
2327 :
2328 220 : return allowed;
2329 : }
2330 :
2331 : /**
2332 : * @brief Tell the framework the required size of buffer based on the info of the other side pad. Note that this is always the same with the input. optional vmethod of BaseTransform
2333 : */
2334 : static gboolean
2335 22507 : gst_tensor_transform_transform_size (GstBaseTransform * trans,
2336 : GstPadDirection direction, GstCaps * caps, gsize size, GstCaps * othercaps,
2337 : gsize * othersize)
2338 : {
2339 : UNUSED (trans);
2340 : UNUSED (direction);
2341 : UNUSED (caps);
2342 : UNUSED (size);
2343 : UNUSED (othercaps);
2344 : /**
2345 : * Consider multi-tensors.
2346 : * Set each memory block in transform()
2347 : */
2348 22507 : *othersize = 0;
2349 :
2350 22507 : return TRUE;
2351 : }
|
