Line data Source code
1 : /**
2 : * GStreamer
3 : * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
4 : * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
5 : * Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>
6 : *
7 : * This library is free software; you can redistribute it and/or
8 : * modify it under the terms of the GNU Library General Public
9 : * License as published by the Free Software Foundation;
10 : * version 2.1 of the License.
11 : *
12 : * This library is distributed in the hope that it will be useful,
13 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : * Library General Public License for more details.
16 : *
17 : */
18 : /**
19 : * @file gsttensor_transform.c
20 : * @date 10 Jul 2018
21 : * @brief GStreamer plugin to transform tensor dimension or type
22 : * @see https://github.com/nnstreamer/nnstreamer
23 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
24 : * @bug This is NYI.
25 : *
26 : */
27 :
28 : /**
29 : * SECTION:element-tensor_transform
30 : *
31 : * A filter that transforms tensors dimension or type.
32 : * The input and output is always in the format of other/tensor or other/tensors.
33 : *
34 : * <refsect2>
35 : * <title>Example launch line</title>
36 : * |[
37 : * gst-launch -v -m fakesrc ! tensor_converter ! tensor_transform mode=dimchg option=0:2 ! fakesink silent=TRUE
38 : * ]|
39 : * <title>How to use dimchg</title>
40 : * |[
41 : * option=0:2 # Move 0th dim to 2nd dim. I.e., [a][H][W][C] ==> [a][C][H][W]
42 : * ]|
43 : * </refsect2>
44 : */
45 :
46 : #ifdef HAVE_CONFIG_H
47 : #include <config.h>
48 : #endif
49 :
50 : #include <string.h>
51 : #include <math.h>
52 : #include <nnstreamer_log.h>
53 : #include <nnstreamer_util.h>
54 : #include "gsttensor_transform.h"
55 :
56 : #ifdef HAVE_ORC
57 : #include "nnstreamer-orc.h"
58 : #endif
59 :
60 : /**
61 : * @brief Macro for debug mode.
62 : */
63 : #ifndef DBG
64 : #define DBG (!filter->silent)
65 : #endif
66 :
67 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_transform_debug);
68 : #define GST_CAT_DEFAULT gst_tensor_transform_debug
69 : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_MAKE ("{ static, flexible }")
70 : #define REGEX_DIMCHG_OPTION "^([0-9]|1[0-5]):([0-9]|1[0-5])$"
71 : #define REGEX_TYPECAST_OPTION "(^[u]?int(8|16|32|64)$|^float(16|32|64)$)"
72 : #define REGEX_TRANSPOSE_OPTION "^(?:([0-2]):(?!.*\\1)){3}3$"
73 : #define REGEX_STAND_OPTION "^(default|dc-average)(:([u]?int(8|16|32|64)|float(16|32|64)))?(,per-channel:(true|false))?$"
74 : #define REGEX_CLAMP_OPTION "^((([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?))):"\
75 : "((([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?)))$"
76 : #define REGEX_PADDING_OPTION "^((left|right|top|bottom|front|back):(\\d)(,)?)+(layout:(NCHW|NHWC))?$"
77 : #define REGEX_ARITH_OPTION "^(typecast:([u]?int(8|16|32|64)|float(16|32|64)),)?"\
78 : "(per-channel:(false|true@[0-9]+),)?"\
79 : "(((add|mul|div)(:([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?))+(@[0-9]+)?)(,|))+$"
80 :
81 : #define REGEX_ARITH_OPTION_TYPECAST "(typecast:([u]?int(8|16|32|64)|float(16|32|64)))"
82 :
83 : /**
84 : * @brief The transpose rank is fixed to 4.
85 : * This RANK does not affect other/tensors(s)'s NNS_TENSOR_RANK_LIMIT.
86 : */
87 : #define NNS_TENSOR_TRANSPOSE_RANK_LIMIT (4)
88 :
89 : /**
90 : * @brief The padding rank is fixed to 3.
91 : * This RANK does not affect other/tensors(s)'s NNS_TENSOR_RANK_LIMIT.
92 : */
93 : #define NNS_TENSOR_PADDING_RANK_LIMIT (3)
94 :
95 : /**
96 : * @brief tensor_transform properties
97 : */
98 : enum
99 : {
100 : PROP_0,
101 : PROP_SILENT,
102 : PROP_MODE,
103 : PROP_OPTION,
104 : PROP_ACCELERATION,
105 : PROP_APPLY,
106 : PROP_TRANSPOSE_RANK_LIMIT
107 : };
108 :
109 : /**
110 : * @brief Flag to set orc acceleration.
111 : */
112 : #ifdef HAVE_ORC
113 : #define DEFAULT_ACCELERATION TRUE
114 : #else
115 : #define DEFAULT_ACCELERATION FALSE
116 : #endif
117 :
118 : static const gchar *gst_tensor_transform_stand_string[] = {
119 : [STAND_DEFAULT] = "default",
120 : [STAND_DC_AVERAGE] = "dc-average",
121 : [STAND_END] = NULL
122 : };
123 :
124 : static const gchar *gst_tensor_transform_operator_string[] = {
125 : [GTT_OP_TYPECAST] = "typecast",
126 : [GTT_OP_ADD] = "add",
127 : [GTT_OP_MUL] = "mul",
128 : [GTT_OP_DIV] = "div",
129 : [GTT_OP_UNKNOWN] = NULL
130 : };
131 :
132 : /**
133 : * @brief The capabilities of the inputs
134 : */
135 : static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
136 : GST_PAD_SINK,
137 : GST_PAD_ALWAYS,
138 : GST_STATIC_CAPS (CAPS_STRING));
139 :
140 : /**
141 : * @brief The capabilities of the outputs
142 : */
143 : static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
144 : GST_PAD_SRC,
145 : GST_PAD_ALWAYS,
146 : GST_STATIC_CAPS (CAPS_STRING));
147 :
148 : #define gst_tensor_transform_parent_class parent_class
149 1828 : G_DEFINE_TYPE (GstTensorTransform, gst_tensor_transform,
150 : GST_TYPE_BASE_TRANSFORM);
151 :
152 : /* GObject vmethod implementations */
153 : static void gst_tensor_transform_set_property (GObject * object, guint prop_id,
154 : const GValue * value, GParamSpec * pspec);
155 : static void gst_tensor_transform_get_property (GObject * object, guint prop_id,
156 : GValue * value, GParamSpec * pspec);
157 : static void gst_tensor_transform_finalize (GObject * object);
158 :
159 : /* GstBaseTransformer vmethod implementations */
160 : static GstFlowReturn gst_tensor_transform_transform (GstBaseTransform * trans,
161 : GstBuffer * inbuf, GstBuffer * outbuf);
162 : static GstCaps *gst_tensor_transform_transform_caps (GstBaseTransform * trans,
163 : GstPadDirection direction, GstCaps * caps, GstCaps * filter);
164 : static GstCaps *gst_tensor_transform_fixate_caps (GstBaseTransform * trans,
165 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
166 : static gboolean gst_tensor_transform_set_caps (GstBaseTransform * trans,
167 : GstCaps * incaps, GstCaps * outcaps);
168 : static gboolean gst_tensor_transform_transform_size (GstBaseTransform * trans,
169 : GstPadDirection direction, GstCaps * caps, gsize size,
170 : GstCaps * othercaps, gsize * othersize);
171 :
172 : static gboolean gst_tensor_transform_convert_dimension (GstTensorTransform *
173 : filter, GstPadDirection direction, guint idx, const GstTensorInfo * in_info,
174 : GstTensorInfo * out_info);
175 :
176 : #define GST_TYPE_TENSOR_TRANSFORM_MODE (gst_tensor_transform_mode_get_type ())
177 : /**
178 : * @brief A private function to register GEnumValue array for the 'mode' property
179 : * to a GType and return it
180 : */
181 : static GType
182 97 : gst_tensor_transform_mode_get_type (void)
183 : {
184 : static GType mode_type = 0;
185 :
186 97 : if (mode_type == 0) {
187 : static GEnumValue mode_types[] = {
188 : {GTT_DIMCHG, "Mode for changing tensor dimensions, "
189 : "option=FROM_DIM:TO_DIM (with a regex, " REGEX_DIMCHG_OPTION
190 : ", where NNS_TENSOR_RANK_LIMIT is 16)",
191 : "dimchg"},
192 : {GTT_TYPECAST, "Mode for casting type of tensor, "
193 : "option=" REGEX_TYPECAST_OPTION, "typecast"},
194 : {GTT_ARITHMETIC, "Mode for arithmetic operations with tensor, "
195 : "option=[typecast:TYPE,][per-channel:(false|true@DIM),]add|mul|div:NUMBER[@CH_IDX], ...",
196 : "arithmetic"},
197 : {GTT_TRANSPOSE, "Mode for transposing shape of tensor, "
198 : "option=D1\':D2\':D3\':D4 (fixed to 3)",
199 : "transpose"},
200 : {GTT_STAND, "Mode for statistical standardization of tensor, "
201 : "option=(default|dc-average)[:TYPE][,per-channel:(false|true)]",
202 : "stand"},
203 : {GTT_CLAMP, "Mode for clamping all elements of tensor into the range, "
204 : "option=CLAMP_MIN:CLAMP_MAX",
205 : "clamp"},
206 : {GTT_PADDING, "Mode for padding of tensor, "
207 : "option=left|right|top|bottom|front|back:NUMBER[,layout:(NCHW|NHWC)]",
208 : "padding"},
209 : {GTT_UNKNOWN, "Unknown or not-implemented-yet mode",
210 : "unknown"},
211 : {0, NULL, NULL},
212 : };
213 :
214 97 : mode_type = g_enum_register_static ("gtt_mode_type", mode_types);
215 : }
216 :
217 97 : return mode_type;
218 : }
219 :
220 : /**
221 : * @brief initialize the tensor_transform's class
222 : */
223 : static void
224 97 : gst_tensor_transform_class_init (GstTensorTransformClass * klass)
225 : {
226 : GObjectClass *gobject_class;
227 : GstElementClass *gstelement_class;
228 : GstBaseTransformClass *trans_class;
229 :
230 97 : GST_DEBUG_CATEGORY_INIT (gst_tensor_transform_debug, "tensor_transform", 0,
231 : "Element to transforms tensor dimension or type");
232 :
233 97 : trans_class = (GstBaseTransformClass *) klass;
234 97 : gstelement_class = (GstElementClass *) trans_class;
235 97 : gobject_class = (GObjectClass *) gstelement_class;
236 :
237 97 : gobject_class->set_property = gst_tensor_transform_set_property;
238 97 : gobject_class->get_property = gst_tensor_transform_get_property;
239 97 : gobject_class->finalize = gst_tensor_transform_finalize;
240 :
241 97 : g_object_class_install_property (gobject_class, PROP_SILENT,
242 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output ?",
243 : FALSE, G_PARAM_READWRITE));
244 97 : g_object_class_install_property (gobject_class, PROP_MODE,
245 : g_param_spec_enum ("mode", "Mode", "Mode used for transforming tensor",
246 : GST_TYPE_TENSOR_TRANSFORM_MODE, GTT_UNKNOWN,
247 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
248 97 : g_object_class_install_property (gobject_class, PROP_OPTION,
249 : g_param_spec_string ("option", "Option",
250 : "Option for the tensor transform mode ?", "", G_PARAM_READWRITE));
251 97 : g_object_class_install_property (gobject_class, PROP_ACCELERATION,
252 : g_param_spec_boolean ("acceleration", "Acceleration", "Orc acceleration",
253 : DEFAULT_ACCELERATION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
254 97 : g_object_class_install_property (gobject_class, PROP_APPLY,
255 : g_param_spec_string ("apply", "Apply", "Select tensors to apply, "
256 : "separated with ',' in case of multiple tensors. Default to apply all tensors.",
257 : "", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
258 97 : g_object_class_install_property (gobject_class, PROP_TRANSPOSE_RANK_LIMIT,
259 : g_param_spec_uint ("transpose-rank-limit", "Transpose rank limit",
260 : "The rank limit of transpose, which varies per version of nnstreamer and may be lower than the global rank limit if it is over 4.",
261 : 0, NNS_TENSOR_RANK_LIMIT, NNS_TENSOR_TRANSPOSE_RANK_LIMIT,
262 : G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
263 :
264 97 : gst_element_class_set_details_simple (gstelement_class,
265 : "TensorTransform",
266 : "Filter/Tensor",
267 : "Transforms other/tensor dimensions for different models or frameworks",
268 : "MyungJoo Ham <myungjoo.ham@samsung.com>");
269 :
270 97 : gst_element_class_add_pad_template (gstelement_class,
271 : gst_static_pad_template_get (&src_factory));
272 97 : gst_element_class_add_pad_template (gstelement_class,
273 : gst_static_pad_template_get (&sink_factory));
274 : /* Refer: https://gstreamer.freedesktop.org/documentation/design/element-transform.html */
275 97 : trans_class->passthrough_on_same_caps = FALSE;
276 :
277 : /* Processing units */
278 97 : trans_class->transform = GST_DEBUG_FUNCPTR (gst_tensor_transform_transform);
279 :
280 : /* Negotiation units */
281 97 : trans_class->transform_caps =
282 97 : GST_DEBUG_FUNCPTR (gst_tensor_transform_transform_caps);
283 97 : trans_class->fixate_caps =
284 97 : GST_DEBUG_FUNCPTR (gst_tensor_transform_fixate_caps);
285 97 : trans_class->set_caps = GST_DEBUG_FUNCPTR (gst_tensor_transform_set_caps);
286 :
287 : /* Allocation units */
288 97 : trans_class->transform_size =
289 97 : GST_DEBUG_FUNCPTR (gst_tensor_transform_transform_size);
290 97 : }
291 :
292 : /**
293 : * @brief initialize the new element (G_DEFINE_TYPE requires this)
294 : * instantiate pads and add them to element
295 : * set pad callback functions
296 : * initialize instance structure
297 : */
298 : static void
299 257 : gst_tensor_transform_init (GstTensorTransform * filter)
300 : {
301 257 : filter->silent = TRUE;
302 257 : filter->mode = GTT_UNKNOWN;
303 257 : filter->option = NULL;
304 257 : filter->loaded = FALSE;
305 257 : filter->operators = NULL;
306 257 : filter->acceleration = DEFAULT_ACCELERATION;
307 257 : filter->apply = NULL;
308 :
309 257 : gst_tensors_config_init (&filter->in_config);
310 257 : gst_tensors_config_init (&filter->out_config);
311 257 : }
312 :
313 : /**
314 : * @brief Get the corresponding operator from the string value
315 : * @param[in] str The string value for the operator
316 : * @return corresponding operator for the string (GTT_OP_UNKNOWN for errors)
317 : */
318 : static tensor_transform_operator
319 178 : gst_tensor_transform_get_operator (const gchar * str)
320 : {
321 : int index;
322 :
323 178 : index = find_key_strv (gst_tensor_transform_operator_string, str);
324 :
325 178 : return (index < 0) ? GTT_OP_UNKNOWN : index;
326 : }
327 :
328 : /**
329 : * @brief Get the corresponding mode from the string value
330 : * @param[in] str The string value for the mode
331 : * @return corresponding mode for the string. STAND_END for errors
332 : */
333 : static tensor_transform_stand_mode
334 4 : gst_tensor_transform_get_stand_mode (const gchar * str)
335 : {
336 : int index;
337 :
338 4 : index = find_key_strv (gst_tensor_transform_stand_string, str);
339 :
340 4 : return (index < 0) ? STAND_END : index;
341 : }
342 :
343 : #ifndef FLOAT16_SUPPORT
344 : /**
345 : * @brief Generate error if float16 is required.
346 : */
347 : static void
348 0 : float16_not_supported (void)
349 : {
350 0 : ml_loge
351 : ("Tensor_transform does not support float16 operators. Apply -Denable-float16=true for meson build option if your architecture support float16. Note that tensor-transform's float16 is adhoc and does NOT perform good (slow!).\n");
352 0 : g_assert (0);
353 : }
354 : #endif
355 :
356 : #ifdef FLOAT16_SUPPORT
357 : /**
358 : * @brief Refrain from heavy operations on float16
359 : * @todo Remove this after applying SIMD or ORC
360 : */
361 : static void
362 : refrain_from_heavy_op_on_float16 (gulong n)
363 : {
364 : static int warned = 0;
365 : /* 1 million */
366 : if (n > 1000000) {
367 : if (warned)
368 : return;
369 : ml_logw
370 : ("Tensor_transform implementation for float16 does not support SIMD. Heavy tensor-transform operations of float16 is not recommended. Try to apply heavy ops with other types (e.g., float32) and convert it to float16 at the time when it's really needed.\n");
371 : warned = 1;
372 : }
373 : }
374 :
375 : /** @todo Make this use SIMD or ORC */
376 : #define _conv_to_f16(intype, o, i, n) \
377 : do { \
378 : float16 *op = (gpointer) (o); \
379 : intype *ip = (gpointer) (i); \
380 : gulong idx; \
381 : refrain_from_heavy_op_on_float16 (n); \
382 : for (idx = 0; idx < n; idx++) \
383 : *(op + idx) = (float16) *(ip + idx); \
384 : } while (0)
385 :
386 : /** @todo Make this use SIMD or ORC */
387 : #define _conv_from_f16_action(n, op, ip, otypename) \
388 : do { \
389 : gulong idx; \
390 : for (idx = 0; idx < n; idx++) \
391 : *(op + idx) = (otypename) *(ip + idx); \
392 : } while (0)
393 :
394 : /** @todo Make this use SIMD or ORC */
395 : #define _conv_from_f16(otype, o, i, n) \
396 : do { \
397 : float16 *ip = (gpointer) (i); \
398 : refrain_from_heavy_op_on_float16 (n); \
399 : switch (otype) { \
400 : case _NNS_INT32: { \
401 : int32_t *op = (gpointer) (o); \
402 : _conv_from_f16_action (n, op, ip, int32_t); \
403 : break; } \
404 : case _NNS_UINT32: { \
405 : uint32_t *op = (gpointer) (o); \
406 : _conv_from_f16_action (n, op, ip, uint32_t); \
407 : break; } \
408 : case _NNS_INT16: { \
409 : int16_t *op = (gpointer) (o); \
410 : _conv_from_f16_action (n, op, ip, int16_t); \
411 : break; } \
412 : case _NNS_UINT16: { \
413 : uint16_t *op = (gpointer) (o); \
414 : _conv_from_f16_action (n, op, ip, uint16_t); \
415 : break; } \
416 : case _NNS_INT8: { \
417 : int8_t *op = (gpointer) (o); \
418 : _conv_from_f16_action (n, op, ip, int8_t); \
419 : break; } \
420 : case _NNS_UINT8: { \
421 : uint8_t *op = (gpointer) (o); \
422 : _conv_from_f16_action (n, op, ip, uint8_t); \
423 : break; } \
424 : case _NNS_FLOAT64: { \
425 : double *op = (gpointer) (o); \
426 : _conv_from_f16_action (n, op, ip, double); \
427 : break; } \
428 : case _NNS_FLOAT32: { \
429 : float *op = (gpointer) (o); \
430 : _conv_from_f16_action (n, op, ip, float); \
431 : break; } \
432 : case _NNS_FLOAT16: { \
433 : float16 *op = (gpointer) (o); \
434 : _conv_from_f16_action (n, op, ip, float16); \
435 : break; } \
436 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (otype)); g_assert (0); \
437 : } \
438 : } while (0)
439 :
440 : /** @todo Make this use SIMD or ORC */
441 : #define _op_float16(i, n, v, op) \
442 : do { \
443 : gulong idx; \
444 : float16 *data_in = (float16 *) (i); \
445 : refrain_from_heavy_op_on_float16 (n); \
446 : switch (op) { \
447 : case GTT_OP_ADD: \
448 : for (idx = 0; idx < n; idx++) \
449 : data_in[idx] = data_in[idx] + (v); \
450 : break; \
451 : case GTT_OP_MUL: \
452 : for (idx = 0; idx < n; idx++) \
453 : data_in[idx] = data_in[idx] * (v); \
454 : break; \
455 : case GTT_OP_DIV: \
456 : for (idx = 0; idx < n; idx++) \
457 : data_in[idx] = data_in[idx] / (v); \
458 : break; \
459 : default: GST_ERROR_OBJECT (filter, "Unknown operator for float16: %d", op); break; \
460 : } \
461 : } while (0)
462 :
463 : #else /* ! FLOAT16_SUPPORT */
464 : #define _conv_to_f16(intype, o, i, n) do { float16_not_supported (); } while (0)
465 : #define _conv_from_f16(otype, o, i, n) do { float16_not_supported (); } while (0)
466 : #define _op_float16(i, n, v, op) do { float16_not_supported (); } while (0)
467 : #endif /* FLOAT16_SUPPORT */
468 :
469 : #ifdef HAVE_ORC
470 : /* define macros for orc */
471 : #define orc_func_conv(intype,outtype) nns_orc_conv_ ## intype ## _to_ ## outtype
472 : #define orc_func_add(intype) nns_orc_add_c_ ## intype
473 : #define orc_func_mul(intype) nns_orc_mul_c_ ## intype
474 : #define orc_func_div(intype) nns_orc_div_c_ ## intype
475 :
476 : #define orc_typecast_to(i,o,n,intype,otype,intypename) do { \
477 : switch (otype) { \
478 : case _NNS_INT32: orc_func_conv (intype, s32) ((gpointer) o, (gpointer) i, n); break; \
479 : case _NNS_UINT32: orc_func_conv (intype, u32) ((gpointer) o, (gpointer) i, n); break; \
480 : case _NNS_INT16: orc_func_conv (intype, s16) ((gpointer) o, (gpointer) i, n); break; \
481 : case _NNS_UINT16: orc_func_conv (intype, u16) ((gpointer) o, (gpointer) i, n); break; \
482 : case _NNS_INT8: orc_func_conv (intype, s8) ((gpointer) o, (gpointer) i, n); break; \
483 : case _NNS_UINT8: orc_func_conv (intype, u8) ((gpointer) o, (gpointer) i, n); break; \
484 : case _NNS_FLOAT64: orc_func_conv (intype, f64) ((gpointer) o, (gpointer) i, n); break; \
485 : case _NNS_FLOAT32: orc_func_conv (intype, f32) ((gpointer) o, (gpointer) i, n); break; \
486 : case _NNS_INT64: orc_func_conv (intype, s64) ((gpointer) o, (gpointer) i, n); break; \
487 : case _NNS_UINT64: orc_func_conv (intype, u64) ((gpointer) o, (gpointer) i, n); break; \
488 : case _NNS_FLOAT16: _conv_to_f16 (intypename, o, i, n); break; \
489 : default: GST_ERROR_OBJECT (filter, "Unsupported output type %d", otype); g_assert (0); break; \
490 : } \
491 : } while (0)
492 :
493 : #define orc_typecast(i,o,n,itype,otype) do { \
494 : switch (itype) { \
495 : case _NNS_INT32: orc_typecast_to (i, o, n, s32, otype, int32_t); break; \
496 : case _NNS_UINT32: orc_typecast_to (i, o, n, u32, otype, uint32_t); break; \
497 : case _NNS_INT16: orc_typecast_to (i, o, n, s16, otype, int16_t); break; \
498 : case _NNS_UINT16: orc_typecast_to (i, o, n, u16, otype, uint16_t); break; \
499 : case _NNS_INT8: orc_typecast_to (i, o, n, s8, otype, int8_t); break; \
500 : case _NNS_UINT8: orc_typecast_to (i, o, n, u8, otype, uint8_t); break; \
501 : case _NNS_FLOAT64: orc_typecast_to (i, o, n, f64, otype, double); break; \
502 : case _NNS_FLOAT32: orc_typecast_to (i, o, n, f32, otype, float); break; \
503 : case _NNS_INT64: orc_typecast_to (i, o, n, s64, otype, int64_t); break; \
504 : case _NNS_UINT64: orc_typecast_to (i, o, n, u64, otype, uint64_t); break; \
505 : case _NNS_FLOAT16: _conv_from_f16 (otype, o, i, n); break; \
506 : default: GST_ERROR_OBJECT (filter, "Unsupported input type %d", itype); g_assert (0); break; \
507 : } \
508 : } while (0)
509 :
510 : #define orc_typesize(size, type) do { \
511 : switch (type) { \
512 : case _NNS_INT32: size = sizeof(int32_t); break; \
513 : case _NNS_UINT32: size = sizeof(uint32_t); break; \
514 : case _NNS_INT16: size = sizeof(int16_t); break; \
515 : case _NNS_UINT16: size = sizeof(uint16_t); break; \
516 : case _NNS_INT8: size = sizeof(int8_t); break; \
517 : case _NNS_UINT8: size = sizeof(uint8_t); break; \
518 : case _NNS_FLOAT64: size = sizeof(double); break; \
519 : case _NNS_FLOAT32: size = sizeof(float); break; \
520 : case _NNS_INT64: size = sizeof(int64_t); break; \
521 : case _NNS_UINT64: size = sizeof(uint64_t); break; \
522 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", type); g_assert (0); break; \
523 : } \
524 : } while (0)
525 :
526 : #define orc_operator_func(i,n,v,opfunc,op) do { \
527 : switch ((v)->type) { \
528 : case _NNS_INT32: opfunc (s32) ((gpointer) i, (v)->data._int32_t, n); break; \
529 : case _NNS_UINT32: opfunc (u32) ((gpointer) i, (v)->data._uint32_t, n); break; \
530 : case _NNS_INT16: opfunc (s16) ((gpointer) i, (v)->data._int16_t, n); break; \
531 : case _NNS_UINT16: opfunc (u16) ((gpointer) i, (v)->data._uint16_t, n); break; \
532 : case _NNS_INT8: opfunc (s8) ((gpointer) i, (v)->data._int8_t, n); break; \
533 : case _NNS_UINT8: opfunc (u8) ((gpointer) i, (v)->data._uint8_t, n); break; \
534 : case _NNS_FLOAT64: opfunc (f64) ((gpointer) i, (v)->data._double, n); break; \
535 : case _NNS_FLOAT32: opfunc (f32) ((gpointer) i, (v)->data._float, n); break; \
536 : case _NNS_INT64: opfunc (s64) ((gpointer) i, (v)->data._int64_t, n); break; \
537 : case _NNS_UINT64: opfunc (u64) ((gpointer) i, (v)->data._uint64_t, n); break; \
538 : case _NNS_FLOAT16: _op_float16 (i, n, (v)->data._float16, op); break; \
539 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (v)->type); g_assert (0); break; \
540 : } \
541 : } while (0)
542 :
543 : #define orc_operator_div_loop(i,n,val,typename) do { \
544 : gsize idx_div; \
545 : typename *data_in = (typename *) (i); \
546 : for (idx_div = 0; idx_div < (n); ++idx_div) { \
547 : data_in[idx_div] = data_in[idx_div] / (val); \
548 : } \
549 : } while (0)
550 :
551 : #define orc_operator(i,n,v,op) do { \
552 : switch (op) { \
553 : case GTT_OP_ADD: orc_operator_func (i, n, v, orc_func_add, op); break; \
554 : case GTT_OP_MUL: orc_operator_func (i, n, v, orc_func_mul, op); break; \
555 : case GTT_OP_DIV: \
556 : switch ((v)->type) { \
557 : case _NNS_INT32: orc_operator_div_loop (i, n, (v)->data._int32_t, int32_t); break; \
558 : case _NNS_UINT32: orc_operator_div_loop (i, n, (v)->data._uint32_t, uint32_t); break; \
559 : case _NNS_INT16: orc_operator_div_loop (i, n, (v)->data._int16_t, int16_t); break; \
560 : case _NNS_UINT16: orc_operator_div_loop (i, n, (v)->data._uint16_t, uint16_t); break; \
561 : case _NNS_INT8: orc_operator_div_loop (i, n, (v)->data._int8_t, int8_t); break; \
562 : case _NNS_UINT8: orc_operator_div_loop (i, n, (v)->data._uint8_t, uint8_t); break; \
563 : case _NNS_FLOAT64: orc_func_div (f64) ((gpointer) i, (v)->data._double, n); break; \
564 : case _NNS_FLOAT32: orc_func_div (f32) ((gpointer) i, (v)->data._float, n); break; \
565 : case _NNS_INT64: orc_operator_div_loop (i, n, (v)->data._int64_t, int64_t); break; \
566 : case _NNS_UINT64: orc_operator_div_loop (i, n, (v)->data._uint64_t, uint64_t); break; \
567 : case _NNS_FLOAT16: _op_float16 (i, n, (v)->data._float16, op); break; \
568 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (v)->type); g_assert (0); break; \
569 : } \
570 : break; \
571 : default: GST_ERROR_OBJECT (filter, "Unknown operator %d", op); break; \
572 : } \
573 : } while (0)
574 : #endif /* HAVE_ORC */
575 :
576 : /**
577 : * @brief Macro for operator
578 : */
579 : #define handle_operator(d,v,oper,vtype) do { \
580 : switch (oper) { \
581 : case GTT_OP_ADD: \
582 : (d)->data._##vtype += (v)->data._##vtype; \
583 : break; \
584 : case GTT_OP_MUL: \
585 : (d)->data._##vtype *= (v)->data._##vtype; \
586 : break; \
587 : case GTT_OP_DIV: \
588 : if ((v)->data._##vtype == 0) { \
589 : GST_ERROR_OBJECT (filter, "Invalid state, denominator is 0."); \
590 : return FALSE; \
591 : } \
592 : (d)->data._##vtype /= (v)->data._##vtype; \
593 : break; \
594 : default: \
595 : GST_ERROR_OBJECT (filter, "Unknown operator %d", oper); \
596 : return FALSE; \
597 : } \
598 : } while (0)
599 :
600 : /**
601 : * @brief Handle operators for tensor value
602 : * @param filter "this" pointer
603 : * @param desc struct for tensor value
604 : * @param val struct for tensor value
605 : * @param op operator for given tensor value
606 : * @return TRUE if no error
607 : */
608 : static gboolean
609 61615 : gst_tensor_transform_do_operator (GstTensorTransform * filter,
610 : tensor_data_s * desc, const tensor_data_s * val,
611 : tensor_transform_operator op)
612 : {
613 61615 : g_return_val_if_fail (desc != NULL, FALSE);
614 61615 : g_return_val_if_fail (val != NULL, FALSE);
615 61615 : g_return_val_if_fail (desc->type == val->type, FALSE);
616 :
617 61615 : switch (desc->type) {
618 7755 : case _NNS_INT32:
619 7755 : handle_operator (desc, val, op, int32_t);
620 7755 : break;
621 7680 : case _NNS_UINT32:
622 7680 : handle_operator (desc, val, op, uint32_t);
623 7680 : break;
624 7680 : case _NNS_INT16:
625 7680 : handle_operator (desc, val, op, int16_t);
626 7680 : break;
627 7680 : case _NNS_UINT16:
628 7680 : handle_operator (desc, val, op, uint16_t);
629 7680 : break;
630 7680 : case _NNS_INT8:
631 7680 : handle_operator (desc, val, op, int8_t);
632 7680 : break;
633 7680 : case _NNS_UINT8:
634 7680 : handle_operator (desc, val, op, uint8_t);
635 7680 : break;
636 45 : case _NNS_FLOAT64:
637 45 : handle_operator (desc, val, op, double);
638 45 : break;
639 55 : case _NNS_FLOAT32:
640 55 : handle_operator (desc, val, op, float);
641 55 : break;
642 0 : case _NNS_FLOAT16:
643 : #ifdef FLOAT16_SUPPORT
644 : handle_operator (desc, val, op, float16);
645 : #else
646 0 : float16_not_supported ();
647 : #endif
648 0 : break;
649 7680 : case _NNS_INT64:
650 7680 : handle_operator (desc, val, op, int64_t);
651 7680 : break;
652 7680 : case _NNS_UINT64:
653 7680 : handle_operator (desc, val, op, uint64_t);
654 7680 : break;
655 0 : default:
656 0 : GST_ERROR_OBJECT (filter, "Unknown tensor type %d", desc->type);
657 0 : return FALSE;
658 : }
659 :
660 61615 : return TRUE;
661 : }
662 :
663 : /**
664 : * @brief Setup internal data (data_* in GstTensorTransform)
665 : * @param[in/out] filter "this" pointer. mode & option MUST BE set already.
666 : * @retval TRUE if OK or operation-skipped, FALSE if fatal-error.
667 : */
668 : static gboolean
669 547 : gst_tensor_transform_set_option_data (GstTensorTransform * filter)
670 : {
671 : gchar *filter_name;
672 547 : gboolean ret = FALSE;
673 :
674 547 : if (filter->mode == GTT_UNKNOWN || filter->option == NULL)
675 286 : return TRUE;
676 :
677 261 : filter_name = gst_object_get_name ((GstObject *) filter);
678 :
679 261 : switch (filter->mode) {
680 8 : case GTT_DIMCHG:
681 : {
682 8 : gchar **strv = NULL;
683 :
684 8 : if (!g_regex_match_simple (REGEX_DIMCHG_OPTION, filter->option,
685 : G_REGEX_CASELESS, 0)) {
686 3 : ml_loge
687 : ("%s: dimchg: \'%s\' is not valid option string: it should be in the form of IDX_DIM_FROM:IDX_DIM_TO: with a regex, "
688 : REGEX_DIMCHG_OPTION "\n", filter_name, filter->option);
689 3 : break;
690 : }
691 :
692 5 : strv = g_strsplit (filter->option, ":", 2);
693 :
694 5 : filter->data_dimchg.from = (int) g_ascii_strtoll (strv[0], NULL, 10);
695 5 : filter->data_dimchg.to = (int) g_ascii_strtoll (strv[1], NULL, 10);
696 5 : ret = filter->loaded = TRUE;
697 5 : g_strfreev (strv);
698 5 : break;
699 : }
700 119 : case GTT_TYPECAST:
701 : {
702 119 : if (g_regex_match_simple (REGEX_TYPECAST_OPTION, filter->option,
703 : G_REGEX_CASELESS, 0)) {
704 118 : filter->data_typecast.to = gst_tensor_get_type (filter->option);
705 118 : ret = filter->loaded = TRUE;
706 : } else {
707 1 : ml_loge
708 : ("%s: typecast: \'%s\' is not valid data type for tensor: data type of tensor should be one of %s\n",
709 : filter_name, filter->option, GST_TENSOR_TYPE_ALL);
710 : }
711 119 : break;
712 : }
713 91 : case GTT_ARITHMETIC:
714 : {
715 : gchar *str_option;
716 : gchar **str_operators;
717 : gchar **str_op;
718 : tensor_transform_operator_s *op_s;
719 : guint i, num_operators, num_op;
720 : GRegex *regex_option_tc;
721 :
722 91 : filter->data_arithmetic.out_type = _NNS_END;
723 91 : filter->data_arithmetic.per_channel_arith = FALSE;
724 :
725 91 : if (filter->operators) {
726 2 : GST_WARNING_OBJECT (filter,
727 : "There exists pre-defined operators (total %d), now reset these.",
728 : g_slist_length (filter->operators));
729 :
730 2 : g_slist_free_full (filter->operators, g_free);
731 2 : filter->operators = NULL;
732 : }
733 :
734 91 : regex_option_tc = g_regex_new (REGEX_ARITH_OPTION_TYPECAST,
735 : G_REGEX_CASELESS, 0, 0);
736 :
737 91 : if (!regex_option_tc) {
738 0 : GST_ERROR_OBJECT (filter,
739 : "arithmetic: failed to create a GRegex structure for %s\n",
740 : REGEX_ARITH_OPTION_TYPECAST);
741 0 : break;
742 : }
743 :
744 91 : if (g_regex_match_full (regex_option_tc, filter->option, -1,
745 : 1, 0, NULL, NULL)) {
746 2 : str_option = g_regex_replace (regex_option_tc, filter->option, -1, 1,
747 : "", 0, 0);
748 2 : ml_loge
749 : ("%s: arithmetic: [typecast:TYPE,] should be located at the first to prevent memory re-allocation: typecast(s) in the middle of \'%s\' will be ignored\n",
750 : filter_name, filter->option);
751 : } else {
752 178 : str_option = g_strdup (filter->option);
753 : }
754 91 : g_regex_unref (regex_option_tc);
755 :
756 91 : if (!g_regex_match_simple (REGEX_ARITH_OPTION, str_option,
757 : G_REGEX_CASELESS, 0)) {
758 13 : ml_loge
759 : ("%s: arithmetic: \'%s\' is not valid option string: it should be in the form of [typecast:TYPE,][per-channel:(false|true@DIM),]add|mul|div:NUMBER[@CH_IDX]..., ...\n",
760 : filter_name, str_option);
761 13 : g_free (str_option);
762 13 : break;
763 : }
764 78 : str_operators = g_strsplit (str_option, ",", -1);
765 78 : num_operators = g_strv_length (str_operators);
766 :
767 260 : for (i = 0; i < num_operators; ++i) {
768 182 : str_op = g_strsplit (str_operators[i], ":", -1);
769 182 : num_op = g_strv_length (str_op);
770 :
771 182 : if (str_op[0]) {
772 180 : gchar **values = g_strsplit (str_op[1], "@", -1);
773 180 : guint num_values = g_strv_length (values);
774 :
775 : /* check whether per-channel */
776 180 : if (g_ascii_strcasecmp (str_op[0], "per-channel") == 0) {
777 2 : if (num_values > 1 && g_ascii_strcasecmp (values[0], "true") == 0) {
778 2 : ml_logi
779 : ("Set per-channel for arithmetic and assume that %s-th dim is the channel",
780 : values[1]);
781 2 : filter->data_arithmetic.per_channel_arith = TRUE;
782 2 : filter->data_arithmetic.ch_dim =
783 2 : (guint) g_ascii_strtoull (values[1], NULL, 10);
784 : }
785 :
786 2 : g_strfreev (values);
787 2 : g_strfreev (str_op);
788 2 : continue;
789 : }
790 :
791 178 : op_s = g_new0 (tensor_transform_operator_s, 1);
792 178 : g_assert (op_s);
793 :
794 178 : op_s->op = gst_tensor_transform_get_operator (str_op[0]);
795 178 : op_s->applying_ch = -1; /* -1 means applying to all channels */
796 178 : switch (op_s->op) {
797 61 : case GTT_OP_TYPECAST:
798 61 : if (num_op > 1 && str_op[1]) {
799 61 : op_s->value.type = gst_tensor_get_type (values[0]);
800 61 : filter->data_arithmetic.out_type = op_s->value.type;
801 : } else {
802 0 : GST_WARNING_OBJECT (filter, "Invalid option for typecast %s",
803 : str_operators[i]);
804 0 : op_s->op = GTT_OP_UNKNOWN;
805 : }
806 61 : break;
807 117 : case GTT_OP_ADD:
808 : case GTT_OP_MUL:
809 : case GTT_OP_DIV:
810 117 : if (num_op > 1 && str_op[1]) {
811 : /* get operand */
812 117 : if (strchr (values[0], '.') || strchr (values[0], 'e') ||
813 117 : strchr (values[0], 'E')) {
814 : double val;
815 :
816 81 : val = g_ascii_strtod (values[0], NULL);
817 81 : gst_tensor_data_set (&op_s->value, _NNS_FLOAT64, &val);
818 : } else {
819 : int64_t val;
820 :
821 36 : val = g_ascii_strtoll (values[0], NULL, 10);
822 36 : gst_tensor_data_set (&op_s->value, _NNS_INT64, &val);
823 : }
824 :
825 117 : if (filter->data_arithmetic.per_channel_arith && num_values > 1) {
826 11 : op_s->applying_ch = g_ascii_strtoll (values[1], NULL, 10);
827 : }
828 :
829 : } else {
830 0 : GST_WARNING_OBJECT (filter,
831 : "Invalid option for arithmetic %s", str_operators[i]);
832 0 : op_s->op = GTT_OP_UNKNOWN;
833 : }
834 117 : break;
835 0 : default:
836 0 : GST_WARNING_OBJECT (filter, "Unknown operator %s", str_op[0]);
837 0 : break;
838 : }
839 :
840 : /* append operator */
841 178 : if (op_s->op != GTT_OP_UNKNOWN) {
842 178 : filter->operators = g_slist_append (filter->operators, op_s);
843 : } else {
844 0 : g_free (op_s);
845 : }
846 :
847 178 : g_strfreev (values);
848 : } else {
849 2 : GST_WARNING_OBJECT (filter, "Invalid option %s", str_operators[i]);
850 : }
851 :
852 180 : g_strfreev (str_op);
853 : }
854 :
855 78 : ret = filter->loaded = (filter->operators != NULL);
856 78 : g_strfreev (str_operators);
857 78 : g_free (str_option);
858 78 : break;
859 : }
860 20 : case GTT_TRANSPOSE:
861 : {
862 : int i;
863 20 : gchar **strv = NULL;
864 :
865 20 : if (!g_regex_match_simple (REGEX_TRANSPOSE_OPTION, filter->option,
866 : G_REGEX_CASELESS, 0)) {
867 4 : ml_loge
868 : ("%s: transpose: \'%s\' is not valid option string: it should be in the form of NEW_IDX_DIM0:NEW_IDX_DIM1:NEW_IDX_DIM2:3 (Now transpose mode's rank is fixed to 3. Note that the index of the last dim is always fixed to 3)\n",
869 : filter_name, filter->option);
870 4 : break;
871 : }
872 :
873 16 : strv = g_strsplit (filter->option, ":", NNS_TENSOR_TRANSPOSE_RANK_LIMIT);
874 80 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
875 64 : filter->data_transpose.trans_order[i] =
876 64 : (uint8_t) g_ascii_strtoull (strv[i], NULL, 10);
877 : }
878 :
879 16 : ret = filter->loaded = TRUE;
880 16 : g_strfreev (strv);
881 16 : break;
882 : }
883 9 : case GTT_STAND:
884 : {
885 9 : gchar **options = NULL;
886 : guint i, num_options;
887 :
888 9 : if (!g_regex_match_simple (REGEX_STAND_OPTION, filter->option,
889 : G_REGEX_CASELESS, 0)) {
890 5 : ml_loge
891 : ("%s: stand: \'%s\' is not a valid option string: it should be in the form of (default|dc-average)[:TYPE][,per-channel:(false|true)]\n",
892 : filter_name, filter->option);
893 5 : break;
894 : }
895 :
896 4 : filter->data_stand.out_type = _NNS_END;
897 4 : filter->data_stand.per_channel = FALSE;
898 :
899 4 : options = g_strsplit (filter->option, ",", -1);
900 4 : num_options = g_strv_length (options);
901 :
902 10 : for (i = 0; i < num_options; i++) {
903 6 : gchar **strv = g_strsplit (options[i], ":", -1);
904 :
905 6 : if (g_ascii_strcasecmp (strv[0], "default") == 0 ||
906 4 : g_ascii_strcasecmp (strv[0], "dc-average") == 0) {
907 4 : filter->data_stand.mode =
908 4 : gst_tensor_transform_get_stand_mode (strv[0]);
909 6 : if (g_strv_length (strv) > 1)
910 2 : filter->data_stand.out_type = gst_tensor_get_type (strv[1]);
911 2 : } else if (g_ascii_strcasecmp (strv[0], "per-channel") == 0) {
912 2 : if (g_strv_length (strv) > 1 &&
913 2 : g_ascii_strcasecmp (strv[1], "true") == 0)
914 2 : filter->data_stand.per_channel = TRUE;
915 : } else {
916 0 : filter->data_stand.mode = STAND_END;
917 0 : ml_logw ("Unknown option for stand mode: %s", strv[0]);
918 : }
919 :
920 6 : g_strfreev (strv);
921 : }
922 :
923 4 : g_strfreev (options);
924 4 : ret = filter->loaded = TRUE;
925 4 : break;
926 : }
927 10 : case GTT_CLAMP:
928 : {
929 10 : gchar **strv = NULL;
930 :
931 10 : if (!g_regex_match_simple (REGEX_CLAMP_OPTION, filter->option,
932 : G_REGEX_CASELESS, 0)) {
933 1 : ml_loge
934 : ("%s: clamp: \'%s\' is not valid option string: it should be in the form of [CLAMP_MIN:CLAMP_MAX]\n",
935 : filter_name, filter->option);
936 1 : break;
937 : }
938 :
939 9 : strv = g_strsplit (filter->option, ":", 2);
940 :
941 9 : filter->data_clamp.min = g_ascii_strtod (strv[0], NULL);
942 9 : if (errno == ERANGE) {
943 1 : ml_loge ("%s: clamp: CLAMP_MIN value has an invalid range\n",
944 : filter_name);
945 1 : g_strfreev (strv);
946 1 : break;
947 : }
948 8 : filter->data_clamp.max = g_ascii_strtod (strv[1], NULL);
949 8 : if (errno == ERANGE) {
950 1 : ml_loge ("%s: clamp: CLAMP_MAX value has an invalid range\n",
951 : filter_name);
952 1 : g_strfreev (strv);
953 1 : break;
954 : }
955 :
956 7 : g_strfreev (strv);
957 :
958 7 : if (filter->data_clamp.min > filter->data_clamp.max) {
959 2 : ml_loge ("%s: clamp: CLAMP_MIN is larger than CLAMP_MAX\n",
960 : filter_name);
961 2 : break;
962 : }
963 :
964 5 : ret = filter->loaded = TRUE;
965 5 : break;
966 : }
967 4 : case GTT_PADDING:
968 : {
969 4 : gchar **options = NULL;
970 : guint i, num_options;
971 :
972 4 : if (!g_regex_match_simple (REGEX_PADDING_OPTION, filter->option,
973 : G_REGEX_CASELESS, 0)) {
974 0 : ml_loge
975 : ("%s: padding: \'%s\' is not valid option string: it should be in the form of left|right|top|bottom|front|back:PADDING,[layout:(NCHW|NHWC)]\n",
976 : filter_name, filter->option);
977 0 : break;
978 : }
979 :
980 68 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++)
981 64 : filter->data_padding.pad[i] = 0;
982 4 : filter->data_padding.layout = _NNS_LAYOUT_ANY;
983 :
984 4 : options = g_strsplit (filter->option, ",", -1);
985 4 : num_options = g_strv_length (options);
986 :
987 23 : for (i = 0; i < num_options; i++) {
988 19 : gchar **strv = g_strsplit (options[i], ":", 2);
989 19 : if (g_ascii_strcasecmp (strv[0], "left") == 0) {
990 4 : filter->data_padding.pad[PADDING_LEFT] =
991 4 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
992 15 : } else if (g_ascii_strcasecmp (strv[0], "right") == 0) {
993 4 : filter->data_padding.pad[PADDING_RIGHT] =
994 4 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
995 11 : } else if (g_ascii_strcasecmp (strv[0], "top") == 0) {
996 3 : filter->data_padding.pad[PADDING_TOP] =
997 3 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
998 8 : } else if (g_ascii_strcasecmp (strv[0], "bottom") == 0) {
999 3 : filter->data_padding.pad[PADDING_BOTTOM] =
1000 3 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1001 5 : } else if (g_ascii_strcasecmp (strv[0], "front") == 0) {
1002 2 : filter->data_padding.pad[PADDING_FRONT] =
1003 2 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1004 3 : } else if (g_ascii_strcasecmp (strv[0], "back") == 0) {
1005 2 : filter->data_padding.pad[PADDING_BACK] =
1006 2 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1007 1 : } else if (g_ascii_strcasecmp (strv[0], "layout") == 0) {
1008 1 : if (g_ascii_strcasecmp (strv[1], "NHWC") == 0)
1009 1 : filter->data_padding.layout = _NNS_LAYOUT_NHWC;
1010 : else
1011 0 : filter->data_padding.layout = _NNS_LAYOUT_NCHW;
1012 : } else {
1013 0 : ml_logw ("Unknown option for padding mode: %s", strv[0]);
1014 : }
1015 19 : g_strfreev (strv);
1016 : }
1017 4 : g_strfreev (options);
1018 :
1019 4 : if (filter->data_padding.layout == _NNS_LAYOUT_NHWC) {
1020 1 : guint prev_left = filter->data_padding.pad[PADDING_LEFT],
1021 1 : prev_right = filter->data_padding.pad[PADDING_RIGHT];
1022 1 : filter->data_padding.pad[PADDING_LEFT] =
1023 1 : filter->data_padding.pad[PADDING_FRONT];
1024 1 : filter->data_padding.pad[PADDING_RIGHT] =
1025 1 : filter->data_padding.pad[PADDING_BACK];
1026 1 : filter->data_padding.pad[PADDING_FRONT] = prev_left;
1027 1 : filter->data_padding.pad[PADDING_BACK] = prev_right;
1028 : }
1029 :
1030 4 : ret = filter->loaded = TRUE;
1031 4 : break;
1032 : }
1033 0 : default:
1034 0 : GST_ERROR_OBJECT (filter, "Cannot identify mode\n");
1035 0 : ret = FALSE;
1036 : }
1037 :
1038 261 : g_free (filter_name);
1039 261 : return ret;
1040 : }
1041 :
1042 : /**
1043 : * @brief Set property (gst element vmethod)
1044 : */
1045 : static void
1046 578 : gst_tensor_transform_set_property (GObject * object, guint prop_id,
1047 : const GValue * value, GParamSpec * pspec)
1048 : {
1049 578 : GstTensorTransform *filter = GST_TENSOR_TRANSFORM (object);
1050 :
1051 578 : switch (prop_id) {
1052 1 : case PROP_SILENT:
1053 1 : filter->silent = g_value_get_boolean (value);
1054 1 : break;
1055 256 : case PROP_MODE:
1056 256 : filter->mode = g_value_get_enum (value);
1057 256 : gst_tensor_transform_set_option_data (filter);
1058 256 : break;
1059 260 : case PROP_OPTION:
1060 : {
1061 260 : gchar *backup_option = filter->option;
1062 260 : filter->option = g_value_dup_string (value);
1063 260 : if (gst_tensor_transform_set_option_data (filter)) {
1064 229 : silent_debug (filter, "Option = %s --> %s\n", backup_option,
1065 : filter->option);
1066 229 : g_free (backup_option);
1067 : } else {
1068 : /* ERROR! Revert the change! */
1069 31 : g_free (filter->option);
1070 31 : filter->option = backup_option;
1071 31 : gst_tensor_transform_set_option_data (filter);
1072 : }
1073 260 : break;
1074 : }
1075 58 : case PROP_ACCELERATION:
1076 : #ifdef HAVE_ORC
1077 58 : filter->acceleration = g_value_get_boolean (value);
1078 58 : silent_debug (filter, "acceleration = %d\n", filter->acceleration);
1079 : #else
1080 : GST_WARNING_OBJECT (filter, "Orc acceleration is not supported");
1081 : filter->acceleration = FALSE;
1082 : #endif
1083 58 : break;
1084 3 : case PROP_APPLY:
1085 : {
1086 : gint64 val;
1087 3 : const gchar *param = g_value_get_string (value);
1088 3 : gchar **strv = g_strsplit_set (param, ",", -1);
1089 3 : guint i, num = g_strv_length (strv);
1090 3 : gchar *endptr = NULL;
1091 :
1092 10 : for (i = 0; i < num; i++) {
1093 7 : errno = 0;
1094 7 : val = g_ascii_strtoll (strv[i], &endptr, 10);
1095 7 : if (errno == ERANGE || errno == EINVAL || (endptr == strv[i])) {
1096 0 : ml_loge ("Cannot convert string %s to a gint64 value", strv[i]);
1097 : }
1098 7 : filter->apply = g_list_append (filter->apply, GINT_TO_POINTER (val));
1099 : }
1100 3 : g_strfreev (strv);
1101 3 : break;
1102 : }
1103 0 : default:
1104 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1105 0 : break;
1106 : }
1107 578 : }
1108 :
1109 : /**
1110 : * @brief Get property (gst element vmethod)
1111 : */
1112 : static void
1113 41 : gst_tensor_transform_get_property (GObject * object, guint prop_id,
1114 : GValue * value, GParamSpec * pspec)
1115 : {
1116 41 : GstTensorTransform *filter = GST_TENSOR_TRANSFORM (object);
1117 :
1118 41 : switch (prop_id) {
1119 2 : case PROP_SILENT:
1120 2 : g_value_set_boolean (value, filter->silent);
1121 2 : break;
1122 1 : case PROP_MODE:
1123 1 : g_value_set_enum (value, filter->mode);
1124 1 : break;
1125 36 : case PROP_OPTION:
1126 36 : g_value_set_string (value, filter->option);
1127 36 : break;
1128 2 : case PROP_ACCELERATION:
1129 2 : g_value_set_boolean (value, filter->acceleration);
1130 2 : break;
1131 0 : case PROP_APPLY:
1132 : {
1133 : GList *list;
1134 : gchar *p;
1135 : GPtrArray *arr;
1136 : gchar **strings;
1137 :
1138 0 : if (filter->apply == NULL) {
1139 0 : g_value_set_string (value, "");
1140 0 : return;
1141 : }
1142 :
1143 0 : arr = g_ptr_array_new ();
1144 0 : for (list = filter->apply; list != NULL; list = list->next) {
1145 0 : g_ptr_array_add (arr, g_strdup_printf ("%i",
1146 0 : GPOINTER_TO_INT (list->data)));
1147 : }
1148 0 : g_ptr_array_add (arr, NULL);
1149 0 : strings = (gchar **) g_ptr_array_free (arr, FALSE);
1150 0 : p = g_strjoinv (",", strings);
1151 :
1152 0 : g_strfreev (strings);
1153 0 : g_value_take_string (value, p);
1154 0 : break;
1155 : }
1156 0 : case PROP_TRANSPOSE_RANK_LIMIT:
1157 0 : g_value_set_uint (value, NNS_TENSOR_TRANSPOSE_RANK_LIMIT);
1158 0 : break;
1159 0 : default:
1160 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1161 0 : break;
1162 : }
1163 : }
1164 :
1165 : /**
1166 : * @brief Function to finalize instance (gst element vmethod)
1167 : */
1168 : static void
1169 240 : gst_tensor_transform_finalize (GObject * object)
1170 : {
1171 : GstTensorTransform *filter;
1172 :
1173 240 : filter = GST_TENSOR_TRANSFORM (object);
1174 :
1175 240 : if (filter->option) {
1176 201 : g_free (filter->option);
1177 201 : filter->option = NULL;
1178 : }
1179 :
1180 240 : if (filter->operators) {
1181 61 : g_slist_free_full (filter->operators, g_free);
1182 61 : filter->operators = NULL;
1183 : }
1184 :
1185 240 : if (filter->apply) {
1186 3 : g_list_free (filter->apply);
1187 3 : filter->apply = NULL;
1188 : }
1189 :
1190 240 : G_OBJECT_CLASS (parent_class)->finalize (object);
1191 240 : }
1192 :
1193 : /**
1194 : * @brief subrouting for tensor-transform, "dimchg" case.
1195 : * @param[in/out] filter "this" pointer
1196 : * @param[in] in_info input tensor info
1197 : * @param[in] out_info output tensor info
1198 : * @param[in] inptr input tensor
1199 : * @param[out] outptr output tensor
1200 : * @return Gst flow status
1201 : */
1202 : static GstFlowReturn
1203 52 : gst_tensor_transform_dimchg (GstTensorTransform * filter,
1204 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1205 : const uint8_t * inptr, uint8_t * outptr)
1206 : {
1207 52 : uint32_t *fromDim = in_info->dimension;
1208 52 : uint32_t *toDim = out_info->dimension;
1209 52 : unsigned int from = filter->data_dimchg.from;
1210 52 : unsigned int to = filter->data_dimchg.to;
1211 : unsigned int i, j, k;
1212 52 : unsigned int loopLimit = 1;
1213 : gsize loopBlockSize, copyblocksize, copyblocklimit;
1214 :
1215 52 : if (from == to) {
1216 : /** Useless memcpy. Do not call this or @todo do "IP" operation */
1217 0 : nns_memcpy (outptr, inptr, gst_tensor_info_get_size (in_info));
1218 0 : GST_WARNING_OBJECT (filter,
1219 : "Calling tensor_transform with high memcpy overhead WITHOUT any effects! Check your stream whether you really need tensor_transform.\n");
1220 0 : return GST_FLOW_OK;
1221 : }
1222 :
1223 52 : g_assert (from < NNS_TENSOR_RANK_LIMIT);
1224 52 : g_assert (to < NNS_TENSOR_RANK_LIMIT);
1225 52 : g_assert (fromDim[from] == toDim[to]);
1226 :
1227 52 : loopBlockSize = copyblocksize = gst_tensor_get_element_size (in_info->type);
1228 52 : copyblocklimit = 1;
1229 :
1230 52 : if (from < to) {
1231 : /**
1232 : * Smaller-loop-ed a to larger-loop-ed b
1233 : * E.g., [N][H][W][c] (c:W:H:N) --> [N][c][H][W] (W:H:c:N)
1234 : *
1235 : * @todo CRITICAL-TODO: Optimize the performance!
1236 : */
1237 726 : for (i = NNS_TENSOR_RANK_LIMIT - 1; i > to; i--) {
1238 674 : if (toDim[i] == 0)
1239 620 : continue;
1240 54 : loopLimit *= toDim[i];
1241 : }
1242 :
1243 158 : for (i = 0; i < to; i++) {
1244 106 : if (toDim[i] == 0)
1245 0 : break;
1246 106 : loopBlockSize *= toDim[i];
1247 : }
1248 :
1249 52 : for (i = 0; i < from; i++) {
1250 0 : if (fromDim[i] == 0)
1251 0 : break;
1252 0 : copyblocksize *= fromDim[i];
1253 : }
1254 158 : for (i = 0; i < to; i++) {
1255 106 : if (toDim[i] == 0)
1256 0 : break;
1257 106 : copyblocklimit *= toDim[i];
1258 : }
1259 :
1260 228 : for (i = 0; i < loopLimit; i++) {
1261 : /* [i1][i2][...][iN][b][...] i = i1 x i2 x ... x iN */
1262 176 : uint8_t *destptr = outptr + loopBlockSize * toDim[to] * i;
1263 176 : const uint8_t *srcptr = inptr + loopBlockSize * toDim[to] * i;
1264 :
1265 744 : for (j = 0; j < toDim[to]; j++) {
1266 568 : uint8_t *j_destptr = destptr + loopBlockSize * j;
1267 121208 : for (k = 0; k < copyblocklimit; k++) {
1268 120640 : nns_memcpy (j_destptr + copyblocksize * k,
1269 : srcptr + k * copyblocksize * toDim[to] + j * copyblocksize,
1270 : copyblocksize);
1271 : }
1272 : }
1273 : }
1274 : } else {
1275 : /**
1276 : * Larger-loop-ed a to smaller-loop-ed b
1277 : * E.g., [N][c][H][W] (W:H:c:N) --> [N][H][W][c] (c:W:H:N)
1278 : * @todo NYI
1279 : */
1280 0 : ml_loge
1281 : ("tensor-transform/dimchg operation is not permitted if from >= to.\n");
1282 0 : return GST_FLOW_ERROR;
1283 : }
1284 :
1285 52 : return GST_FLOW_OK;
1286 : }
1287 :
1288 : /**
1289 : * @brief subrouting for tensor-transform, "typecast" case.
1290 : * @param[in/out] filter "this" pointer
1291 : * @param[in] in_info input tensor info
1292 : * @param[in] out_info output tensor info
1293 : * @param[in] inptr input tensor
1294 : * @param[out] outptr output tensor
1295 : * @return Gst flow status
1296 : */
1297 : static GstFlowReturn
1298 588 : gst_tensor_transform_typecast (GstTensorTransform * filter,
1299 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1300 : const uint8_t * inptr, uint8_t * outptr)
1301 : {
1302 : gulong i, num;
1303 : gsize in_element_size, out_element_size;
1304 :
1305 588 : num = gst_tensor_get_element_count (in_info->dimension);
1306 :
1307 : #ifdef HAVE_ORC
1308 588 : if (filter->acceleration) {
1309 466 : orc_typecast (inptr, outptr, num, in_info->type, out_info->type);
1310 466 : return GST_FLOW_OK;
1311 : }
1312 : #endif
1313 :
1314 122 : in_element_size = gst_tensor_get_element_size (in_info->type);
1315 122 : out_element_size = gst_tensor_get_element_size (out_info->type);
1316 :
1317 61772 : for (i = 0; i < num; ++i) {
1318 61650 : gst_tensor_data_raw_typecast (
1319 61650 : (gpointer) (inptr + in_element_size * i), in_info->type,
1320 61650 : (gpointer) (outptr + out_element_size * i), out_info->type);
1321 : }
1322 :
1323 122 : return GST_FLOW_OK;
1324 : }
1325 :
1326 : /**
1327 : * @brief subrouting for tensor-transform, "arithmetic" case.
1328 : * @param[in/out] filter "this" pointer
1329 : * @param[in] in_info input tensor info
1330 : * @param[in] out_info output tensor info
1331 : * @param[in] inptr input tensor
1332 : * @param[out] outptr output tensor
1333 : * @return Gst flow status
1334 : */
1335 : static GstFlowReturn
1336 21397 : gst_tensor_transform_arithmetic (GstTensorTransform * filter,
1337 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1338 : const uint8_t * inptr, uint8_t * outptr)
1339 : {
1340 : gulong i, num, j, ch;
1341 : gsize in_element_size, out_element_size;
1342 :
1343 : GSList *walk;
1344 : tensor_transform_operator_s *op_s;
1345 : tensor_data_s value;
1346 :
1347 21397 : num = gst_tensor_get_element_count (in_info->dimension);
1348 :
1349 : #ifdef HAVE_ORC
1350 21397 : if (filter->acceleration) {
1351 21297 : walk = filter->operators;
1352 : /**
1353 : * Typecast should be called at the first.
1354 : * Do the typecast. If in/out type is same, this will copy the input array to output.
1355 : */
1356 21297 : orc_typecast (inptr, outptr, num, in_info->type, out_info->type);
1357 :
1358 21297 : if (!filter->data_arithmetic.per_channel_arith) {
1359 65093 : while (walk) {
1360 43797 : op_s = (tensor_transform_operator_s *) walk->data;
1361 :
1362 43797 : if (op_s->op != GTT_OP_TYPECAST) {
1363 22579 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1364 23607 : orc_operator (outptr, num, &op_s->value, op_s->op);
1365 : }
1366 :
1367 43797 : walk = g_slist_next (walk);
1368 : }
1369 : } else {
1370 1 : gsize typesize = 0;
1371 1 : guint ch_dim = filter->data_arithmetic.ch_dim;
1372 1 : gsize ch_offset, ch_size = 1;
1373 1 : uint8_t *tmp_outptr = NULL;
1374 :
1375 1 : for (i = 0; i < ch_dim; ++i) {
1376 0 : ch_size *= in_info->dimension[i];
1377 : }
1378 1 : ch_offset = ch_size * in_info->dimension[ch_dim];
1379 1 : orc_typesize (typesize, out_info->type);
1380 :
1381 2 : while (walk) {
1382 1 : op_s = (tensor_transform_operator_s *) walk->data;
1383 1 : if (op_s->op == GTT_OP_TYPECAST) {
1384 0 : walk = g_slist_next (walk);
1385 0 : continue;
1386 : }
1387 :
1388 1 : if (op_s->applying_ch == -1) {
1389 0 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1390 0 : orc_operator (outptr, num, &op_s->value, op_s->op);
1391 : } else {
1392 33 : for (i = 0; i < num / ch_offset; ++i) {
1393 32 : tmp_outptr =
1394 32 : outptr + (ch_size * op_s->applying_ch +
1395 32 : ch_offset * i) * typesize;
1396 32 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1397 32 : orc_operator (tmp_outptr, ch_size, &op_s->value, op_s->op);
1398 : }
1399 : }
1400 1 : walk = g_slist_next (walk);
1401 : }
1402 : }
1403 21397 : return GST_FLOW_OK;
1404 : }
1405 : #endif
1406 :
1407 100 : in_element_size = gst_tensor_get_element_size (in_info->type);
1408 100 : out_element_size = gst_tensor_get_element_size (out_info->type);
1409 :
1410 : /* per-channel */
1411 100 : if (filter->data_arithmetic.per_channel_arith) {
1412 3 : guint ch_dim = filter->data_arithmetic.ch_dim;
1413 3 : gsize ch_offset, ch_size = 1;
1414 3 : for (i = 0; i < ch_dim; ++i) {
1415 0 : ch_size *= in_info->dimension[i];
1416 : }
1417 3 : ch_offset = ch_size * in_info->dimension[ch_dim];
1418 :
1419 : /** In case of 3:4:4:1,
1420 : * ch_dim:0 -> #ch: 3, ch_size: 1, ch_offset: 3
1421 : * ch_dim:1 -> #ch: 4, ch_size: 3, ch_offset: 12
1422 : * ch_dim:2 -> #ch: 4, ch_size: 12, ch_offset: 48
1423 : * ch_dim:3 -> #ch: 1, ch_size: 48, ch_offset: 48 * 4
1424 : */
1425 :
1426 6 : for (i = 0; i < num / ch_offset; ++i) {
1427 18 : for (ch = 0; ch < in_info->dimension[ch_dim]; ++ch) {
1428 30 : for (j = 0; j < ch_size; ++j) {
1429 15 : gulong data_idx = (i * ch_offset) + (ch * ch_size) + j;
1430 15 : gst_tensor_data_set (&value, in_info->type,
1431 15 : (gpointer) (inptr + in_element_size * data_idx));
1432 :
1433 15 : walk = filter->operators;
1434 180 : while (walk) {
1435 165 : op_s = (tensor_transform_operator_s *) walk->data;
1436 165 : switch (op_s->op) {
1437 15 : case GTT_OP_TYPECAST:
1438 15 : gst_tensor_data_typecast (&value, op_s->value.type);
1439 15 : break;
1440 150 : case GTT_OP_ADD:
1441 : case GTT_OP_MUL:
1442 : case GTT_OP_DIV:
1443 : {
1444 150 : gst_tensor_data_typecast (&op_s->value, value.type);
1445 :
1446 150 : if (op_s->applying_ch == (int) ch || op_s->applying_ch == -1) {
1447 30 : gst_tensor_transform_do_operator (filter, &value,
1448 30 : &op_s->value, op_s->op);
1449 : }
1450 150 : break;
1451 : }
1452 0 : default:
1453 0 : g_assert (0);
1454 : return GST_FLOW_ERROR;
1455 : }
1456 :
1457 165 : walk = g_slist_next (walk);
1458 : }
1459 :
1460 : /* set output value */
1461 15 : g_assert (out_info->type == value.type);
1462 15 : gst_tensor_data_get (&value, outptr + out_element_size * data_idx);
1463 : }
1464 : }
1465 : }
1466 :
1467 3 : return GST_FLOW_OK;
1468 : }
1469 :
1470 61622 : for (i = 0; i < num; ++i) {
1471 : /* init value with input tensor type */
1472 61525 : gst_tensor_data_set (&value, in_info->type,
1473 61525 : (gpointer) (inptr + in_element_size * i));
1474 :
1475 61525 : walk = filter->operators;
1476 184595 : while (walk) {
1477 123070 : op_s = (tensor_transform_operator_s *) walk->data;
1478 :
1479 : /**
1480 : * @todo add more options
1481 : */
1482 123070 : switch (op_s->op) {
1483 61485 : case GTT_OP_TYPECAST:
1484 61485 : gst_tensor_data_typecast (&value, op_s->value.type);
1485 61485 : break;
1486 61585 : case GTT_OP_ADD:
1487 : case GTT_OP_MUL:
1488 : case GTT_OP_DIV:
1489 61585 : gst_tensor_data_typecast (&op_s->value, value.type);
1490 61585 : gst_tensor_transform_do_operator (filter, &value, &op_s->value,
1491 : op_s->op);
1492 61585 : break;
1493 0 : default:
1494 0 : g_assert (0);
1495 : return GST_FLOW_ERROR;
1496 : }
1497 :
1498 123070 : walk = g_slist_next (walk);
1499 : }
1500 :
1501 : /* set output value */
1502 61525 : g_assert (out_info->type == value.type);
1503 61525 : gst_tensor_data_get (&value, outptr + out_element_size * i);
1504 : }
1505 :
1506 97 : return GST_FLOW_OK;
1507 : }
1508 :
1509 : /**
1510 : * Macro to run loop for various data types with transpose
1511 : */
1512 : #define transposeloop(cl,ck,cj,ci,sl,sk,sj,si,typesize) do { \
1513 : size_t i, j, k, l; \
1514 : int inidx = 0, outidx=0; \
1515 : for(cl=0;cl<sl;cl++) \
1516 : for(ci=0;ci<si;ci++) \
1517 : for(cj=0;cj<sj;cj++) \
1518 : for(ck=0;ck<sk;ck++){ \
1519 : const uint8_t *_in; \
1520 : uint8_t *_out; \
1521 : outidx = si*sj*sk*cl + sj*sk*ci + sk*cj + ck; \
1522 : inidx = SK*SJ*SI*l + SJ*SI*k + SI*j + i; \
1523 : _in = inptr + inidx * typesize; \
1524 : _out = outptr + outidx * typesize; \
1525 : nns_memcpy(_out, _in, typesize); \
1526 : } \
1527 : } while(0);
1528 :
1529 : /**
1530 : * @brief subrouting for tensor-transform, "transpose" case.
1531 : * @param[in/out] filter "this" pointer
1532 : * @param[in] in_info input tensor info
1533 : * @param[in] out_info output tensor info
1534 : * @param[in] inptr input tensor
1535 : * @param[out] outptr output tensor
1536 : * @return Gst flow status
1537 : */
1538 : static GstFlowReturn
1539 58 : gst_tensor_transform_transpose (GstTensorTransform * filter,
1540 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1541 : const uint8_t * inptr, uint8_t * outptr)
1542 : {
1543 : int i, from, to;
1544 58 : gboolean checkdim = FALSE;
1545 58 : uint32_t *fromDim = in_info->dimension;
1546 58 : gsize type_size = gst_tensor_get_element_size (in_info->type);
1547 : gsize indexI, indexJ, SL, SI, SJ, SK;
1548 : UNUSED (out_info);
1549 :
1550 58 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
1551 58 : from = i;
1552 58 : to = filter->data_transpose.trans_order[i];
1553 58 : if (from != to) {
1554 58 : checkdim = TRUE;
1555 58 : break;
1556 : }
1557 : }
1558 :
1559 58 : if (!checkdim) {
1560 0 : nns_memcpy (outptr, inptr, gst_tensor_info_get_size (in_info));
1561 0 : GST_WARNING_OBJECT (filter,
1562 : "Calling tensor_transform with high memcpy overhead WITHOUT any effects!");
1563 0 : return GST_FLOW_OK;
1564 : }
1565 :
1566 58 : indexI = filter->data_transpose.trans_order[0];
1567 58 : indexJ = filter->data_transpose.trans_order[1];
1568 58 : SL = fromDim[3] > 0 ? fromDim[3] : 1;
1569 58 : SI = fromDim[0] > 0 ? fromDim[0] : 1;
1570 58 : SJ = fromDim[1] > 0 ? fromDim[1] : 1;
1571 58 : SK = fromDim[2] > 0 ? fromDim[2] : 1;
1572 :
1573 58 : switch (indexI) {
1574 0 : case 0:
1575 0 : if (indexJ == 1) {
1576 0 : transposeloop (l, i, j, k, SL, SI, SJ, SK, type_size);
1577 : } else {
1578 0 : transposeloop (l, i, k, j, SL, SI, SK, SJ, type_size);
1579 : }
1580 0 : break;
1581 54 : case 1:
1582 54 : if (indexJ == 0) {
1583 242420 : transposeloop (l, j, i, k, SL, SJ, SI, SK, type_size);
1584 : } else {
1585 1231478 : transposeloop (l, j, k, i, SL, SJ, SK, SI, type_size);
1586 : }
1587 54 : break;
1588 4 : case 2:
1589 4 : if (indexJ == 0) {
1590 140258 : transposeloop (l, k, i, j, SL, SK, SI, SJ, type_size);
1591 : } else {
1592 0 : transposeloop (l, k, j, i, SL, SK, SJ, SI, type_size);
1593 : }
1594 4 : break;
1595 : }
1596 :
1597 58 : return GST_FLOW_OK;
1598 : }
1599 :
1600 : /**
1601 : * @brief subrouting for tensor-transform, "stand" case.
1602 : * : pixel = abs((pixel - average(tensor))/(std(tensor) + val))
1603 : * @param[in/out] filter "this" pointer
1604 : * @param[in] in_info input tensor info
1605 : * @param[in] out_info output tensor info
1606 : * @param[in] inptr input tensor
1607 : * @param[out] outptr output tensor
1608 : * @return Gst flow status
1609 : */
1610 : static GstFlowReturn
1611 16 : gst_tensor_transform_stand (GstTensorTransform * filter,
1612 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1613 : const uint8_t * inptr, uint8_t * outptr)
1614 : {
1615 16 : GstFlowReturn ret = GST_FLOW_OK;
1616 : gsize in_element_size, out_element_size, data_size, ch_size;
1617 : gulong i, num, data_idx, ch;
1618 : gdouble tmp, *average, *std;
1619 :
1620 16 : in_element_size = gst_tensor_get_element_size (in_info->type);
1621 16 : out_element_size = gst_tensor_get_element_size (out_info->type);
1622 16 : num = gst_tensor_get_element_count (in_info->dimension);
1623 :
1624 16 : data_size = gst_tensor_info_get_size (in_info);
1625 16 : ch_size = in_info->dimension[0];
1626 :
1627 : /* calc average and std */
1628 16 : average = std = NULL;
1629 16 : if (filter->data_stand.per_channel) {
1630 8 : gst_tensor_data_raw_average_per_channel ((gpointer) inptr, data_size,
1631 8 : in_info->type, in_info->dimension, &average);
1632 : /* calculate std only for default mode */
1633 8 : if (filter->data_stand.mode == STAND_DEFAULT)
1634 4 : gst_tensor_data_raw_std_per_channel ((gpointer) inptr, data_size,
1635 4 : in_info->type, in_info->dimension, average, &std);
1636 : } else {
1637 8 : gst_tensor_data_raw_average ((gpointer) inptr, data_size,
1638 : in_info->type, &average);
1639 : /* calculate std only for default mode */
1640 8 : if (filter->data_stand.mode == STAND_DEFAULT)
1641 4 : gst_tensor_data_raw_std ((gpointer) inptr, data_size, in_info->type,
1642 : average, &std);
1643 : }
1644 :
1645 16 : switch (filter->data_stand.mode) {
1646 8 : case STAND_DEFAULT:
1647 : {
1648 8 : if (!filter->data_stand.per_channel) {
1649 20004 : for (i = 0; i < num; i++) {
1650 20000 : data_idx = in_element_size * i;
1651 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1652 : in_info->type, &tmp, _NNS_FLOAT64);
1653 :
1654 20000 : tmp = fabs ((tmp - *average) / *std);
1655 :
1656 20000 : data_idx = out_element_size * i;
1657 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1658 20000 : (gpointer) (outptr + data_idx), out_info->type);
1659 : }
1660 : } else {
1661 204 : for (ch = 0; ch < ch_size; ++ch) {
1662 20200 : for (i = 0; i < num / ch_size; i++) {
1663 20000 : data_idx = in_element_size * ((i * ch_size) + ch);
1664 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1665 : in_info->type, &tmp, _NNS_FLOAT64);
1666 :
1667 20000 : tmp = fabs ((tmp - average[ch]) / std[ch]);
1668 :
1669 20000 : data_idx = out_element_size * ((i * ch_size) + ch);
1670 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1671 20000 : (gpointer) (outptr + data_idx), out_info->type);
1672 : }
1673 : }
1674 : }
1675 8 : break;
1676 : }
1677 8 : case STAND_DC_AVERAGE:
1678 : {
1679 8 : if (!filter->data_stand.per_channel) {
1680 20004 : for (i = 0; i < num; i++) {
1681 20000 : data_idx = in_element_size * i;
1682 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1683 : in_info->type, &tmp, _NNS_FLOAT64);
1684 :
1685 20000 : tmp -= *average;
1686 :
1687 20000 : data_idx = out_element_size * i;
1688 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1689 20000 : (gpointer) (outptr + data_idx), out_info->type);
1690 : }
1691 : } else {
1692 204 : for (ch = 0; ch < ch_size; ++ch) {
1693 20200 : for (i = 0; i < num / ch_size; i++) {
1694 20000 : data_idx = in_element_size * ((i * ch_size) + ch);
1695 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1696 : in_info->type, &tmp, _NNS_FLOAT64);
1697 :
1698 20000 : tmp -= average[ch];
1699 :
1700 20000 : data_idx = out_element_size * ((i * ch_size) + ch);
1701 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1702 20000 : (gpointer) (outptr + data_idx), out_info->type);
1703 : }
1704 : }
1705 : }
1706 8 : break;
1707 : }
1708 0 : default:
1709 0 : GST_ERROR_OBJECT (filter, "Cannot identify mode\n");
1710 0 : ret = GST_FLOW_ERROR;
1711 : }
1712 :
1713 16 : g_free (average);
1714 16 : g_free (std);
1715 :
1716 16 : return ret;
1717 : }
1718 :
1719 : /**
1720 : * @brief subrouting for tensor-transform, "clamp" case.
1721 : * : pixel = if (pixel > max) ? max :
1722 : * if (pixel < min) ? min : pixel
1723 : * @param[in/out] filter "this" pointer
1724 : * @param[in] in_info input tensor info
1725 : * @param[in] out_info output tensor info
1726 : * @param[in] inptr input tensor
1727 : * @param[out] outptr output tensor
1728 : * @return Gst flow status
1729 : */
1730 : static GstFlowReturn
1731 16 : gst_tensor_transform_clamp (GstTensorTransform * filter,
1732 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1733 : const uint8_t * inptr, uint8_t * outptr)
1734 : {
1735 : gsize in_element_size, out_element_size;
1736 : gulong i, num, data_idx;
1737 : gdouble tmp;
1738 :
1739 16 : in_element_size = gst_tensor_get_element_size (in_info->type);
1740 16 : out_element_size = gst_tensor_get_element_size (out_info->type);
1741 16 : num = gst_tensor_get_element_count (in_info->dimension);
1742 :
1743 606016 : for (i = 0; i < num; ++i) {
1744 606000 : data_idx = in_element_size * i;
1745 606000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), in_info->type,
1746 : &tmp, _NNS_FLOAT64);
1747 :
1748 606000 : tmp = CLAMP (tmp, filter->data_clamp.min, filter->data_clamp.max);
1749 :
1750 606000 : data_idx = out_element_size * i;
1751 606000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, outptr + data_idx,
1752 : out_info->type);
1753 : }
1754 :
1755 16 : return GST_FLOW_OK;
1756 : }
1757 :
1758 : /**
1759 : * @brief subrouting for tensor-transform, "padding" case.
1760 : * @param[in/out] filter "this" pointer
1761 : * @param[in] in_info input tensor info
1762 : * @param[in] out_info output tensor info
1763 : * @param[in] inptr input tensor
1764 : * @param[out] outptr output tensor
1765 : * @return Gst flow status
1766 : */
1767 : static GstFlowReturn
1768 6 : gst_tensor_transform_padding (GstTensorTransform * filter,
1769 : GstTensorInfo * in_info, GstTensorInfo * out_info, const uint8_t * inptr,
1770 : uint8_t * outptr)
1771 : {
1772 : gsize element_size, in_loop_size, out_loop_size, copy_block_size;
1773 6 : guint i, j, k, left, top, front, loop_limit = 1;
1774 6 : element_size = gst_tensor_get_element_size (in_info->type);
1775 :
1776 6 : in_loop_size = (gsize) in_info->dimension[2] * in_info->dimension[1]
1777 6 : * in_info->dimension[0] * element_size;
1778 6 : out_loop_size =(gsize) out_info->dimension[2] * out_info->dimension[1]
1779 6 : * out_info->dimension[0] * element_size;
1780 6 : copy_block_size = in_info->dimension[0] * element_size;
1781 :
1782 9 : for (i = NNS_TENSOR_PADDING_RANK_LIMIT; i < NNS_TENSOR_RANK_LIMIT; i++) {
1783 9 : if (in_info->dimension[i] == 0)
1784 6 : break;
1785 3 : loop_limit *= in_info->dimension[i];
1786 : }
1787 :
1788 6 : left = filter->data_padding.pad[PADDING_LEFT];
1789 6 : top = filter->data_padding.pad[PADDING_TOP];
1790 6 : front = filter->data_padding.pad[PADDING_FRONT];
1791 :
1792 : /** @todo Add constant option instead of using zero padding value */
1793 6 : memset (outptr, 0, out_loop_size * loop_limit);
1794 :
1795 20 : for (i = 0; i < loop_limit; i++)
1796 56 : for (j = 0; j < in_info->dimension[2]; j++)
1797 2142 : for (k = 0; k < in_info->dimension[1]; k++) {
1798 2100 : guint in_idx = j * in_info->dimension[1] * in_info->dimension[0]
1799 2100 : + k * in_info->dimension[0];
1800 2100 : guint out_idx = j * out_info->dimension[1] * out_info->dimension[0]
1801 2100 : + k * out_info->dimension[0];
1802 :
1803 2100 : out_idx += left + top * out_info->dimension[0]
1804 2100 : + front * out_info->dimension[1] * out_info->dimension[0];
1805 :
1806 2100 : memcpy (outptr + out_idx * element_size + out_loop_size * i,
1807 2100 : inptr + in_idx * element_size + in_loop_size * i, copy_block_size);
1808 : }
1809 :
1810 6 : return GST_FLOW_OK;
1811 : }
1812 :
1813 : /**
1814 : * @brief non-ip transform. required vmethod for BaseTransform class.
1815 : * @param[in/out] trans "super" pointer
1816 : * @param[in] inbuf The input gst buffer
1817 : * @param[out] outbuf The output gst buffer
1818 : * @return Gst Flow Status
1819 : */
1820 : static GstFlowReturn
1821 22066 : gst_tensor_transform_transform (GstBaseTransform * trans,
1822 : GstBuffer * inbuf, GstBuffer * outbuf)
1823 : {
1824 : GstTensorTransform *filter;
1825 : GstTensorInfo *in_info, *out_info;
1826 22066 : GstFlowReturn res = GST_FLOW_ERROR;
1827 22066 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT] = { 0, };
1828 22066 : GstMemory *out_mem[NNS_TENSOR_SIZE_LIMIT] = { 0, };
1829 : GstMapInfo in_map[NNS_TENSOR_SIZE_LIMIT];
1830 : GstMapInfo out_map[NNS_TENSOR_SIZE_LIMIT];
1831 : uint8_t *inptr, *outptr;
1832 : guint i, num_tensors, num_mems;
1833 : gsize buf_size, hsize;
1834 : GstTensorMetaInfo meta;
1835 : GstTensorInfo in_flex_info, out_flex_info;
1836 : gboolean in_flexible, out_flexible;
1837 :
1838 22066 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
1839 :
1840 44128 : g_return_val_if_fail (filter->loaded, GST_FLOW_ERROR);
1841 22062 : inbuf = gst_tensor_buffer_from_config (inbuf, &filter->in_config);
1842 :
1843 22062 : in_flexible =
1844 22062 : gst_tensor_pad_caps_is_flexible (GST_BASE_TRANSFORM_SINK_PAD (trans));
1845 22062 : out_flexible =
1846 22062 : gst_tensor_pad_caps_is_flexible (GST_BASE_TRANSFORM_SRC_PAD (trans));
1847 :
1848 22062 : num_mems = gst_tensor_buffer_get_count (inbuf);
1849 22062 : if (in_flexible) {
1850 11 : num_tensors = num_mems;
1851 11 : g_return_val_if_fail (out_flexible, GST_FLOW_ERROR);
1852 : } else {
1853 22051 : num_tensors = filter->in_config.info.num_tensors;
1854 22051 : g_return_val_if_fail (num_mems == num_tensors, GST_FLOW_ERROR);
1855 : }
1856 :
1857 44210 : for (i = 0; i < num_tensors; i++) {
1858 22148 : in_info = gst_tensors_info_get_nth_info (&filter->in_config.info, i);
1859 22148 : out_info = gst_tensors_info_get_nth_info (&filter->out_config.info, i);
1860 :
1861 22148 : if (filter->apply && !g_list_find (filter->apply, GINT_TO_POINTER (i))) {
1862 15 : GstMemory *mem = gst_tensor_buffer_get_nth_memory (inbuf, i);
1863 :
1864 15 : if (!in_flexible && out_flexible) {
1865 0 : GstMemory *old = mem;
1866 :
1867 : /* append meta */
1868 0 : gst_tensor_info_convert_to_meta (out_info, &meta);
1869 0 : mem = gst_tensor_meta_info_append_header (&meta, old);
1870 0 : gst_memory_unref (old);
1871 : }
1872 :
1873 15 : gst_tensor_buffer_append_memory (outbuf, mem, out_info);
1874 15 : continue;
1875 : }
1876 :
1877 : /* parse input buffer */
1878 22133 : in_mem[i] = gst_tensor_buffer_get_nth_memory (inbuf, i);
1879 22133 : if (!gst_memory_map (in_mem[i], &in_map[i], GST_MAP_READ)) {
1880 0 : ml_loge ("Cannot map input buffer to gst-buf at tensor-transform.\n");
1881 0 : res = GST_FLOW_ERROR;
1882 0 : goto done;
1883 : }
1884 22133 : inptr = in_map[i].data;
1885 :
1886 22133 : if (in_flexible) {
1887 11 : in_info = &in_flex_info;
1888 11 : out_info = &out_flex_info;
1889 :
1890 11 : gst_tensor_meta_info_parse_header (&meta, inptr);
1891 : /** @todo max rank supported in tensor-transform is 4 */
1892 11 : if (!gst_tensor_meta_info_convert (&meta, in_info)) {
1893 0 : res = GST_FLOW_ERROR;
1894 0 : goto done;
1895 : }
1896 :
1897 11 : gst_tensor_transform_convert_dimension (filter, GST_PAD_SINK,
1898 : i, in_info, out_info);
1899 :
1900 11 : hsize = gst_tensor_meta_info_get_header_size (&meta);
1901 11 : inptr += hsize;
1902 : }
1903 :
1904 : /* prepare output buffer */
1905 22133 : buf_size = gst_tensor_info_get_size (out_info);
1906 22133 : if (out_flexible) {
1907 20 : gst_tensor_info_convert_to_meta (out_info, &meta);
1908 20 : hsize = gst_tensor_meta_info_get_header_size (&meta);
1909 20 : buf_size += hsize;
1910 : }
1911 :
1912 22133 : out_mem[i] = gst_allocator_alloc (NULL, buf_size, NULL);
1913 22133 : gst_tensor_buffer_append_memory (outbuf, out_mem[i], out_info);
1914 :
1915 22133 : if (!gst_memory_map (out_mem[i], &out_map[i], GST_MAP_WRITE)) {
1916 0 : ml_loge ("Cannot map output buffer to gst-buf at tensor-transform.\n");
1917 0 : res = GST_FLOW_ERROR;
1918 0 : goto done;
1919 : }
1920 22133 : outptr = out_map[i].data;
1921 :
1922 22133 : if (out_flexible) {
1923 20 : gst_tensor_meta_info_update_header (&meta, outptr);
1924 20 : outptr += hsize;
1925 : }
1926 :
1927 22133 : switch (filter->mode) {
1928 52 : case GTT_DIMCHG:
1929 52 : res = gst_tensor_transform_dimchg (filter, in_info, out_info,
1930 : inptr, outptr);
1931 52 : break;
1932 588 : case GTT_TYPECAST:
1933 588 : res = gst_tensor_transform_typecast (filter, in_info, out_info,
1934 : inptr, outptr);
1935 588 : break;
1936 21397 : case GTT_ARITHMETIC:
1937 21397 : res = gst_tensor_transform_arithmetic (filter, in_info, out_info,
1938 : inptr, outptr);
1939 21397 : break;
1940 58 : case GTT_TRANSPOSE:
1941 58 : res = gst_tensor_transform_transpose (filter, in_info, out_info,
1942 : inptr, outptr);
1943 58 : break;
1944 16 : case GTT_STAND:
1945 16 : res = gst_tensor_transform_stand (filter, in_info, out_info,
1946 : inptr, outptr);
1947 16 : break;
1948 16 : case GTT_CLAMP:
1949 16 : res = gst_tensor_transform_clamp (filter, in_info, out_info,
1950 : inptr, outptr);
1951 16 : break;
1952 6 : case GTT_PADDING:
1953 6 : res = gst_tensor_transform_padding (filter, in_info, out_info,
1954 : inptr, outptr);
1955 6 : break;
1956 0 : default:
1957 0 : ml_loge ("Not supported tensor transform mode");
1958 0 : res = GST_FLOW_NOT_SUPPORTED;
1959 0 : goto done;
1960 : }
1961 : }
1962 :
1963 22062 : done:
1964 44210 : for (i = 0; i < num_tensors; i++) {
1965 22148 : if (in_mem[i]) {
1966 22133 : gst_memory_unmap (in_mem[i], &in_map[i]);
1967 22133 : gst_memory_unref (in_mem[i]);
1968 : }
1969 22148 : if (out_mem[i])
1970 22133 : gst_memory_unmap (out_mem[i], &out_map[i]);
1971 : }
1972 :
1973 22062 : return res;
1974 : }
1975 :
1976 : /**
1977 : * @brief Dimension conversion calculation
1978 : * @param[in] filter "this" pointer
1979 : * @param[in] direction GST_PAD_SINK if input->output conv
1980 : * @param[in] idx index of the input tensors
1981 : * @param[in] in_info tensor info structure of source tensor (input if direction is SINK)
1982 : * @param[out] out_info tensor info structure of destination tensor (output if direction is SINK)
1983 : * @return TRUE if success
1984 : */
1985 : static gboolean
1986 4367 : gst_tensor_transform_convert_dimension (GstTensorTransform * filter,
1987 : GstPadDirection direction, guint idx, const GstTensorInfo * in_info,
1988 : GstTensorInfo * out_info)
1989 : {
1990 : guint i;
1991 :
1992 : /* copy input info first, then update output info */
1993 4367 : gst_tensor_info_copy (out_info, in_info);
1994 :
1995 4367 : if (filter->apply && !g_list_find (filter->apply, GINT_TO_POINTER (idx)))
1996 10 : return TRUE;
1997 :
1998 4357 : switch (filter->mode) {
1999 70 : case GTT_DIMCHG:
2000 : {
2001 70 : unsigned int from = filter->data_dimchg.from;
2002 70 : unsigned int to = filter->data_dimchg.to;
2003 :
2004 70 : if (direction == GST_PAD_SINK) {
2005 850 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) {
2006 800 : if ((i < from && i < to) || (i > from && i > to) || from == to) {
2007 634 : out_info->dimension[i] = in_info->dimension[i];
2008 166 : } else if (i == to) {
2009 50 : out_info->dimension[i] = in_info->dimension[from];
2010 116 : } else if (from > to) {
2011 0 : g_assert (i > 0 && i > to);
2012 0 : out_info->dimension[i] = in_info->dimension[i - 1];
2013 : } else {
2014 116 : g_assert (i < to && i < (NNS_TENSOR_RANK_LIMIT - 1));
2015 116 : out_info->dimension[i] = in_info->dimension[i + 1];
2016 : }
2017 : }
2018 : } else {
2019 340 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) {
2020 320 : if ((i < from && i < to) || (i > from && i > to) || from == to) {
2021 252 : out_info->dimension[i] = in_info->dimension[i];
2022 68 : } else if (i == from) {
2023 20 : out_info->dimension[i] = in_info->dimension[to];
2024 48 : } else if (from > to) {
2025 0 : g_assert (i < from && i < (NNS_TENSOR_RANK_LIMIT - 1));
2026 0 : out_info->dimension[i] = in_info->dimension[i + 1];
2027 : } else {
2028 48 : g_assert (i > 0 && i > from);
2029 48 : out_info->dimension[i] = in_info->dimension[i - 1];
2030 : }
2031 : }
2032 : }
2033 70 : break;
2034 : }
2035 2364 : case GTT_TYPECAST:
2036 : /** For both directions, dimension does not change */
2037 2364 : if (direction == GST_PAD_SINK) {
2038 : /** src = SINKPAD / dest = SRCPAD */
2039 1364 : out_info->type = filter->data_typecast.to;
2040 : } else {
2041 : /* cannot get the incoming data type on sink pad */
2042 1000 : out_info->type = _NNS_END;
2043 : }
2044 2364 : break;
2045 :
2046 1222 : case GTT_ARITHMETIC:
2047 : /* check arith mode option has typecast operator */
2048 1222 : if (filter->data_arithmetic.out_type != _NNS_END) {
2049 876 : if (direction == GST_PAD_SINK) {
2050 499 : out_info->type = filter->data_arithmetic.out_type;
2051 : } else {
2052 : /* cannot get the incoming data type on sink pad */
2053 377 : out_info->type = _NNS_END;
2054 : }
2055 : }
2056 1222 : break;
2057 :
2058 287 : case GTT_TRANSPOSE:
2059 287 : if (direction == GST_PAD_SINK) {
2060 715 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
2061 572 : out_info->dimension[i] =
2062 572 : in_info->dimension[filter->data_transpose.trans_order[i]];
2063 : }
2064 : } else {
2065 720 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
2066 576 : g_assert (filter->data_transpose.trans_order[i] <
2067 : NNS_TENSOR_RANK_LIMIT);
2068 576 : out_info->dimension[filter->data_transpose.trans_order[i]] =
2069 576 : in_info->dimension[i];
2070 : }
2071 : }
2072 287 : break;
2073 :
2074 40 : case GTT_STAND:
2075 : /** For both directions, dimension does not change */
2076 40 : if (direction == GST_PAD_SINK) {
2077 24 : if (filter->data_stand.out_type != _NNS_END)
2078 12 : out_info->type = filter->data_stand.out_type;
2079 : } else {
2080 : /* cannot get the incoming data type on sink pad */
2081 16 : out_info->type = _NNS_END;
2082 : }
2083 40 : break;
2084 :
2085 176 : case GTT_CLAMP:
2086 : /* same tensors info, do nothing. */
2087 176 : break;
2088 :
2089 44 : case GTT_PADDING:
2090 44 : if (direction == GST_PAD_SINK) {
2091 30 : out_info->dimension[0] +=
2092 30 : filter->data_padding.pad[PADDING_LEFT] +
2093 30 : filter->data_padding.pad[PADDING_RIGHT];
2094 30 : out_info->dimension[1] +=
2095 30 : filter->data_padding.pad[PADDING_TOP] +
2096 30 : filter->data_padding.pad[PADDING_BOTTOM];
2097 30 : out_info->dimension[2] +=
2098 30 : filter->data_padding.pad[PADDING_FRONT] +
2099 30 : filter->data_padding.pad[PADDING_BACK];
2100 : }
2101 44 : break;
2102 154 : default:
2103 154 : return FALSE;
2104 : }
2105 :
2106 4203 : return TRUE;
2107 : }
2108 :
2109 : /**
2110 : * @brief configure srcpad cap from "proposed" cap. (required vmethod for BaseTransform)
2111 : *
2112 : * @param trans ("this" pointer)
2113 : * @param direction (why do we need this?)
2114 : * @param caps sinkpad cap
2115 : * @param filtercap this element's cap (don't know specifically.)
2116 : *
2117 : * @todo Get to know what the heck is @filtercap and use it!
2118 : */
2119 : static GstCaps *
2120 3296 : gst_tensor_transform_transform_caps (GstBaseTransform * trans,
2121 : GstPadDirection direction, GstCaps * caps, GstCaps * filtercap)
2122 : {
2123 : GstTensorTransform *filter;
2124 3296 : GstCaps *result = NULL;
2125 : GstStructure *structure;
2126 : GstPad *pad;
2127 : guint i, j;
2128 :
2129 3296 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2130 :
2131 3296 : silent_debug (filter, "Calling TransformCaps, direction = %d\n", direction);
2132 3296 : silent_debug_caps (filter, caps, "from");
2133 3296 : silent_debug_caps (filter, filtercap, "filter");
2134 :
2135 3296 : if (direction == GST_PAD_SRC) {
2136 1530 : pad = GST_BASE_TRANSFORM_SINK_PAD (trans);
2137 : } else {
2138 1766 : pad = GST_BASE_TRANSFORM_SRC_PAD (trans);
2139 : }
2140 :
2141 3296 : result = gst_caps_new_empty ();
2142 10472 : for (i = 0; i < gst_caps_get_size (caps); i++) {
2143 : GstTensorsConfig in_config, out_config;
2144 : GstTensorInfo *in_info, *out_info;
2145 : GstCaps *out_caps;
2146 :
2147 7176 : structure = gst_caps_get_structure (caps, i);
2148 7176 : gst_tensors_config_from_structure (&in_config, structure);
2149 :
2150 7176 : gst_tensors_config_copy (&out_config, &in_config);
2151 :
2152 7176 : if (out_config.info.format == _NNS_TENSOR_FORMAT_END) {
2153 1512 : out_caps = gst_pad_get_pad_template_caps (pad);
2154 : } else {
2155 5664 : if (gst_tensors_config_is_static (&out_config)) {
2156 8784 : for (j = 0; j < in_config.info.num_tensors; j++) {
2157 4128 : in_info = gst_tensors_info_get_nth_info (&in_config.info, j);
2158 4128 : out_info = gst_tensors_info_get_nth_info (&out_config.info, j);
2159 :
2160 4128 : gst_tensor_transform_convert_dimension (filter, direction,
2161 : j, in_info, out_info);
2162 : }
2163 : }
2164 :
2165 5664 : out_caps = gst_tensor_pad_possible_caps_from_config (pad, &out_config);
2166 : }
2167 :
2168 7176 : if (out_caps) {
2169 7176 : result = gst_caps_merge (result, out_caps);
2170 : }
2171 :
2172 7176 : gst_tensors_config_free (&in_config);
2173 7176 : gst_tensors_config_free (&out_config);
2174 : }
2175 :
2176 3296 : if (filtercap && gst_caps_get_size (filtercap) > 0) {
2177 : GstCaps *intersection;
2178 :
2179 : intersection =
2180 694 : gst_caps_intersect_full (result, filtercap, GST_CAPS_INTERSECT_FIRST);
2181 :
2182 694 : gst_caps_unref (result);
2183 694 : result = intersection;
2184 : }
2185 :
2186 3296 : silent_debug_caps (filter, result, "to");
2187 3296 : return result;
2188 : }
2189 :
2190 : /**
2191 : * @brief fixate caps. required vmethod of BaseTransform
2192 : */
2193 : static GstCaps *
2194 240 : gst_tensor_transform_fixate_caps (GstBaseTransform * trans,
2195 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
2196 : {
2197 : GstTensorTransform *filter;
2198 : GstCaps *result;
2199 :
2200 240 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2201 :
2202 240 : silent_debug (filter, "Calling FixateCaps, direction = %d\n", direction);
2203 240 : silent_debug_caps (filter, caps, "caps");
2204 240 : silent_debug_caps (filter, othercaps, "othercaps");
2205 :
2206 : result =
2207 240 : gst_tensor_transform_transform_caps (trans, direction, caps, othercaps);
2208 240 : gst_caps_unref (othercaps);
2209 :
2210 240 : result = gst_caps_make_writable (result);
2211 240 : result = gst_caps_fixate (result);
2212 :
2213 240 : silent_debug_caps (filter, result, "result");
2214 240 : return result;
2215 : }
2216 :
2217 : /**
2218 : * @brief set caps. required vmethod of BaseTransform
2219 : */
2220 : static gboolean
2221 222 : gst_tensor_transform_set_caps (GstBaseTransform * trans,
2222 : GstCaps * incaps, GstCaps * outcaps)
2223 : {
2224 : GstTensorTransform *filter;
2225 : GstTensorsConfig in_config, out_config;
2226 : GstTensorsConfig config;
2227 : GstTensorInfo *in_info, *out_info;
2228 : gboolean in_flexible, out_flexible;
2229 222 : gboolean allowed = FALSE;
2230 : guint i;
2231 :
2232 222 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2233 :
2234 222 : silent_debug (filter, "Calling SetCaps\n");
2235 222 : silent_debug_caps (filter, incaps, "incaps");
2236 222 : silent_debug_caps (filter, outcaps, "outcaps");
2237 :
2238 222 : if (!gst_tensors_config_from_caps (&in_config, incaps, TRUE)) {
2239 0 : GST_ERROR_OBJECT (filter, "Cannot read cap of incaps\n");
2240 0 : goto error;
2241 : }
2242 :
2243 222 : if (!gst_tensors_config_from_caps (&out_config, outcaps, TRUE)) {
2244 0 : GST_ERROR_OBJECT (filter, "Cannot read cap of outcaps\n");
2245 0 : goto error;
2246 : }
2247 :
2248 222 : in_flexible = gst_tensors_config_is_flexible (&in_config);
2249 222 : out_flexible = gst_tensors_config_is_flexible (&out_config);
2250 :
2251 : /* compare type and dimension */
2252 222 : gst_tensors_config_init (&config);
2253 222 : config.info.format = out_config.info.format;
2254 :
2255 222 : config.rate_n = in_config.rate_n;
2256 222 : config.rate_d = in_config.rate_d;
2257 222 : config.info.num_tensors = in_config.info.num_tensors;
2258 :
2259 222 : if (!in_flexible) {
2260 440 : for (i = 0; i < in_config.info.num_tensors; i++) {
2261 228 : in_info = gst_tensors_info_get_nth_info (&in_config.info, i);
2262 228 : out_info = gst_tensors_info_get_nth_info (&config.info, i);
2263 :
2264 228 : if (!gst_tensor_transform_convert_dimension (filter, GST_PAD_SINK,
2265 : i, in_info, out_info)) {
2266 0 : GST_ERROR_OBJECT (filter,
2267 : "Tensor info is not matched with given properties.");
2268 0 : goto error;
2269 : }
2270 : }
2271 : }
2272 :
2273 222 : if (out_flexible) {
2274 15 : GST_INFO_OBJECT (filter, "Output tensor is flexible.");
2275 :
2276 : /* set output configuration if input is static */
2277 15 : if (!in_flexible)
2278 5 : out_config = config;
2279 207 : } else if (!gst_tensors_config_is_equal (&out_config, &config)) {
2280 0 : GST_ERROR_OBJECT (filter,
2281 : "Tensor info is not matched with given properties.\n");
2282 0 : goto error;
2283 : }
2284 :
2285 : /* set in/out tensor info */
2286 222 : filter->in_config = in_config;
2287 222 : filter->out_config = out_config;
2288 222 : allowed = TRUE;
2289 :
2290 222 : error:
2291 222 : if (!allowed)
2292 0 : GST_ERROR_OBJECT (filter, "Set Caps Failed!\n");
2293 :
2294 222 : return allowed;
2295 : }
2296 :
2297 : /**
2298 : * @brief Tell the framework the required size of buffer based on the info of the other side pad. Note that this is always the same with the input. optional vmethod of BaseTransform
2299 : */
2300 : static gboolean
2301 22066 : gst_tensor_transform_transform_size (GstBaseTransform * trans,
2302 : GstPadDirection direction, GstCaps * caps, gsize size, GstCaps * othercaps,
2303 : gsize * othersize)
2304 : {
2305 : UNUSED (trans);
2306 : UNUSED (direction);
2307 : UNUSED (caps);
2308 : UNUSED (size);
2309 : UNUSED (othercaps);
2310 : /**
2311 : * Consider multi-tensors.
2312 : * Set each memory block in transform()
2313 : */
2314 22066 : *othersize = 0;
2315 :
2316 22066 : return TRUE;
2317 : }
|
