Line data Source code
1 : /**
2 : * GStreamer
3 : * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
4 : * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
5 : * Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>
6 : *
7 : * This library is free software; you can redistribute it and/or
8 : * modify it under the terms of the GNU Library General Public
9 : * License as published by the Free Software Foundation;
10 : * version 2.1 of the License.
11 : *
12 : * This library is distributed in the hope that it will be useful,
13 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : * Library General Public License for more details.
16 : *
17 : */
18 : /**
19 : * @file gsttensor_transform.c
20 : * @date 10 Jul 2018
21 : * @brief GStreamer plugin to transform tensor dimension or type
22 : * @see https://github.com/nnstreamer/nnstreamer
23 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
24 : * @bug This is NYI.
25 : *
26 : */
27 :
28 : /**
29 : * SECTION:element-tensor_transform
30 : *
31 : * A filter that transforms tensors dimension or type.
32 : * The input and output is always in the format of other/tensor or other/tensors.
33 : *
34 : * <refsect2>
35 : * <title>Example launch line</title>
36 : * |[
37 : * gst-launch -v -m fakesrc ! tensor_converter ! tensor_transform mode=dimchg option=0:2 ! fakesink silent=TRUE
38 : * ]|
39 : * <title>How to use dimchg</title>
40 : * |[
41 : * option=0:2 # Move 0th dim to 2nd dim. I.e., [a][H][W][C] ==> [a][C][H][W]
42 : * ]|
43 : * </refsect2>
44 : */
45 :
46 : #ifdef HAVE_CONFIG_H
47 : #include <config.h>
48 : #endif
49 :
50 : #include <string.h>
51 : #include <math.h>
52 : #include <nnstreamer_log.h>
53 : #include <nnstreamer_util.h>
54 : #include "gsttensor_transform.h"
55 :
56 : #ifdef HAVE_ORC
57 : #include "nnstreamer-orc.h"
58 : #endif
59 :
60 : /**
61 : * @brief Macro for debug mode.
62 : */
63 : #ifndef DBG
64 : #define DBG (!filter->silent)
65 : #endif
66 :
67 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_transform_debug);
68 : #define GST_CAT_DEFAULT gst_tensor_transform_debug
69 : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_MAKE ("{ static, flexible }")
70 : #define REGEX_DIMCHG_OPTION "^([0-9]|1[0-5]):([0-9]|1[0-5])$"
71 : #define REGEX_TYPECAST_OPTION "(^[u]?int(8|16|32|64)$|^float(16|32|64)$)"
72 : #define REGEX_TRANSPOSE_OPTION "^(?:([0-2]):(?!.*\\1)){3}3$"
73 : #define REGEX_STAND_OPTION "^(default|dc-average)(:([u]?int(8|16|32|64)|float(16|32|64)))?(,per-channel:(true|false))?$"
74 : #define REGEX_CLAMP_OPTION "^((([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?))):"\
75 : "((([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?)))$"
76 : #define REGEX_PADDING_OPTION "^((left|right|top|bottom|front|back):(\\d)(,)?)+(layout:(NCHW|NHWC))?$"
77 : #define REGEX_ARITH_OPTION "^(typecast:([u]?int(8|16|32|64)|float(16|32|64)),)?"\
78 : "(per-channel:(false|true@[0-9]+),)?"\
79 : "(((add|mul|div)(:([-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?))+(@[0-9]+)?)(,|))+$"
80 :
81 : #define REGEX_ARITH_OPTION_TYPECAST "(typecast:([u]?int(8|16|32|64)|float(16|32|64)))"
82 :
83 : /**
84 : * @brief The transpose rank is fixed to 4.
85 : * This RANK does not affect other/tensors(s)'s NNS_TENSOR_RANK_LIMIT.
86 : */
87 : #define NNS_TENSOR_TRANSPOSE_RANK_LIMIT (4)
88 :
89 : /**
90 : * @brief The padding rank is fixed to 3.
91 : * This RANK does not affect other/tensors(s)'s NNS_TENSOR_RANK_LIMIT.
92 : */
93 : #define NNS_TENSOR_PADDING_RANK_LIMIT (3)
94 :
95 : /**
96 : * @brief tensor_transform properties
97 : */
98 : enum
99 : {
100 : PROP_0,
101 : PROP_SILENT,
102 : PROP_MODE,
103 : PROP_OPTION,
104 : PROP_ACCELERATION,
105 : PROP_APPLY,
106 : PROP_TRANSPOSE_RANK_LIMIT
107 : };
108 :
109 : /**
110 : * @brief Flag to set orc acceleration.
111 : */
112 : #ifdef HAVE_ORC
113 : #define DEFAULT_ACCELERATION TRUE
114 : #else
115 : #define DEFAULT_ACCELERATION FALSE
116 : #endif
117 :
118 : static const gchar *gst_tensor_transform_stand_string[] = {
119 : [STAND_DEFAULT] = "default",
120 : [STAND_DC_AVERAGE] = "dc-average",
121 : [STAND_END] = NULL
122 : };
123 :
124 : static const gchar *gst_tensor_transform_operator_string[] = {
125 : [GTT_OP_TYPECAST] = "typecast",
126 : [GTT_OP_ADD] = "add",
127 : [GTT_OP_MUL] = "mul",
128 : [GTT_OP_DIV] = "div",
129 : [GTT_OP_UNKNOWN] = NULL
130 : };
131 :
132 : /**
133 : * @brief The capabilities of the inputs
134 : */
135 : static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
136 : GST_PAD_SINK,
137 : GST_PAD_ALWAYS,
138 : GST_STATIC_CAPS (CAPS_STRING));
139 :
140 : /**
141 : * @brief The capabilities of the outputs
142 : */
143 : static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
144 : GST_PAD_SRC,
145 : GST_PAD_ALWAYS,
146 : GST_STATIC_CAPS (CAPS_STRING));
147 :
148 : #define gst_tensor_transform_parent_class parent_class
149 1855 : G_DEFINE_TYPE (GstTensorTransform, gst_tensor_transform,
150 : GST_TYPE_BASE_TRANSFORM);
151 :
152 : /* GObject vmethod implementations */
153 : static void gst_tensor_transform_set_property (GObject * object, guint prop_id,
154 : const GValue * value, GParamSpec * pspec);
155 : static void gst_tensor_transform_get_property (GObject * object, guint prop_id,
156 : GValue * value, GParamSpec * pspec);
157 : static void gst_tensor_transform_finalize (GObject * object);
158 :
159 : /* GstBaseTransformer vmethod implementations */
160 : static GstFlowReturn gst_tensor_transform_transform (GstBaseTransform * trans,
161 : GstBuffer * inbuf, GstBuffer * outbuf);
162 : static GstCaps *gst_tensor_transform_transform_caps (GstBaseTransform * trans,
163 : GstPadDirection direction, GstCaps * caps, GstCaps * filter);
164 : static GstCaps *gst_tensor_transform_fixate_caps (GstBaseTransform * trans,
165 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
166 : static gboolean gst_tensor_transform_set_caps (GstBaseTransform * trans,
167 : GstCaps * incaps, GstCaps * outcaps);
168 : static gboolean gst_tensor_transform_transform_size (GstBaseTransform * trans,
169 : GstPadDirection direction, GstCaps * caps, gsize size,
170 : GstCaps * othercaps, gsize * othersize);
171 :
172 : static gboolean gst_tensor_transform_convert_dimension (GstTensorTransform *
173 : filter, GstPadDirection direction, guint idx, const GstTensorInfo * in_info,
174 : GstTensorInfo * out_info);
175 :
176 : #define GST_TYPE_TENSOR_TRANSFORM_MODE (gst_tensor_transform_mode_get_type ())
177 : /**
178 : * @brief A private function to register GEnumValue array for the 'mode' property
179 : * to a GType and return it
180 : */
181 : static GType
182 106 : gst_tensor_transform_mode_get_type (void)
183 : {
184 : static GType mode_type = 0;
185 :
186 106 : if (mode_type == 0) {
187 : static GEnumValue mode_types[] = {
188 : {GTT_DIMCHG, "Mode for changing tensor dimensions, "
189 : "option=FROM_DIM:TO_DIM (with a regex, " REGEX_DIMCHG_OPTION
190 : ", where NNS_TENSOR_RANK_LIMIT is 16)",
191 : "dimchg"},
192 : {GTT_TYPECAST, "Mode for casting type of tensor, "
193 : "option=" REGEX_TYPECAST_OPTION, "typecast"},
194 : {GTT_ARITHMETIC, "Mode for arithmetic operations with tensor, "
195 : "option=[typecast:TYPE,][per-channel:(false|true@DIM),]add|mul|div:NUMBER[@CH_IDX], ...",
196 : "arithmetic"},
197 : {GTT_TRANSPOSE, "Mode for transposing shape of tensor, "
198 : "option=D1\':D2\':D3\':D4 (fixed to 3)",
199 : "transpose"},
200 : {GTT_STAND, "Mode for statistical standardization of tensor, "
201 : "option=(default|dc-average)[:TYPE][,per-channel:(false|true)]",
202 : "stand"},
203 : {GTT_CLAMP, "Mode for clamping all elements of tensor into the range, "
204 : "option=CLAMP_MIN:CLAMP_MAX",
205 : "clamp"},
206 : {GTT_PADDING, "Mode for padding of tensor, "
207 : "option=left|right|top|bottom|front|back:NUMBER[,layout:(NCHW|NHWC)]",
208 : "padding"},
209 : {GTT_UNKNOWN, "Unknown or not-implemented-yet mode",
210 : "unknown"},
211 : {0, NULL, NULL},
212 : };
213 :
214 106 : mode_type = g_enum_register_static ("gtt_mode_type", mode_types);
215 : }
216 :
217 106 : return mode_type;
218 : }
219 :
220 : /**
221 : * @brief initialize the tensor_transform's class
222 : */
223 : static void
224 106 : gst_tensor_transform_class_init (GstTensorTransformClass * klass)
225 : {
226 : GObjectClass *gobject_class;
227 : GstElementClass *gstelement_class;
228 : GstBaseTransformClass *trans_class;
229 :
230 106 : GST_DEBUG_CATEGORY_INIT (gst_tensor_transform_debug, "tensor_transform", 0,
231 : "Element to transforms tensor dimension or type");
232 :
233 106 : trans_class = (GstBaseTransformClass *) klass;
234 106 : gstelement_class = (GstElementClass *) trans_class;
235 106 : gobject_class = (GObjectClass *) gstelement_class;
236 :
237 106 : gobject_class->set_property = gst_tensor_transform_set_property;
238 106 : gobject_class->get_property = gst_tensor_transform_get_property;
239 106 : gobject_class->finalize = gst_tensor_transform_finalize;
240 :
241 106 : g_object_class_install_property (gobject_class, PROP_SILENT,
242 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output ?",
243 : FALSE, G_PARAM_READWRITE));
244 106 : g_object_class_install_property (gobject_class, PROP_MODE,
245 : g_param_spec_enum ("mode", "Mode", "Mode used for transforming tensor",
246 : GST_TYPE_TENSOR_TRANSFORM_MODE, GTT_UNKNOWN,
247 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
248 106 : g_object_class_install_property (gobject_class, PROP_OPTION,
249 : g_param_spec_string ("option", "Option",
250 : "Option for the tensor transform mode ?", "", G_PARAM_READWRITE));
251 106 : g_object_class_install_property (gobject_class, PROP_ACCELERATION,
252 : g_param_spec_boolean ("acceleration", "Acceleration", "Orc acceleration",
253 : DEFAULT_ACCELERATION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
254 106 : g_object_class_install_property (gobject_class, PROP_APPLY,
255 : g_param_spec_string ("apply", "Apply", "Select tensors to apply, "
256 : "separated with ',' in case of multiple tensors. Default to apply all tensors.",
257 : "", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
258 106 : g_object_class_install_property (gobject_class, PROP_TRANSPOSE_RANK_LIMIT,
259 : g_param_spec_uint ("transpose-rank-limit", "Transpose rank limit",
260 : "The rank limit of transpose, which varies per version of nnstreamer and may be lower than the global rank limit if it is over 4.",
261 : 0, NNS_TENSOR_RANK_LIMIT, NNS_TENSOR_TRANSPOSE_RANK_LIMIT,
262 : G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
263 :
264 106 : gst_element_class_set_details_simple (gstelement_class,
265 : "TensorTransform",
266 : "Filter/Tensor",
267 : "Transforms other/tensor dimensions for different models or frameworks",
268 : "MyungJoo Ham <myungjoo.ham@samsung.com>");
269 :
270 106 : gst_element_class_add_pad_template (gstelement_class,
271 : gst_static_pad_template_get (&src_factory));
272 106 : gst_element_class_add_pad_template (gstelement_class,
273 : gst_static_pad_template_get (&sink_factory));
274 : /* Refer: https://gstreamer.freedesktop.org/documentation/design/element-transform.html */
275 106 : trans_class->passthrough_on_same_caps = FALSE;
276 :
277 : /* Processing units */
278 106 : trans_class->transform = GST_DEBUG_FUNCPTR (gst_tensor_transform_transform);
279 :
280 : /* Negotiation units */
281 106 : trans_class->transform_caps =
282 106 : GST_DEBUG_FUNCPTR (gst_tensor_transform_transform_caps);
283 106 : trans_class->fixate_caps =
284 106 : GST_DEBUG_FUNCPTR (gst_tensor_transform_fixate_caps);
285 106 : trans_class->set_caps = GST_DEBUG_FUNCPTR (gst_tensor_transform_set_caps);
286 :
287 : /* Allocation units */
288 106 : trans_class->transform_size =
289 106 : GST_DEBUG_FUNCPTR (gst_tensor_transform_transform_size);
290 106 : }
291 :
292 : /**
293 : * @brief initialize the new element (G_DEFINE_TYPE requires this)
294 : * instantiate pads and add them to element
295 : * set pad callback functions
296 : * initialize instance structure
297 : */
298 : static void
299 257 : gst_tensor_transform_init (GstTensorTransform * filter)
300 : {
301 257 : filter->silent = TRUE;
302 257 : filter->mode = GTT_UNKNOWN;
303 257 : filter->option = NULL;
304 257 : filter->loaded = FALSE;
305 257 : filter->operators = NULL;
306 257 : filter->acceleration = DEFAULT_ACCELERATION;
307 257 : filter->apply = NULL;
308 :
309 257 : gst_tensors_config_init (&filter->in_config);
310 257 : gst_tensors_config_init (&filter->out_config);
311 257 : }
312 :
313 : /**
314 : * @brief Get the corresponding operator from the string value
315 : * @param[in] str The string value for the operator
316 : * @return corresponding operator for the string (GTT_OP_UNKNOWN for errors)
317 : */
318 : static tensor_transform_operator
319 178 : gst_tensor_transform_get_operator (const gchar * str)
320 : {
321 : int index;
322 :
323 178 : index = find_key_strv (gst_tensor_transform_operator_string, str);
324 :
325 178 : return (index < 0) ? GTT_OP_UNKNOWN : index;
326 : }
327 :
328 : /**
329 : * @brief Get the corresponding mode from the string value
330 : * @param[in] str The string value for the mode
331 : * @return corresponding mode for the string. STAND_END for errors
332 : */
333 : static tensor_transform_stand_mode
334 4 : gst_tensor_transform_get_stand_mode (const gchar * str)
335 : {
336 : int index;
337 :
338 4 : index = find_key_strv (gst_tensor_transform_stand_string, str);
339 :
340 4 : return (index < 0) ? STAND_END : index;
341 : }
342 :
343 : #ifndef FLOAT16_SUPPORT
344 : /**
345 : * @brief Generate error if float16 is required.
346 : */
347 : static void
348 0 : float16_not_supported (void)
349 : {
350 0 : ml_loge
351 : ("Tensor_transform does not support float16 operators. Apply -Denable-float16=true for meson build option if your architecture support float16. Note that tensor-transform's float16 is adhoc and does NOT perform good (slow!).\n");
352 0 : g_assert (0);
353 : }
354 : #endif
355 :
356 : #ifdef FLOAT16_SUPPORT
357 : /**
358 : * @brief Refrain from heavy operations on float16
359 : * @todo Remove this after applying SIMD or ORC
360 : */
361 : static void
362 : refrain_from_heavy_op_on_float16 (gulong n)
363 : {
364 : static int warned = 0;
365 : /* 1 million */
366 : if (n > 1000000) {
367 : if (warned)
368 : return;
369 : ml_logw
370 : ("Tensor_transform implementation for float16 does not support SIMD. Heavy tensor-transform operations of float16 is not recommended. Try to apply heavy ops with other types (e.g., float32) and convert it to float16 at the time when it's really needed.\n");
371 : warned = 1;
372 : }
373 : }
374 :
375 : /** @todo Make this use SIMD or ORC */
376 : #define _conv_to_f16(intype, o, i, n) \
377 : do { \
378 : float16 *op = (gpointer) (o); \
379 : intype *ip = (gpointer) (i); \
380 : gulong idx; \
381 : refrain_from_heavy_op_on_float16 (n); \
382 : for (idx = 0; idx < n; idx++) \
383 : *(op + idx) = (float16) *(ip + idx); \
384 : } while (0)
385 :
386 : /** @todo Make this use SIMD or ORC */
387 : #define _conv_from_f16_action(n, op, ip, otypename) \
388 : do { \
389 : gulong idx; \
390 : for (idx = 0; idx < n; idx++) \
391 : *(op + idx) = (otypename) *(ip + idx); \
392 : } while (0)
393 :
394 : /** @todo Make this use SIMD or ORC */
395 : #define _conv_from_f16(otype, o, i, n) \
396 : do { \
397 : float16 *ip = (gpointer) (i); \
398 : refrain_from_heavy_op_on_float16 (n); \
399 : switch (otype) { \
400 : case _NNS_INT32: { \
401 : int32_t *op = (gpointer) (o); \
402 : _conv_from_f16_action (n, op, ip, int32_t); \
403 : break; } \
404 : case _NNS_UINT32: { \
405 : uint32_t *op = (gpointer) (o); \
406 : _conv_from_f16_action (n, op, ip, uint32_t); \
407 : break; } \
408 : case _NNS_INT16: { \
409 : int16_t *op = (gpointer) (o); \
410 : _conv_from_f16_action (n, op, ip, int16_t); \
411 : break; } \
412 : case _NNS_UINT16: { \
413 : uint16_t *op = (gpointer) (o); \
414 : _conv_from_f16_action (n, op, ip, uint16_t); \
415 : break; } \
416 : case _NNS_INT8: { \
417 : int8_t *op = (gpointer) (o); \
418 : _conv_from_f16_action (n, op, ip, int8_t); \
419 : break; } \
420 : case _NNS_UINT8: { \
421 : uint8_t *op = (gpointer) (o); \
422 : _conv_from_f16_action (n, op, ip, uint8_t); \
423 : break; } \
424 : case _NNS_FLOAT64: { \
425 : double *op = (gpointer) (o); \
426 : _conv_from_f16_action (n, op, ip, double); \
427 : break; } \
428 : case _NNS_FLOAT32: { \
429 : float *op = (gpointer) (o); \
430 : _conv_from_f16_action (n, op, ip, float); \
431 : break; } \
432 : case _NNS_FLOAT16: { \
433 : float16 *op = (gpointer) (o); \
434 : _conv_from_f16_action (n, op, ip, float16); \
435 : break; } \
436 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (otype)); g_assert (0); \
437 : } \
438 : } while (0)
439 :
440 : /** @todo Make this use SIMD or ORC */
441 : #define _op_float16(i, n, v, op) \
442 : do { \
443 : gulong idx; \
444 : float16 *data_in = (float16 *) (i); \
445 : refrain_from_heavy_op_on_float16 (n); \
446 : switch (op) { \
447 : case GTT_OP_ADD: \
448 : for (idx = 0; idx < n; idx++) \
449 : data_in[idx] = data_in[idx] + (v); \
450 : break; \
451 : case GTT_OP_MUL: \
452 : for (idx = 0; idx < n; idx++) \
453 : data_in[idx] = data_in[idx] * (v); \
454 : break; \
455 : case GTT_OP_DIV: \
456 : for (idx = 0; idx < n; idx++) \
457 : data_in[idx] = data_in[idx] / (v); \
458 : break; \
459 : default: GST_ERROR_OBJECT (filter, "Unknown operator for float16: %d", op); break; \
460 : } \
461 : } while (0)
462 :
463 : #else /* ! FLOAT16_SUPPORT */
464 : #define _conv_to_f16(intype, o, i, n) do { float16_not_supported (); } while (0)
465 : #define _conv_from_f16(otype, o, i, n) do { float16_not_supported (); } while (0)
466 : #define _op_float16(i, n, v, op) do { float16_not_supported (); } while (0)
467 : #endif /* FLOAT16_SUPPORT */
468 :
469 : #ifdef HAVE_ORC
470 : /* define macros for orc */
471 : #define orc_func_conv(intype,outtype) nns_orc_conv_ ## intype ## _to_ ## outtype
472 : #define orc_func_add(intype) nns_orc_add_c_ ## intype
473 : #define orc_func_mul(intype) nns_orc_mul_c_ ## intype
474 : #define orc_func_div(intype) nns_orc_div_c_ ## intype
475 :
476 : #define orc_typecast_to(i,o,n,intype,otype,intypename) do { \
477 : switch (otype) { \
478 : case _NNS_INT32: orc_func_conv (intype, s32) ((gpointer) o, (gpointer) i, n); break; \
479 : case _NNS_UINT32: orc_func_conv (intype, u32) ((gpointer) o, (gpointer) i, n); break; \
480 : case _NNS_INT16: orc_func_conv (intype, s16) ((gpointer) o, (gpointer) i, n); break; \
481 : case _NNS_UINT16: orc_func_conv (intype, u16) ((gpointer) o, (gpointer) i, n); break; \
482 : case _NNS_INT8: orc_func_conv (intype, s8) ((gpointer) o, (gpointer) i, n); break; \
483 : case _NNS_UINT8: orc_func_conv (intype, u8) ((gpointer) o, (gpointer) i, n); break; \
484 : case _NNS_FLOAT64: orc_func_conv (intype, f64) ((gpointer) o, (gpointer) i, n); break; \
485 : case _NNS_FLOAT32: orc_func_conv (intype, f32) ((gpointer) o, (gpointer) i, n); break; \
486 : case _NNS_INT64: orc_func_conv (intype, s64) ((gpointer) o, (gpointer) i, n); break; \
487 : case _NNS_UINT64: orc_func_conv (intype, u64) ((gpointer) o, (gpointer) i, n); break; \
488 : case _NNS_FLOAT16: _conv_to_f16 (intypename, o, i, n); break; \
489 : default: GST_ERROR_OBJECT (filter, "Unsupported output type %d", otype); g_assert (0); break; \
490 : } \
491 : } while (0)
492 :
493 : #define orc_typecast(i,o,n,itype,otype) do { \
494 : switch (itype) { \
495 : case _NNS_INT32: orc_typecast_to (i, o, n, s32, otype, int32_t); break; \
496 : case _NNS_UINT32: orc_typecast_to (i, o, n, u32, otype, uint32_t); break; \
497 : case _NNS_INT16: orc_typecast_to (i, o, n, s16, otype, int16_t); break; \
498 : case _NNS_UINT16: orc_typecast_to (i, o, n, u16, otype, uint16_t); break; \
499 : case _NNS_INT8: orc_typecast_to (i, o, n, s8, otype, int8_t); break; \
500 : case _NNS_UINT8: orc_typecast_to (i, o, n, u8, otype, uint8_t); break; \
501 : case _NNS_FLOAT64: orc_typecast_to (i, o, n, f64, otype, double); break; \
502 : case _NNS_FLOAT32: orc_typecast_to (i, o, n, f32, otype, float); break; \
503 : case _NNS_INT64: orc_typecast_to (i, o, n, s64, otype, int64_t); break; \
504 : case _NNS_UINT64: orc_typecast_to (i, o, n, u64, otype, uint64_t); break; \
505 : case _NNS_FLOAT16: _conv_from_f16 (otype, o, i, n); break; \
506 : default: GST_ERROR_OBJECT (filter, "Unsupported input type %d", itype); g_assert (0); break; \
507 : } \
508 : } while (0)
509 :
510 : #define orc_typesize(size, type) do { \
511 : switch (type) { \
512 : case _NNS_INT32: size = sizeof(int32_t); break; \
513 : case _NNS_UINT32: size = sizeof(uint32_t); break; \
514 : case _NNS_INT16: size = sizeof(int16_t); break; \
515 : case _NNS_UINT16: size = sizeof(uint16_t); break; \
516 : case _NNS_INT8: size = sizeof(int8_t); break; \
517 : case _NNS_UINT8: size = sizeof(uint8_t); break; \
518 : case _NNS_FLOAT64: size = sizeof(double); break; \
519 : case _NNS_FLOAT32: size = sizeof(float); break; \
520 : case _NNS_INT64: size = sizeof(int64_t); break; \
521 : case _NNS_UINT64: size = sizeof(uint64_t); break; \
522 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", type); g_assert (0); break; \
523 : } \
524 : } while (0)
525 :
526 : #define orc_operator_func(i,n,v,opfunc,op) do { \
527 : switch ((v)->type) { \
528 : case _NNS_INT32: opfunc (s32) ((gpointer) i, (v)->data._int32_t, n); break; \
529 : case _NNS_UINT32: opfunc (u32) ((gpointer) i, (v)->data._uint32_t, n); break; \
530 : case _NNS_INT16: opfunc (s16) ((gpointer) i, (v)->data._int16_t, n); break; \
531 : case _NNS_UINT16: opfunc (u16) ((gpointer) i, (v)->data._uint16_t, n); break; \
532 : case _NNS_INT8: opfunc (s8) ((gpointer) i, (v)->data._int8_t, n); break; \
533 : case _NNS_UINT8: opfunc (u8) ((gpointer) i, (v)->data._uint8_t, n); break; \
534 : case _NNS_FLOAT64: opfunc (f64) ((gpointer) i, (v)->data._double, n); break; \
535 : case _NNS_FLOAT32: opfunc (f32) ((gpointer) i, (v)->data._float, n); break; \
536 : case _NNS_INT64: opfunc (s64) ((gpointer) i, (v)->data._int64_t, n); break; \
537 : case _NNS_UINT64: opfunc (u64) ((gpointer) i, (v)->data._uint64_t, n); break; \
538 : case _NNS_FLOAT16: _op_float16 (i, n, (v)->data._float16, op); break; \
539 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (v)->type); g_assert (0); break; \
540 : } \
541 : } while (0)
542 :
543 : #define orc_operator_div_loop(i,n,val,typename) do { \
544 : gsize idx_div; \
545 : typename *data_in = (typename *) (i); \
546 : for (idx_div = 0; idx_div < (n); ++idx_div) { \
547 : data_in[idx_div] = data_in[idx_div] / (val); \
548 : } \
549 : } while (0)
550 :
551 : #define orc_operator(i,n,v,op) do { \
552 : switch (op) { \
553 : case GTT_OP_ADD: orc_operator_func (i, n, v, orc_func_add, op); break; \
554 : case GTT_OP_MUL: orc_operator_func (i, n, v, orc_func_mul, op); break; \
555 : case GTT_OP_DIV: \
556 : switch ((v)->type) { \
557 : case _NNS_INT32: orc_operator_div_loop (i, n, (v)->data._int32_t, int32_t); break; \
558 : case _NNS_UINT32: orc_operator_div_loop (i, n, (v)->data._uint32_t, uint32_t); break; \
559 : case _NNS_INT16: orc_operator_div_loop (i, n, (v)->data._int16_t, int16_t); break; \
560 : case _NNS_UINT16: orc_operator_div_loop (i, n, (v)->data._uint16_t, uint16_t); break; \
561 : case _NNS_INT8: orc_operator_div_loop (i, n, (v)->data._int8_t, int8_t); break; \
562 : case _NNS_UINT8: orc_operator_div_loop (i, n, (v)->data._uint8_t, uint8_t); break; \
563 : case _NNS_FLOAT64: orc_func_div (f64) ((gpointer) i, (v)->data._double, n); break; \
564 : case _NNS_FLOAT32: orc_func_div (f32) ((gpointer) i, (v)->data._float, n); break; \
565 : case _NNS_INT64: orc_operator_div_loop (i, n, (v)->data._int64_t, int64_t); break; \
566 : case _NNS_UINT64: orc_operator_div_loop (i, n, (v)->data._uint64_t, uint64_t); break; \
567 : case _NNS_FLOAT16: _op_float16 (i, n, (v)->data._float16, op); break; \
568 : default: GST_ERROR_OBJECT (filter, "Unsupported type %d", (v)->type); g_assert (0); break; \
569 : } \
570 : break; \
571 : default: GST_ERROR_OBJECT (filter, "Unknown operator %d", op); break; \
572 : } \
573 : } while (0)
574 : #endif /* HAVE_ORC */
575 :
576 : /**
577 : * @brief Macro for operator
578 : */
579 : #define handle_operator(d,v,oper,vtype) do { \
580 : switch (oper) { \
581 : case GTT_OP_ADD: \
582 : (d)->data._##vtype += (v)->data._##vtype; \
583 : break; \
584 : case GTT_OP_MUL: \
585 : (d)->data._##vtype *= (v)->data._##vtype; \
586 : break; \
587 : case GTT_OP_DIV: \
588 : if ((v)->data._##vtype == 0) { \
589 : GST_ERROR_OBJECT (filter, "Invalid state, denominator is 0."); \
590 : return FALSE; \
591 : } \
592 : (d)->data._##vtype /= (v)->data._##vtype; \
593 : break; \
594 : default: \
595 : GST_ERROR_OBJECT (filter, "Unknown operator %d", oper); \
596 : return FALSE; \
597 : } \
598 : } while (0)
599 :
600 : /**
601 : * @brief Handle operators for tensor value
602 : * @param filter "this" pointer
603 : * @param desc struct for tensor value
604 : * @param val struct for tensor value
605 : * @param op operator for given tensor value
606 : * @return TRUE if no error
607 : */
608 : static gboolean
609 61615 : gst_tensor_transform_do_operator (GstTensorTransform * filter,
610 : tensor_data_s * desc, const tensor_data_s * val,
611 : tensor_transform_operator op)
612 : {
613 61615 : g_return_val_if_fail (desc != NULL, FALSE);
614 61615 : g_return_val_if_fail (val != NULL, FALSE);
615 61615 : g_return_val_if_fail (desc->type == val->type, FALSE);
616 :
617 61615 : switch (desc->type) {
618 7755 : case _NNS_INT32:
619 7755 : handle_operator (desc, val, op, int32_t);
620 7755 : break;
621 7680 : case _NNS_UINT32:
622 7680 : handle_operator (desc, val, op, uint32_t);
623 7680 : break;
624 7680 : case _NNS_INT16:
625 7680 : handle_operator (desc, val, op, int16_t);
626 7680 : break;
627 7680 : case _NNS_UINT16:
628 7680 : handle_operator (desc, val, op, uint16_t);
629 7680 : break;
630 7680 : case _NNS_INT8:
631 7680 : handle_operator (desc, val, op, int8_t);
632 7680 : break;
633 7680 : case _NNS_UINT8:
634 7680 : handle_operator (desc, val, op, uint8_t);
635 7680 : break;
636 45 : case _NNS_FLOAT64:
637 45 : handle_operator (desc, val, op, double);
638 45 : break;
639 55 : case _NNS_FLOAT32:
640 55 : handle_operator (desc, val, op, float);
641 55 : break;
642 0 : case _NNS_FLOAT16:
643 : #ifdef FLOAT16_SUPPORT
644 : handle_operator (desc, val, op, float16);
645 : #else
646 0 : float16_not_supported ();
647 : #endif
648 0 : break;
649 7680 : case _NNS_INT64:
650 7680 : handle_operator (desc, val, op, int64_t);
651 7680 : break;
652 7680 : case _NNS_UINT64:
653 7680 : handle_operator (desc, val, op, uint64_t);
654 7680 : break;
655 0 : default:
656 0 : GST_ERROR_OBJECT (filter, "Unknown tensor type %d", desc->type);
657 0 : return FALSE;
658 : }
659 :
660 61615 : return TRUE;
661 : }
662 :
663 : /**
664 : * @brief Setup internal data (data_* in GstTensorTransform)
665 : * @param[in/out] filter "this" pointer. mode & option MUST BE set already.
666 : * @retval TRUE if OK or operation-skipped, FALSE if fatal-error.
667 : */
668 : static gboolean
669 547 : gst_tensor_transform_set_option_data (GstTensorTransform * filter)
670 : {
671 : gchar *filter_name;
672 547 : gboolean ret = FALSE;
673 :
674 547 : if (filter->mode == GTT_UNKNOWN || filter->option == NULL)
675 286 : return TRUE;
676 :
677 261 : filter_name = gst_object_get_name ((GstObject *) filter);
678 :
679 261 : switch (filter->mode) {
680 8 : case GTT_DIMCHG:
681 : {
682 8 : gchar **strv = NULL;
683 :
684 8 : if (!g_regex_match_simple (REGEX_DIMCHG_OPTION, filter->option,
685 : G_REGEX_CASELESS, 0)) {
686 3 : ml_loge
687 : ("%s: dimchg: \'%s\' is not valid option string: it should be in the form of IDX_DIM_FROM:IDX_DIM_TO: with a regex, "
688 : REGEX_DIMCHG_OPTION "\n", filter_name, filter->option);
689 3 : break;
690 : }
691 :
692 5 : strv = g_strsplit (filter->option, ":", 2);
693 :
694 5 : filter->data_dimchg.from = (int) g_ascii_strtoll (strv[0], NULL, 10);
695 5 : filter->data_dimchg.to = (int) g_ascii_strtoll (strv[1], NULL, 10);
696 5 : ret = filter->loaded = TRUE;
697 5 : g_strfreev (strv);
698 5 : break;
699 : }
700 119 : case GTT_TYPECAST:
701 : {
702 119 : if (g_regex_match_simple (REGEX_TYPECAST_OPTION, filter->option,
703 : G_REGEX_CASELESS, 0)) {
704 118 : filter->data_typecast.to = gst_tensor_get_type (filter->option);
705 118 : ret = filter->loaded = TRUE;
706 : } else {
707 1 : ml_loge
708 : ("%s: typecast: \'%s\' is not valid data type for tensor: data type of tensor should be one of %s\n",
709 : filter_name, filter->option, GST_TENSOR_TYPE_ALL);
710 : }
711 119 : break;
712 : }
713 91 : case GTT_ARITHMETIC:
714 : {
715 : gchar *str_option;
716 : gchar **str_operators;
717 : gchar **str_op;
718 : tensor_transform_operator_s *op_s;
719 : guint i, num_operators, num_op;
720 : GRegex *regex_option_tc;
721 :
722 91 : filter->data_arithmetic.out_type = _NNS_END;
723 91 : filter->data_arithmetic.per_channel_arith = FALSE;
724 :
725 91 : if (filter->operators) {
726 2 : GST_WARNING_OBJECT (filter,
727 : "There exists pre-defined operators (total %d), now reset these.",
728 : g_slist_length (filter->operators));
729 :
730 2 : g_slist_free_full (filter->operators, g_free);
731 2 : filter->operators = NULL;
732 : }
733 :
734 91 : regex_option_tc = g_regex_new (REGEX_ARITH_OPTION_TYPECAST,
735 : G_REGEX_CASELESS, 0, 0);
736 :
737 91 : if (!regex_option_tc) {
738 0 : GST_ERROR_OBJECT (filter,
739 : "arithmetic: failed to create a GRegex structure for %s\n",
740 : REGEX_ARITH_OPTION_TYPECAST);
741 0 : break;
742 : }
743 :
744 91 : if (g_regex_match_full (regex_option_tc, filter->option, -1,
745 : 1, 0, NULL, NULL)) {
746 2 : str_option = g_regex_replace (regex_option_tc, filter->option, -1, 1,
747 : "", 0, 0);
748 2 : ml_loge
749 : ("%s: arithmetic: [typecast:TYPE,] should be located at the first to prevent memory re-allocation: typecast(s) in the middle of \'%s\' will be ignored\n",
750 : filter_name, filter->option);
751 : } else {
752 178 : str_option = g_strdup (filter->option);
753 : }
754 91 : g_regex_unref (regex_option_tc);
755 :
756 91 : if (!g_regex_match_simple (REGEX_ARITH_OPTION, str_option,
757 : G_REGEX_CASELESS, 0)) {
758 13 : ml_loge
759 : ("%s: arithmetic: \'%s\' is not valid option string: it should be in the form of [typecast:TYPE,][per-channel:(false|true@DIM),]add|mul|div:NUMBER[@CH_IDX]..., ...\n",
760 : filter_name, str_option);
761 13 : g_free (str_option);
762 13 : break;
763 : }
764 78 : str_operators = g_strsplit (str_option, ",", -1);
765 78 : num_operators = g_strv_length (str_operators);
766 :
767 260 : for (i = 0; i < num_operators; ++i) {
768 182 : str_op = g_strsplit (str_operators[i], ":", -1);
769 182 : num_op = g_strv_length (str_op);
770 :
771 182 : if (str_op[0]) {
772 180 : gchar **values = g_strsplit (str_op[1], "@", -1);
773 180 : guint num_values = g_strv_length (values);
774 :
775 : /* check whether per-channel */
776 180 : if (g_ascii_strcasecmp (str_op[0], "per-channel") == 0) {
777 2 : if (num_values > 1 && g_ascii_strcasecmp (values[0], "true") == 0) {
778 2 : ml_logi
779 : ("Set per-channel for arithmetic and assume that %s-th dim is the channel",
780 : values[1]);
781 2 : filter->data_arithmetic.per_channel_arith = TRUE;
782 2 : filter->data_arithmetic.ch_dim =
783 2 : (guint) g_ascii_strtoull (values[1], NULL, 10);
784 : }
785 :
786 2 : g_strfreev (values);
787 2 : g_strfreev (str_op);
788 2 : continue;
789 : }
790 :
791 178 : op_s = g_new0 (tensor_transform_operator_s, 1);
792 178 : g_assert (op_s);
793 :
794 178 : op_s->op = gst_tensor_transform_get_operator (str_op[0]);
795 178 : op_s->applying_ch = -1; /* -1 means applying to all channels */
796 178 : switch (op_s->op) {
797 61 : case GTT_OP_TYPECAST:
798 61 : if (num_op > 1 && str_op[1]) {
799 61 : op_s->value.type = gst_tensor_get_type (values[0]);
800 61 : filter->data_arithmetic.out_type = op_s->value.type;
801 : } else {
802 0 : GST_WARNING_OBJECT (filter, "Invalid option for typecast %s",
803 : str_operators[i]);
804 0 : op_s->op = GTT_OP_UNKNOWN;
805 : }
806 61 : break;
807 117 : case GTT_OP_ADD:
808 : case GTT_OP_MUL:
809 : case GTT_OP_DIV:
810 117 : if (num_op > 1 && str_op[1]) {
811 : /* get operand */
812 117 : if (strchr (values[0], '.') || strchr (values[0], 'e') ||
813 117 : strchr (values[0], 'E')) {
814 : double val;
815 :
816 81 : val = g_ascii_strtod (values[0], NULL);
817 81 : gst_tensor_data_set (&op_s->value, _NNS_FLOAT64, &val);
818 : } else {
819 : int64_t val;
820 :
821 36 : val = g_ascii_strtoll (values[0], NULL, 10);
822 36 : gst_tensor_data_set (&op_s->value, _NNS_INT64, &val);
823 : }
824 :
825 117 : if (filter->data_arithmetic.per_channel_arith && num_values > 1) {
826 11 : op_s->applying_ch = g_ascii_strtoll (values[1], NULL, 10);
827 : }
828 :
829 : } else {
830 0 : GST_WARNING_OBJECT (filter,
831 : "Invalid option for arithmetic %s", str_operators[i]);
832 0 : op_s->op = GTT_OP_UNKNOWN;
833 : }
834 117 : break;
835 0 : default:
836 0 : GST_WARNING_OBJECT (filter, "Unknown operator %s", str_op[0]);
837 0 : break;
838 : }
839 :
840 : /* append operator */
841 178 : if (op_s->op != GTT_OP_UNKNOWN) {
842 178 : filter->operators = g_slist_append (filter->operators, op_s);
843 : } else {
844 0 : g_free (op_s);
845 : }
846 :
847 178 : g_strfreev (values);
848 : } else {
849 2 : GST_WARNING_OBJECT (filter, "Invalid option %s", str_operators[i]);
850 : }
851 :
852 180 : g_strfreev (str_op);
853 : }
854 :
855 78 : ret = filter->loaded = (filter->operators != NULL);
856 78 : g_strfreev (str_operators);
857 78 : g_free (str_option);
858 78 : break;
859 : }
860 20 : case GTT_TRANSPOSE:
861 : {
862 : int i;
863 20 : gchar **strv = NULL;
864 :
865 20 : if (!g_regex_match_simple (REGEX_TRANSPOSE_OPTION, filter->option,
866 : G_REGEX_CASELESS, 0)) {
867 4 : ml_loge
868 : ("%s: transpose: \'%s\' is not valid option string: it should be in the form of NEW_IDX_DIM0:NEW_IDX_DIM1:NEW_IDX_DIM2:3 (Now transpose mode's rank is fixed to 3. Note that the index of the last dim is always fixed to 3)\n",
869 : filter_name, filter->option);
870 4 : break;
871 : }
872 :
873 16 : strv = g_strsplit (filter->option, ":", NNS_TENSOR_TRANSPOSE_RANK_LIMIT);
874 80 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
875 64 : filter->data_transpose.trans_order[i] =
876 64 : (uint8_t) g_ascii_strtoull (strv[i], NULL, 10);
877 : }
878 :
879 16 : ret = filter->loaded = TRUE;
880 16 : g_strfreev (strv);
881 16 : break;
882 : }
883 9 : case GTT_STAND:
884 : {
885 9 : gchar **options = NULL;
886 : guint i, num_options;
887 :
888 9 : if (!g_regex_match_simple (REGEX_STAND_OPTION, filter->option,
889 : G_REGEX_CASELESS, 0)) {
890 5 : ml_loge
891 : ("%s: stand: \'%s\' is not a valid option string: it should be in the form of (default|dc-average)[:TYPE][,per-channel:(false|true)]\n",
892 : filter_name, filter->option);
893 5 : break;
894 : }
895 :
896 4 : filter->data_stand.out_type = _NNS_END;
897 4 : filter->data_stand.per_channel = FALSE;
898 :
899 4 : options = g_strsplit (filter->option, ",", -1);
900 4 : num_options = g_strv_length (options);
901 :
902 10 : for (i = 0; i < num_options; i++) {
903 6 : gchar **strv = g_strsplit (options[i], ":", -1);
904 :
905 6 : if (g_ascii_strcasecmp (strv[0], "default") == 0 ||
906 4 : g_ascii_strcasecmp (strv[0], "dc-average") == 0) {
907 4 : filter->data_stand.mode =
908 4 : gst_tensor_transform_get_stand_mode (strv[0]);
909 6 : if (g_strv_length (strv) > 1)
910 2 : filter->data_stand.out_type = gst_tensor_get_type (strv[1]);
911 2 : } else if (g_ascii_strcasecmp (strv[0], "per-channel") == 0) {
912 2 : if (g_strv_length (strv) > 1 &&
913 2 : g_ascii_strcasecmp (strv[1], "true") == 0)
914 2 : filter->data_stand.per_channel = TRUE;
915 : } else {
916 0 : filter->data_stand.mode = STAND_END;
917 0 : ml_logw ("Unknown option for stand mode: %s", strv[0]);
918 : }
919 :
920 6 : g_strfreev (strv);
921 : }
922 :
923 4 : g_strfreev (options);
924 4 : ret = filter->loaded = TRUE;
925 4 : break;
926 : }
927 10 : case GTT_CLAMP:
928 : {
929 10 : gchar **strv = NULL;
930 :
931 10 : if (!g_regex_match_simple (REGEX_CLAMP_OPTION, filter->option,
932 : G_REGEX_CASELESS, 0)) {
933 1 : ml_loge
934 : ("%s: clamp: \'%s\' is not valid option string: it should be in the form of [CLAMP_MIN:CLAMP_MAX]\n",
935 : filter_name, filter->option);
936 1 : break;
937 : }
938 :
939 9 : strv = g_strsplit (filter->option, ":", 2);
940 :
941 9 : filter->data_clamp.min = g_ascii_strtod (strv[0], NULL);
942 9 : if (errno == ERANGE) {
943 1 : ml_loge ("%s: clamp: CLAMP_MIN value has an invalid range\n",
944 : filter_name);
945 1 : g_strfreev (strv);
946 1 : break;
947 : }
948 8 : filter->data_clamp.max = g_ascii_strtod (strv[1], NULL);
949 8 : if (errno == ERANGE) {
950 1 : ml_loge ("%s: clamp: CLAMP_MAX value has an invalid range\n",
951 : filter_name);
952 1 : g_strfreev (strv);
953 1 : break;
954 : }
955 :
956 7 : g_strfreev (strv);
957 :
958 7 : if (filter->data_clamp.min > filter->data_clamp.max) {
959 2 : ml_loge ("%s: clamp: CLAMP_MIN is larger than CLAMP_MAX\n",
960 : filter_name);
961 2 : break;
962 : }
963 :
964 5 : ret = filter->loaded = TRUE;
965 5 : break;
966 : }
967 4 : case GTT_PADDING:
968 : {
969 4 : gchar **options = NULL;
970 : guint i, num_options;
971 :
972 4 : if (!g_regex_match_simple (REGEX_PADDING_OPTION, filter->option,
973 : G_REGEX_CASELESS, 0)) {
974 0 : ml_loge
975 : ("%s: padding: \'%s\' is not valid option string: it should be in the form of left|right|top|bottom|front|back:PADDING,[layout:(NCHW|NHWC)]\n",
976 : filter_name, filter->option);
977 0 : break;
978 : }
979 :
980 68 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++)
981 64 : filter->data_padding.pad[i] = 0;
982 4 : filter->data_padding.layout = _NNS_LAYOUT_ANY;
983 :
984 4 : options = g_strsplit (filter->option, ",", -1);
985 4 : num_options = g_strv_length (options);
986 :
987 23 : for (i = 0; i < num_options; i++) {
988 19 : gchar **strv = g_strsplit (options[i], ":", 2);
989 19 : if (g_ascii_strcasecmp (strv[0], "left") == 0) {
990 4 : filter->data_padding.pad[PADDING_LEFT] =
991 4 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
992 15 : } else if (g_ascii_strcasecmp (strv[0], "right") == 0) {
993 4 : filter->data_padding.pad[PADDING_RIGHT] =
994 4 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
995 11 : } else if (g_ascii_strcasecmp (strv[0], "top") == 0) {
996 3 : filter->data_padding.pad[PADDING_TOP] =
997 3 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
998 8 : } else if (g_ascii_strcasecmp (strv[0], "bottom") == 0) {
999 3 : filter->data_padding.pad[PADDING_BOTTOM] =
1000 3 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1001 5 : } else if (g_ascii_strcasecmp (strv[0], "front") == 0) {
1002 2 : filter->data_padding.pad[PADDING_FRONT] =
1003 2 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1004 3 : } else if (g_ascii_strcasecmp (strv[0], "back") == 0) {
1005 2 : filter->data_padding.pad[PADDING_BACK] =
1006 2 : (guint) g_ascii_strtoull (strv[1], NULL, 10);
1007 1 : } else if (g_ascii_strcasecmp (strv[0], "layout") == 0) {
1008 1 : if (g_ascii_strcasecmp (strv[1], "NHWC") == 0)
1009 1 : filter->data_padding.layout = _NNS_LAYOUT_NHWC;
1010 : else
1011 0 : filter->data_padding.layout = _NNS_LAYOUT_NCHW;
1012 : } else {
1013 0 : ml_logw ("Unknown option for padding mode: %s", strv[0]);
1014 : }
1015 19 : g_strfreev (strv);
1016 : }
1017 4 : g_strfreev (options);
1018 :
1019 4 : if (filter->data_padding.layout == _NNS_LAYOUT_NHWC) {
1020 1 : guint prev_left = filter->data_padding.pad[PADDING_LEFT],
1021 1 : prev_right = filter->data_padding.pad[PADDING_RIGHT];
1022 1 : filter->data_padding.pad[PADDING_LEFT] =
1023 1 : filter->data_padding.pad[PADDING_FRONT];
1024 1 : filter->data_padding.pad[PADDING_RIGHT] =
1025 1 : filter->data_padding.pad[PADDING_BACK];
1026 1 : filter->data_padding.pad[PADDING_FRONT] = prev_left;
1027 1 : filter->data_padding.pad[PADDING_BACK] = prev_right;
1028 : }
1029 :
1030 4 : ret = filter->loaded = TRUE;
1031 4 : break;
1032 : }
1033 0 : default:
1034 0 : GST_ERROR_OBJECT (filter, "Cannot identify mode\n");
1035 0 : ret = FALSE;
1036 : }
1037 :
1038 261 : g_free (filter_name);
1039 261 : return ret;
1040 : }
1041 :
1042 : /**
1043 : * @brief Set property (gst element vmethod)
1044 : */
1045 : static void
1046 578 : gst_tensor_transform_set_property (GObject * object, guint prop_id,
1047 : const GValue * value, GParamSpec * pspec)
1048 : {
1049 578 : GstTensorTransform *filter = GST_TENSOR_TRANSFORM (object);
1050 :
1051 578 : switch (prop_id) {
1052 1 : case PROP_SILENT:
1053 1 : filter->silent = g_value_get_boolean (value);
1054 1 : break;
1055 256 : case PROP_MODE:
1056 256 : filter->mode = g_value_get_enum (value);
1057 256 : gst_tensor_transform_set_option_data (filter);
1058 256 : break;
1059 260 : case PROP_OPTION:
1060 : {
1061 260 : gchar *backup_option = filter->option;
1062 260 : filter->option = g_value_dup_string (value);
1063 260 : if (gst_tensor_transform_set_option_data (filter)) {
1064 229 : silent_debug (filter, "Option = %s --> %s\n", backup_option,
1065 : filter->option);
1066 229 : g_free (backup_option);
1067 : } else {
1068 : /* ERROR! Revert the change! */
1069 31 : g_free (filter->option);
1070 31 : filter->option = backup_option;
1071 31 : gst_tensor_transform_set_option_data (filter);
1072 : }
1073 260 : break;
1074 : }
1075 58 : case PROP_ACCELERATION:
1076 : #ifdef HAVE_ORC
1077 58 : filter->acceleration = g_value_get_boolean (value);
1078 58 : silent_debug (filter, "acceleration = %d\n", filter->acceleration);
1079 : #else
1080 : GST_WARNING_OBJECT (filter, "Orc acceleration is not supported");
1081 : filter->acceleration = FALSE;
1082 : #endif
1083 58 : break;
1084 3 : case PROP_APPLY:
1085 : {
1086 : gint64 val;
1087 3 : const gchar *param = g_value_get_string (value);
1088 3 : gchar **strv = g_strsplit_set (param, ",", -1);
1089 3 : guint i, num = g_strv_length (strv);
1090 3 : gchar *endptr = NULL;
1091 :
1092 10 : for (i = 0; i < num; i++) {
1093 7 : errno = 0;
1094 7 : val = g_ascii_strtoll (strv[i], &endptr, 10);
1095 7 : if (errno == ERANGE || errno == EINVAL || (endptr == strv[i])) {
1096 0 : ml_loge ("Cannot convert string %s to a gint64 value", strv[i]);
1097 : }
1098 7 : filter->apply = g_list_append (filter->apply, GINT_TO_POINTER (val));
1099 : }
1100 3 : g_strfreev (strv);
1101 3 : break;
1102 : }
1103 0 : default:
1104 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1105 0 : break;
1106 : }
1107 578 : }
1108 :
1109 : /**
1110 : * @brief Get property (gst element vmethod)
1111 : */
1112 : static void
1113 41 : gst_tensor_transform_get_property (GObject * object, guint prop_id,
1114 : GValue * value, GParamSpec * pspec)
1115 : {
1116 41 : GstTensorTransform *filter = GST_TENSOR_TRANSFORM (object);
1117 :
1118 41 : switch (prop_id) {
1119 2 : case PROP_SILENT:
1120 2 : g_value_set_boolean (value, filter->silent);
1121 2 : break;
1122 1 : case PROP_MODE:
1123 1 : g_value_set_enum (value, filter->mode);
1124 1 : break;
1125 36 : case PROP_OPTION:
1126 36 : g_value_set_string (value, filter->option);
1127 36 : break;
1128 2 : case PROP_ACCELERATION:
1129 2 : g_value_set_boolean (value, filter->acceleration);
1130 2 : break;
1131 0 : case PROP_APPLY:
1132 : {
1133 : GList *list;
1134 : gchar *p;
1135 : GPtrArray *arr;
1136 : gchar **strings;
1137 :
1138 0 : if (filter->apply == NULL) {
1139 0 : g_value_set_string (value, "");
1140 0 : return;
1141 : }
1142 :
1143 0 : arr = g_ptr_array_new ();
1144 0 : for (list = filter->apply; list != NULL; list = list->next) {
1145 0 : g_ptr_array_add (arr, g_strdup_printf ("%i",
1146 0 : GPOINTER_TO_INT (list->data)));
1147 : }
1148 0 : g_ptr_array_add (arr, NULL);
1149 0 : strings = (gchar **) g_ptr_array_free (arr, FALSE);
1150 0 : p = g_strjoinv (",", strings);
1151 :
1152 0 : g_strfreev (strings);
1153 0 : g_value_take_string (value, p);
1154 0 : break;
1155 : }
1156 0 : case PROP_TRANSPOSE_RANK_LIMIT:
1157 0 : g_value_set_uint (value, NNS_TENSOR_TRANSPOSE_RANK_LIMIT);
1158 0 : break;
1159 0 : default:
1160 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1161 0 : break;
1162 : }
1163 : }
1164 :
1165 : /**
1166 : * @brief Function to finalize instance (gst element vmethod)
1167 : */
1168 : static void
1169 240 : gst_tensor_transform_finalize (GObject * object)
1170 : {
1171 : GstTensorTransform *filter;
1172 :
1173 240 : filter = GST_TENSOR_TRANSFORM (object);
1174 :
1175 240 : if (filter->option) {
1176 201 : g_free (filter->option);
1177 201 : filter->option = NULL;
1178 : }
1179 :
1180 240 : if (filter->operators) {
1181 61 : g_slist_free_full (filter->operators, g_free);
1182 61 : filter->operators = NULL;
1183 : }
1184 :
1185 240 : if (filter->apply) {
1186 3 : g_list_free (filter->apply);
1187 3 : filter->apply = NULL;
1188 : }
1189 :
1190 240 : G_OBJECT_CLASS (parent_class)->finalize (object);
1191 240 : }
1192 :
1193 : /**
1194 : * @brief subrouting for tensor-transform, "dimchg" case.
1195 : * @param[in/out] filter "this" pointer
1196 : * @param[in] in_info input tensor info
1197 : * @param[in] out_info output tensor info
1198 : * @param[in] inptr input tensor
1199 : * @param[out] outptr output tensor
1200 : * @return Gst flow status
1201 : */
1202 : static GstFlowReturn
1203 52 : gst_tensor_transform_dimchg (GstTensorTransform * filter,
1204 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1205 : const uint8_t * inptr, uint8_t * outptr)
1206 : {
1207 52 : uint32_t *fromDim = in_info->dimension;
1208 52 : uint32_t *toDim = out_info->dimension;
1209 52 : unsigned int from = filter->data_dimchg.from;
1210 52 : unsigned int to = filter->data_dimchg.to;
1211 : unsigned int i, j, k;
1212 52 : unsigned int loopLimit = 1;
1213 : gsize loopBlockSize, copyblocksize, copyblocklimit;
1214 :
1215 52 : if (from == to) {
1216 : /** Useless memcpy. Do not call this or @todo do "IP" operation */
1217 0 : nns_memcpy (outptr, inptr, gst_tensor_info_get_size (in_info));
1218 0 : GST_WARNING_OBJECT (filter,
1219 : "Calling tensor_transform with high memcpy overhead WITHOUT any effects! Check your stream whether you really need tensor_transform.\n");
1220 0 : return GST_FLOW_OK;
1221 : }
1222 :
1223 52 : g_assert (from < NNS_TENSOR_RANK_LIMIT);
1224 52 : g_assert (to < NNS_TENSOR_RANK_LIMIT);
1225 52 : g_assert (fromDim[from] == toDim[to]);
1226 :
1227 52 : loopBlockSize = copyblocksize = gst_tensor_get_element_size (in_info->type);
1228 52 : copyblocklimit = 1;
1229 :
1230 52 : if (from < to) {
1231 : /**
1232 : * Smaller-loop-ed a to larger-loop-ed b
1233 : * E.g., [N][H][W][c] (c:W:H:N) --> [N][c][H][W] (W:H:c:N)
1234 : *
1235 : * @todo CRITICAL-TODO: Optimize the performance!
1236 : */
1237 726 : for (i = NNS_TENSOR_RANK_LIMIT - 1; i > to; i--) {
1238 674 : if (toDim[i] == 0)
1239 620 : continue;
1240 54 : loopLimit *= toDim[i];
1241 : }
1242 :
1243 158 : for (i = 0; i < to; i++) {
1244 106 : if (toDim[i] == 0)
1245 0 : break;
1246 106 : loopBlockSize *= toDim[i];
1247 : }
1248 :
1249 52 : for (i = 0; i < from; i++) {
1250 0 : if (fromDim[i] == 0)
1251 0 : break;
1252 0 : copyblocksize *= fromDim[i];
1253 : }
1254 158 : for (i = 0; i < to; i++) {
1255 106 : if (toDim[i] == 0)
1256 0 : break;
1257 106 : copyblocklimit *= toDim[i];
1258 : }
1259 :
1260 228 : for (i = 0; i < loopLimit; i++) {
1261 : /* [i1][i2][...][iN][b][...] i = i1 x i2 x ... x iN */
1262 176 : uint8_t *destptr = outptr + loopBlockSize * toDim[to] * i;
1263 176 : const uint8_t *srcptr = inptr + loopBlockSize * toDim[to] * i;
1264 :
1265 744 : for (j = 0; j < toDim[to]; j++) {
1266 568 : uint8_t *j_destptr = destptr + loopBlockSize * j;
1267 121208 : for (k = 0; k < copyblocklimit; k++) {
1268 120640 : nns_memcpy (j_destptr + copyblocksize * k,
1269 : srcptr + k * copyblocksize * toDim[to] + j * copyblocksize,
1270 : copyblocksize);
1271 : }
1272 : }
1273 : }
1274 : } else {
1275 : /**
1276 : * Larger-loop-ed a to smaller-loop-ed b
1277 : * E.g., [N][c][H][W] (W:H:c:N) --> [N][H][W][c] (c:W:H:N)
1278 : * @todo NYI
1279 : */
1280 0 : ml_loge
1281 : ("tensor-transform/dimchg operation is not permitted if from >= to.\n");
1282 0 : return GST_FLOW_ERROR;
1283 : }
1284 :
1285 52 : return GST_FLOW_OK;
1286 : }
1287 :
1288 : /**
1289 : * @brief subrouting for tensor-transform, "typecast" case.
1290 : * @param[in/out] filter "this" pointer
1291 : * @param[in] in_info input tensor info
1292 : * @param[in] out_info output tensor info
1293 : * @param[in] inptr input tensor
1294 : * @param[out] outptr output tensor
1295 : * @return Gst flow status
1296 : */
1297 : static GstFlowReturn
1298 588 : gst_tensor_transform_typecast (GstTensorTransform * filter,
1299 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1300 : const uint8_t * inptr, uint8_t * outptr)
1301 : {
1302 : gulong i, num;
1303 : gsize in_element_size, out_element_size;
1304 :
1305 588 : num = gst_tensor_get_element_count (in_info->dimension);
1306 :
1307 : #ifdef HAVE_ORC
1308 588 : if (filter->acceleration) {
1309 466 : orc_typecast (inptr, outptr, num, in_info->type, out_info->type);
1310 466 : return GST_FLOW_OK;
1311 : }
1312 : #endif
1313 :
1314 122 : in_element_size = gst_tensor_get_element_size (in_info->type);
1315 122 : out_element_size = gst_tensor_get_element_size (out_info->type);
1316 :
1317 61772 : for (i = 0; i < num; ++i) {
1318 61650 : gst_tensor_data_raw_typecast (
1319 61650 : (gpointer) (inptr + in_element_size * i), in_info->type,
1320 61650 : (gpointer) (outptr + out_element_size * i), out_info->type);
1321 : }
1322 :
1323 122 : return GST_FLOW_OK;
1324 : }
1325 :
1326 : /**
1327 : * @brief subrouting for tensor-transform, "arithmetic" case.
1328 : * @param[in/out] filter "this" pointer
1329 : * @param[in] in_info input tensor info
1330 : * @param[in] out_info output tensor info
1331 : * @param[in] inptr input tensor
1332 : * @param[out] outptr output tensor
1333 : * @return Gst flow status
1334 : */
1335 : static GstFlowReturn
1336 21452 : gst_tensor_transform_arithmetic (GstTensorTransform * filter,
1337 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1338 : const uint8_t * inptr, uint8_t * outptr)
1339 : {
1340 : gulong i, num, j, ch;
1341 : gsize in_element_size, out_element_size;
1342 :
1343 : GSList *walk;
1344 : tensor_transform_operator_s *op_s;
1345 : tensor_data_s value;
1346 :
1347 21452 : num = gst_tensor_get_element_count (in_info->dimension);
1348 :
1349 : #ifdef HAVE_ORC
1350 21452 : if (filter->acceleration) {
1351 21352 : walk = filter->operators;
1352 : /**
1353 : * Typecast should be called at the first.
1354 : * Do the typecast. If in/out type is same, this will copy the input array to output.
1355 : */
1356 21352 : orc_typecast (inptr, outptr, num, in_info->type, out_info->type);
1357 :
1358 21352 : if (!filter->data_arithmetic.per_channel_arith) {
1359 65251 : while (walk) {
1360 43900 : op_s = (tensor_transform_operator_s *) walk->data;
1361 :
1362 43900 : if (op_s->op != GTT_OP_TYPECAST) {
1363 22627 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1364 23655 : orc_operator (outptr, num, &op_s->value, op_s->op);
1365 : }
1366 :
1367 43900 : walk = g_slist_next (walk);
1368 : }
1369 : } else {
1370 1 : gsize typesize = 0;
1371 1 : guint ch_dim = filter->data_arithmetic.ch_dim;
1372 1 : gsize ch_offset, ch_size = 1;
1373 1 : uint8_t *tmp_outptr = NULL;
1374 :
1375 1 : for (i = 0; i < ch_dim; ++i) {
1376 0 : ch_size *= in_info->dimension[i];
1377 : }
1378 1 : ch_offset = ch_size * in_info->dimension[ch_dim];
1379 1 : orc_typesize (typesize, out_info->type);
1380 :
1381 2 : while (walk) {
1382 1 : op_s = (tensor_transform_operator_s *) walk->data;
1383 1 : if (op_s->op == GTT_OP_TYPECAST) {
1384 0 : walk = g_slist_next (walk);
1385 0 : continue;
1386 : }
1387 :
1388 1 : if (op_s->applying_ch == -1) {
1389 0 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1390 0 : orc_operator (outptr, num, &op_s->value, op_s->op);
1391 : } else {
1392 33 : for (i = 0; i < num / ch_offset; ++i) {
1393 32 : tmp_outptr =
1394 32 : outptr + (ch_size * op_s->applying_ch +
1395 32 : ch_offset * i) * typesize;
1396 32 : gst_tensor_data_typecast (&op_s->value, out_info->type);
1397 32 : orc_operator (tmp_outptr, ch_size, &op_s->value, op_s->op);
1398 : }
1399 : }
1400 1 : walk = g_slist_next (walk);
1401 : }
1402 : }
1403 21452 : return GST_FLOW_OK;
1404 : }
1405 : #endif
1406 :
1407 100 : in_element_size = gst_tensor_get_element_size (in_info->type);
1408 100 : out_element_size = gst_tensor_get_element_size (out_info->type);
1409 :
1410 : /* per-channel */
1411 100 : if (filter->data_arithmetic.per_channel_arith) {
1412 3 : guint ch_dim = filter->data_arithmetic.ch_dim;
1413 3 : gsize ch_offset, ch_size = 1;
1414 3 : for (i = 0; i < ch_dim; ++i) {
1415 0 : ch_size *= in_info->dimension[i];
1416 : }
1417 3 : ch_offset = ch_size * in_info->dimension[ch_dim];
1418 :
1419 : /** In case of 3:4:4:1,
1420 : * ch_dim:0 -> #ch: 3, ch_size: 1, ch_offset: 3
1421 : * ch_dim:1 -> #ch: 4, ch_size: 3, ch_offset: 12
1422 : * ch_dim:2 -> #ch: 4, ch_size: 12, ch_offset: 48
1423 : * ch_dim:3 -> #ch: 1, ch_size: 48, ch_offset: 48 * 4
1424 : */
1425 :
1426 6 : for (i = 0; i < num / ch_offset; ++i) {
1427 18 : for (ch = 0; ch < in_info->dimension[ch_dim]; ++ch) {
1428 30 : for (j = 0; j < ch_size; ++j) {
1429 15 : gulong data_idx = (i * ch_offset) + (ch * ch_size) + j;
1430 15 : gst_tensor_data_set (&value, in_info->type,
1431 15 : (gpointer) (inptr + in_element_size * data_idx));
1432 :
1433 15 : walk = filter->operators;
1434 180 : while (walk) {
1435 165 : op_s = (tensor_transform_operator_s *) walk->data;
1436 165 : switch (op_s->op) {
1437 15 : case GTT_OP_TYPECAST:
1438 15 : gst_tensor_data_typecast (&value, op_s->value.type);
1439 15 : break;
1440 150 : case GTT_OP_ADD:
1441 : case GTT_OP_MUL:
1442 : case GTT_OP_DIV:
1443 : {
1444 150 : gst_tensor_data_typecast (&op_s->value, value.type);
1445 :
1446 150 : if (op_s->applying_ch == (int) ch || op_s->applying_ch == -1) {
1447 30 : gst_tensor_transform_do_operator (filter, &value,
1448 30 : &op_s->value, op_s->op);
1449 : }
1450 150 : break;
1451 : }
1452 0 : default:
1453 0 : g_assert (0);
1454 : return GST_FLOW_ERROR;
1455 : }
1456 :
1457 165 : walk = g_slist_next (walk);
1458 : }
1459 :
1460 : /* set output value */
1461 15 : g_assert (out_info->type == value.type);
1462 15 : gst_tensor_data_get (&value, outptr + out_element_size * data_idx);
1463 : }
1464 : }
1465 : }
1466 :
1467 3 : return GST_FLOW_OK;
1468 : }
1469 :
1470 61622 : for (i = 0; i < num; ++i) {
1471 : /* init value with input tensor type */
1472 61525 : gst_tensor_data_set (&value, in_info->type,
1473 61525 : (gpointer) (inptr + in_element_size * i));
1474 :
1475 61525 : walk = filter->operators;
1476 184595 : while (walk) {
1477 123070 : op_s = (tensor_transform_operator_s *) walk->data;
1478 :
1479 : /**
1480 : * @todo add more options
1481 : */
1482 123070 : switch (op_s->op) {
1483 61485 : case GTT_OP_TYPECAST:
1484 61485 : gst_tensor_data_typecast (&value, op_s->value.type);
1485 61485 : break;
1486 61585 : case GTT_OP_ADD:
1487 : case GTT_OP_MUL:
1488 : case GTT_OP_DIV:
1489 61585 : gst_tensor_data_typecast (&op_s->value, value.type);
1490 61585 : gst_tensor_transform_do_operator (filter, &value, &op_s->value,
1491 : op_s->op);
1492 61585 : break;
1493 0 : default:
1494 0 : g_assert (0);
1495 : return GST_FLOW_ERROR;
1496 : }
1497 :
1498 123070 : walk = g_slist_next (walk);
1499 : }
1500 :
1501 : /* set output value */
1502 61525 : g_assert (out_info->type == value.type);
1503 61525 : gst_tensor_data_get (&value, outptr + out_element_size * i);
1504 : }
1505 :
1506 97 : return GST_FLOW_OK;
1507 : }
1508 :
1509 : /**
1510 : * Macro to run loop for various data types with transpose
1511 : */
1512 : #define transposeloop(cl,ck,cj,ci,sl,sk,sj,si,typesize) do { \
1513 : size_t i, j, k, l; \
1514 : int inidx = 0, outidx=0; \
1515 : for(cl=0;cl<sl;cl++) \
1516 : for(ci=0;ci<si;ci++) \
1517 : for(cj=0;cj<sj;cj++) \
1518 : for(ck=0;ck<sk;ck++){ \
1519 : const uint8_t *_in; \
1520 : uint8_t *_out; \
1521 : outidx = si*sj*sk*cl + sj*sk*ci + sk*cj + ck; \
1522 : inidx = SK*SJ*SI*l + SJ*SI*k + SI*j + i; \
1523 : _in = inptr + inidx * typesize; \
1524 : _out = outptr + outidx * typesize; \
1525 : nns_memcpy(_out, _in, typesize); \
1526 : } \
1527 : } while(0);
1528 :
1529 : /**
1530 : * @brief subrouting for tensor-transform, "transpose" case.
1531 : * @param[in/out] filter "this" pointer
1532 : * @param[in] in_info input tensor info
1533 : * @param[in] out_info output tensor info
1534 : * @param[in] inptr input tensor
1535 : * @param[out] outptr output tensor
1536 : * @return Gst flow status
1537 : */
1538 : static GstFlowReturn
1539 58 : gst_tensor_transform_transpose (GstTensorTransform * filter,
1540 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1541 : const uint8_t * inptr, uint8_t * outptr)
1542 : {
1543 : int i, from, to;
1544 58 : gboolean checkdim = FALSE;
1545 58 : uint32_t *fromDim = in_info->dimension;
1546 58 : gsize type_size = gst_tensor_get_element_size (in_info->type);
1547 : gsize indexI, indexJ, SL, SI, SJ, SK;
1548 : UNUSED (out_info);
1549 :
1550 58 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
1551 58 : from = i;
1552 58 : to = filter->data_transpose.trans_order[i];
1553 58 : if (from != to) {
1554 58 : checkdim = TRUE;
1555 58 : break;
1556 : }
1557 : }
1558 :
1559 58 : if (!checkdim) {
1560 0 : nns_memcpy (outptr, inptr, gst_tensor_info_get_size (in_info));
1561 0 : GST_WARNING_OBJECT (filter,
1562 : "Calling tensor_transform with high memcpy overhead WITHOUT any effects!");
1563 0 : return GST_FLOW_OK;
1564 : }
1565 :
1566 58 : indexI = filter->data_transpose.trans_order[0];
1567 58 : indexJ = filter->data_transpose.trans_order[1];
1568 58 : SL = fromDim[3] > 0 ? fromDim[3] : 1;
1569 58 : SI = fromDim[0] > 0 ? fromDim[0] : 1;
1570 58 : SJ = fromDim[1] > 0 ? fromDim[1] : 1;
1571 58 : SK = fromDim[2] > 0 ? fromDim[2] : 1;
1572 :
1573 58 : switch (indexI) {
1574 0 : case 0:
1575 0 : if (indexJ == 1) {
1576 0 : transposeloop (l, i, j, k, SL, SI, SJ, SK, type_size);
1577 : } else {
1578 0 : transposeloop (l, i, k, j, SL, SI, SK, SJ, type_size);
1579 : }
1580 0 : break;
1581 54 : case 1:
1582 54 : if (indexJ == 0) {
1583 242420 : transposeloop (l, j, i, k, SL, SJ, SI, SK, type_size);
1584 : } else {
1585 1231478 : transposeloop (l, j, k, i, SL, SJ, SK, SI, type_size);
1586 : }
1587 54 : break;
1588 4 : case 2:
1589 4 : if (indexJ == 0) {
1590 140258 : transposeloop (l, k, i, j, SL, SK, SI, SJ, type_size);
1591 : } else {
1592 0 : transposeloop (l, k, j, i, SL, SK, SJ, SI, type_size);
1593 : }
1594 4 : break;
1595 : }
1596 :
1597 58 : return GST_FLOW_OK;
1598 : }
1599 :
1600 : /**
1601 : * @brief subrouting for tensor-transform, "stand" case.
1602 : * : pixel = abs((pixel - average(tensor))/(std(tensor) + val))
1603 : * @param[in/out] filter "this" pointer
1604 : * @param[in] in_info input tensor info
1605 : * @param[in] out_info output tensor info
1606 : * @param[in] inptr input tensor
1607 : * @param[out] outptr output tensor
1608 : * @return Gst flow status
1609 : */
1610 : static GstFlowReturn
1611 16 : gst_tensor_transform_stand (GstTensorTransform * filter,
1612 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1613 : const uint8_t * inptr, uint8_t * outptr)
1614 : {
1615 16 : GstFlowReturn ret = GST_FLOW_OK;
1616 : gsize in_element_size, out_element_size, data_size, ch_size;
1617 : gulong i, num, data_idx, ch;
1618 : gdouble tmp, *average, *std;
1619 :
1620 16 : in_element_size = gst_tensor_get_element_size (in_info->type);
1621 16 : out_element_size = gst_tensor_get_element_size (out_info->type);
1622 16 : num = gst_tensor_get_element_count (in_info->dimension);
1623 :
1624 16 : data_size = gst_tensor_info_get_size (in_info);
1625 16 : ch_size = in_info->dimension[0];
1626 :
1627 : /* calc average and std */
1628 16 : average = std = NULL;
1629 16 : if (filter->data_stand.per_channel) {
1630 8 : gst_tensor_data_raw_average_per_channel ((gpointer) inptr, data_size,
1631 8 : in_info->type, in_info->dimension, &average);
1632 : /* calculate std only for default mode */
1633 8 : if (filter->data_stand.mode == STAND_DEFAULT)
1634 4 : gst_tensor_data_raw_std_per_channel ((gpointer) inptr, data_size,
1635 4 : in_info->type, in_info->dimension, average, &std);
1636 : } else {
1637 8 : gst_tensor_data_raw_average ((gpointer) inptr, data_size,
1638 : in_info->type, &average);
1639 : /* calculate std only for default mode */
1640 8 : if (filter->data_stand.mode == STAND_DEFAULT)
1641 4 : gst_tensor_data_raw_std ((gpointer) inptr, data_size, in_info->type,
1642 : average, &std);
1643 : }
1644 :
1645 16 : switch (filter->data_stand.mode) {
1646 8 : case STAND_DEFAULT:
1647 : {
1648 8 : if (!filter->data_stand.per_channel) {
1649 20004 : for (i = 0; i < num; i++) {
1650 20000 : data_idx = in_element_size * i;
1651 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1652 : in_info->type, &tmp, _NNS_FLOAT64);
1653 :
1654 20000 : tmp = fabs ((tmp - *average) / *std);
1655 :
1656 20000 : data_idx = out_element_size * i;
1657 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1658 20000 : (gpointer) (outptr + data_idx), out_info->type);
1659 : }
1660 : } else {
1661 204 : for (ch = 0; ch < ch_size; ++ch) {
1662 20200 : for (i = 0; i < num / ch_size; i++) {
1663 20000 : data_idx = in_element_size * ((i * ch_size) + ch);
1664 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1665 : in_info->type, &tmp, _NNS_FLOAT64);
1666 :
1667 20000 : tmp = fabs ((tmp - average[ch]) / std[ch]);
1668 :
1669 20000 : data_idx = out_element_size * ((i * ch_size) + ch);
1670 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1671 20000 : (gpointer) (outptr + data_idx), out_info->type);
1672 : }
1673 : }
1674 : }
1675 8 : break;
1676 : }
1677 8 : case STAND_DC_AVERAGE:
1678 : {
1679 8 : if (!filter->data_stand.per_channel) {
1680 20004 : for (i = 0; i < num; i++) {
1681 20000 : data_idx = in_element_size * i;
1682 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1683 : in_info->type, &tmp, _NNS_FLOAT64);
1684 :
1685 20000 : tmp -= *average;
1686 :
1687 20000 : data_idx = out_element_size * i;
1688 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1689 20000 : (gpointer) (outptr + data_idx), out_info->type);
1690 : }
1691 : } else {
1692 204 : for (ch = 0; ch < ch_size; ++ch) {
1693 20200 : for (i = 0; i < num / ch_size; i++) {
1694 20000 : data_idx = in_element_size * ((i * ch_size) + ch);
1695 20000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx),
1696 : in_info->type, &tmp, _NNS_FLOAT64);
1697 :
1698 20000 : tmp -= average[ch];
1699 :
1700 20000 : data_idx = out_element_size * ((i * ch_size) + ch);
1701 20000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64,
1702 20000 : (gpointer) (outptr + data_idx), out_info->type);
1703 : }
1704 : }
1705 : }
1706 8 : break;
1707 : }
1708 0 : default:
1709 0 : GST_ERROR_OBJECT (filter, "Cannot identify mode\n");
1710 0 : ret = GST_FLOW_ERROR;
1711 : }
1712 :
1713 16 : g_free (average);
1714 16 : g_free (std);
1715 :
1716 16 : return ret;
1717 : }
1718 :
1719 : /**
1720 : * @brief subrouting for tensor-transform, "clamp" case.
1721 : * : pixel = if (pixel > max) ? max :
1722 : * if (pixel < min) ? min : pixel
1723 : * @param[in/out] filter "this" pointer
1724 : * @param[in] in_info input tensor info
1725 : * @param[in] out_info output tensor info
1726 : * @param[in] inptr input tensor
1727 : * @param[out] outptr output tensor
1728 : * @return Gst flow status
1729 : */
1730 : static GstFlowReturn
1731 16 : gst_tensor_transform_clamp (GstTensorTransform * filter,
1732 : GstTensorInfo * in_info, GstTensorInfo * out_info,
1733 : const uint8_t * inptr, uint8_t * outptr)
1734 : {
1735 : gsize in_element_size, out_element_size;
1736 : gulong i, num, data_idx;
1737 : gdouble tmp;
1738 :
1739 16 : in_element_size = gst_tensor_get_element_size (in_info->type);
1740 16 : out_element_size = gst_tensor_get_element_size (out_info->type);
1741 16 : num = gst_tensor_get_element_count (in_info->dimension);
1742 :
1743 606016 : for (i = 0; i < num; ++i) {
1744 606000 : data_idx = in_element_size * i;
1745 606000 : gst_tensor_data_raw_typecast ((gpointer) (inptr + data_idx), in_info->type,
1746 : &tmp, _NNS_FLOAT64);
1747 :
1748 606000 : tmp = CLAMP (tmp, filter->data_clamp.min, filter->data_clamp.max);
1749 :
1750 606000 : data_idx = out_element_size * i;
1751 606000 : gst_tensor_data_raw_typecast (&tmp, _NNS_FLOAT64, outptr + data_idx,
1752 : out_info->type);
1753 : }
1754 :
1755 16 : return GST_FLOW_OK;
1756 : }
1757 :
1758 : /**
1759 : * @brief subrouting for tensor-transform, "padding" case.
1760 : * @param[in/out] filter "this" pointer
1761 : * @param[in] in_info input tensor info
1762 : * @param[in] out_info output tensor info
1763 : * @param[in] inptr input tensor
1764 : * @param[out] outptr output tensor
1765 : * @return Gst flow status
1766 : */
1767 : static GstFlowReturn
1768 6 : gst_tensor_transform_padding (GstTensorTransform * filter,
1769 : GstTensorInfo * in_info, GstTensorInfo * out_info, const uint8_t * inptr,
1770 : uint8_t * outptr)
1771 : {
1772 : gsize element_size, in_loop_size, out_loop_size, copy_block_size;
1773 6 : guint i, j, k, left, top, front, loop_limit = 1;
1774 6 : element_size = gst_tensor_get_element_size (in_info->type);
1775 :
1776 6 : in_loop_size = (gsize) in_info->dimension[2] * in_info->dimension[1]
1777 6 : * in_info->dimension[0] * element_size;
1778 6 : out_loop_size =(gsize) out_info->dimension[2] * out_info->dimension[1]
1779 6 : * out_info->dimension[0] * element_size;
1780 6 : copy_block_size = in_info->dimension[0] * element_size;
1781 :
1782 9 : for (i = NNS_TENSOR_PADDING_RANK_LIMIT; i < NNS_TENSOR_RANK_LIMIT; i++) {
1783 9 : if (in_info->dimension[i] == 0)
1784 6 : break;
1785 3 : loop_limit *= in_info->dimension[i];
1786 : }
1787 :
1788 6 : left = filter->data_padding.pad[PADDING_LEFT];
1789 6 : top = filter->data_padding.pad[PADDING_TOP];
1790 6 : front = filter->data_padding.pad[PADDING_FRONT];
1791 :
1792 : /** @todo Add constant option instead of using zero padding value */
1793 6 : memset (outptr, 0, out_loop_size * loop_limit);
1794 :
1795 20 : for (i = 0; i < loop_limit; i++)
1796 56 : for (j = 0; j < in_info->dimension[2]; j++)
1797 2142 : for (k = 0; k < in_info->dimension[1]; k++) {
1798 2100 : guint in_idx = j * in_info->dimension[1] * in_info->dimension[0]
1799 2100 : + k * in_info->dimension[0];
1800 2100 : guint out_idx = j * out_info->dimension[1] * out_info->dimension[0]
1801 2100 : + k * out_info->dimension[0];
1802 :
1803 2100 : out_idx += left + top * out_info->dimension[0]
1804 2100 : + front * out_info->dimension[1] * out_info->dimension[0];
1805 :
1806 2100 : memcpy (outptr + out_idx * element_size + out_loop_size * i,
1807 2100 : inptr + in_idx * element_size + in_loop_size * i, copy_block_size);
1808 : }
1809 :
1810 6 : return GST_FLOW_OK;
1811 : }
1812 :
1813 : /**
1814 : * @brief non-ip transform. required vmethod for BaseTransform class.
1815 : * @param[in/out] trans "super" pointer
1816 : * @param[in] inbuf The input gst buffer
1817 : * @param[out] outbuf The output gst buffer
1818 : * @return Gst Flow Status
1819 : */
1820 : static GstFlowReturn
1821 22121 : gst_tensor_transform_transform (GstBaseTransform * trans,
1822 : GstBuffer * inbuf, GstBuffer * outbuf)
1823 : {
1824 : GstTensorTransform *filter;
1825 : GstTensorInfo *in_info, *out_info;
1826 22121 : GstFlowReturn res = GST_FLOW_ERROR;
1827 22121 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT] = { 0, };
1828 22121 : GstMemory *out_mem[NNS_TENSOR_SIZE_LIMIT] = { 0, };
1829 : GstMapInfo in_map[NNS_TENSOR_SIZE_LIMIT];
1830 : GstMapInfo out_map[NNS_TENSOR_SIZE_LIMIT];
1831 : uint8_t *inptr, *outptr;
1832 : guint i, num_tensors, num_mems;
1833 : gsize buf_size, hsize;
1834 : GstTensorMetaInfo meta;
1835 : GstTensorInfo in_flex_info, out_flex_info;
1836 : gboolean in_flexible, out_flexible;
1837 :
1838 22121 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
1839 :
1840 44238 : g_return_val_if_fail (filter->loaded, GST_FLOW_ERROR);
1841 22117 : inbuf = gst_tensor_buffer_from_config (inbuf, &filter->in_config);
1842 :
1843 22117 : in_flexible =
1844 22117 : gst_tensor_pad_caps_is_flexible (GST_BASE_TRANSFORM_SINK_PAD (trans));
1845 22117 : out_flexible =
1846 22117 : gst_tensor_pad_caps_is_flexible (GST_BASE_TRANSFORM_SRC_PAD (trans));
1847 :
1848 22117 : num_mems = gst_tensor_buffer_get_count (inbuf);
1849 22117 : if (in_flexible) {
1850 11 : num_tensors = num_mems;
1851 11 : g_return_val_if_fail (out_flexible, GST_FLOW_ERROR);
1852 : } else {
1853 22106 : num_tensors = filter->in_config.info.num_tensors;
1854 22106 : g_return_val_if_fail (num_mems == num_tensors, GST_FLOW_ERROR);
1855 : }
1856 :
1857 44320 : for (i = 0; i < num_tensors; i++) {
1858 22203 : in_info = gst_tensors_info_get_nth_info (&filter->in_config.info, i);
1859 22203 : out_info = gst_tensors_info_get_nth_info (&filter->out_config.info, i);
1860 :
1861 22203 : if (filter->apply && !g_list_find (filter->apply, GINT_TO_POINTER (i))) {
1862 15 : GstMemory *mem = gst_tensor_buffer_get_nth_memory (inbuf, i);
1863 :
1864 15 : if (!in_flexible && out_flexible) {
1865 0 : GstMemory *old = mem;
1866 :
1867 : /* append meta */
1868 0 : gst_tensor_info_convert_to_meta (out_info, &meta);
1869 0 : mem = gst_tensor_meta_info_append_header (&meta, old);
1870 0 : gst_memory_unref (old);
1871 : }
1872 :
1873 15 : gst_tensor_buffer_append_memory (outbuf, mem, out_info);
1874 15 : continue;
1875 : }
1876 :
1877 : /* parse input buffer */
1878 22188 : in_mem[i] = gst_tensor_buffer_get_nth_memory (inbuf, i);
1879 22188 : if (!gst_memory_map (in_mem[i], &in_map[i], GST_MAP_READ)) {
1880 0 : ml_loge ("Cannot map input buffer to gst-buf at tensor-transform.\n");
1881 0 : res = GST_FLOW_ERROR;
1882 0 : goto done;
1883 : }
1884 22188 : inptr = in_map[i].data;
1885 :
1886 22188 : if (in_flexible) {
1887 11 : in_info = &in_flex_info;
1888 11 : out_info = &out_flex_info;
1889 :
1890 11 : gst_tensor_meta_info_parse_header (&meta, inptr);
1891 : /** @todo max rank supported in tensor-transform is 4 */
1892 11 : if (!gst_tensor_meta_info_convert (&meta, in_info)) {
1893 0 : res = GST_FLOW_ERROR;
1894 0 : goto done;
1895 : }
1896 :
1897 11 : gst_tensor_transform_convert_dimension (filter, GST_PAD_SINK,
1898 : i, in_info, out_info);
1899 :
1900 11 : hsize = gst_tensor_meta_info_get_header_size (&meta);
1901 11 : inptr += hsize;
1902 : }
1903 :
1904 : /* prepare output buffer */
1905 22188 : buf_size = gst_tensor_info_get_size (out_info);
1906 22188 : if (out_flexible) {
1907 20 : gst_tensor_info_convert_to_meta (out_info, &meta);
1908 20 : hsize = gst_tensor_meta_info_get_header_size (&meta);
1909 20 : buf_size += hsize;
1910 : }
1911 :
1912 22188 : out_mem[i] = gst_allocator_alloc (NULL, buf_size, NULL);
1913 22188 : gst_tensor_buffer_append_memory (outbuf, out_mem[i], out_info);
1914 :
1915 22188 : if (!gst_memory_map (out_mem[i], &out_map[i], GST_MAP_WRITE)) {
1916 0 : ml_loge ("Cannot map output buffer to gst-buf at tensor-transform.\n");
1917 0 : res = GST_FLOW_ERROR;
1918 0 : goto done;
1919 : }
1920 22188 : outptr = out_map[i].data;
1921 :
1922 22188 : if (out_flexible) {
1923 20 : gst_tensor_meta_info_update_header (&meta, outptr);
1924 20 : outptr += hsize;
1925 : }
1926 :
1927 22188 : switch (filter->mode) {
1928 52 : case GTT_DIMCHG:
1929 52 : res = gst_tensor_transform_dimchg (filter, in_info, out_info,
1930 : inptr, outptr);
1931 52 : break;
1932 588 : case GTT_TYPECAST:
1933 588 : res = gst_tensor_transform_typecast (filter, in_info, out_info,
1934 : inptr, outptr);
1935 588 : break;
1936 21452 : case GTT_ARITHMETIC:
1937 21452 : res = gst_tensor_transform_arithmetic (filter, in_info, out_info,
1938 : inptr, outptr);
1939 21452 : break;
1940 58 : case GTT_TRANSPOSE:
1941 58 : res = gst_tensor_transform_transpose (filter, in_info, out_info,
1942 : inptr, outptr);
1943 58 : break;
1944 16 : case GTT_STAND:
1945 16 : res = gst_tensor_transform_stand (filter, in_info, out_info,
1946 : inptr, outptr);
1947 16 : break;
1948 16 : case GTT_CLAMP:
1949 16 : res = gst_tensor_transform_clamp (filter, in_info, out_info,
1950 : inptr, outptr);
1951 16 : break;
1952 6 : case GTT_PADDING:
1953 6 : res = gst_tensor_transform_padding (filter, in_info, out_info,
1954 : inptr, outptr);
1955 6 : break;
1956 0 : default:
1957 0 : ml_loge ("Not supported tensor transform mode");
1958 0 : res = GST_FLOW_NOT_SUPPORTED;
1959 0 : goto done;
1960 : }
1961 : }
1962 :
1963 22117 : done:
1964 44320 : for (i = 0; i < num_tensors; i++) {
1965 22203 : if (in_mem[i]) {
1966 22188 : gst_memory_unmap (in_mem[i], &in_map[i]);
1967 22188 : gst_memory_unref (in_mem[i]);
1968 : }
1969 22203 : if (out_mem[i])
1970 22188 : gst_memory_unmap (out_mem[i], &out_map[i]);
1971 : }
1972 :
1973 22117 : return res;
1974 : }
1975 :
1976 : /**
1977 : * @brief Dimension conversion calculation
1978 : * @param[in] filter "this" pointer
1979 : * @param[in] direction GST_PAD_SINK if input->output conv
1980 : * @param[in] idx index of the input tensors
1981 : * @param[in] in_info tensor info structure of source tensor (input if direction is SINK)
1982 : * @param[out] out_info tensor info structure of destination tensor (output if direction is SINK)
1983 : * @return TRUE if success
1984 : */
1985 : static gboolean
1986 4358 : gst_tensor_transform_convert_dimension (GstTensorTransform * filter,
1987 : GstPadDirection direction, guint idx, const GstTensorInfo * in_info,
1988 : GstTensorInfo * out_info)
1989 : {
1990 : guint i;
1991 :
1992 : /* copy input info first, then update output info */
1993 4358 : gst_tensor_info_copy (out_info, in_info);
1994 :
1995 4358 : if (filter->apply && !g_list_find (filter->apply, GINT_TO_POINTER (idx)))
1996 10 : return TRUE;
1997 :
1998 4348 : switch (filter->mode) {
1999 70 : case GTT_DIMCHG:
2000 : {
2001 70 : unsigned int from = filter->data_dimchg.from;
2002 70 : unsigned int to = filter->data_dimchg.to;
2003 :
2004 70 : if (direction == GST_PAD_SINK) {
2005 850 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) {
2006 800 : if ((i < from && i < to) || (i > from && i > to) || from == to) {
2007 634 : out_info->dimension[i] = in_info->dimension[i];
2008 166 : } else if (i == to) {
2009 50 : out_info->dimension[i] = in_info->dimension[from];
2010 116 : } else if (from > to) {
2011 0 : g_assert (i > 0 && i > to);
2012 0 : out_info->dimension[i] = in_info->dimension[i - 1];
2013 : } else {
2014 116 : g_assert (i < to && i < (NNS_TENSOR_RANK_LIMIT - 1));
2015 116 : out_info->dimension[i] = in_info->dimension[i + 1];
2016 : }
2017 : }
2018 : } else {
2019 340 : for (i = 0; i < NNS_TENSOR_RANK_LIMIT; i++) {
2020 320 : if ((i < from && i < to) || (i > from && i > to) || from == to) {
2021 252 : out_info->dimension[i] = in_info->dimension[i];
2022 68 : } else if (i == from) {
2023 20 : out_info->dimension[i] = in_info->dimension[to];
2024 48 : } else if (from > to) {
2025 0 : g_assert (i < from && i < (NNS_TENSOR_RANK_LIMIT - 1));
2026 0 : out_info->dimension[i] = in_info->dimension[i + 1];
2027 : } else {
2028 48 : g_assert (i > 0 && i > from);
2029 48 : out_info->dimension[i] = in_info->dimension[i - 1];
2030 : }
2031 : }
2032 : }
2033 70 : break;
2034 : }
2035 2364 : case GTT_TYPECAST:
2036 : /** For both directions, dimension does not change */
2037 2364 : if (direction == GST_PAD_SINK) {
2038 : /** src = SINKPAD / dest = SRCPAD */
2039 1364 : out_info->type = filter->data_typecast.to;
2040 : } else {
2041 : /* cannot get the incoming data type on sink pad */
2042 1000 : out_info->type = _NNS_END;
2043 : }
2044 2364 : break;
2045 :
2046 1213 : case GTT_ARITHMETIC:
2047 : /* check arith mode option has typecast operator */
2048 1213 : if (filter->data_arithmetic.out_type != _NNS_END) {
2049 867 : if (direction == GST_PAD_SINK) {
2050 491 : out_info->type = filter->data_arithmetic.out_type;
2051 : } else {
2052 : /* cannot get the incoming data type on sink pad */
2053 376 : out_info->type = _NNS_END;
2054 : }
2055 : }
2056 1213 : break;
2057 :
2058 287 : case GTT_TRANSPOSE:
2059 287 : if (direction == GST_PAD_SINK) {
2060 715 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
2061 572 : out_info->dimension[i] =
2062 572 : in_info->dimension[filter->data_transpose.trans_order[i]];
2063 : }
2064 : } else {
2065 720 : for (i = 0; i < NNS_TENSOR_TRANSPOSE_RANK_LIMIT; i++) {
2066 576 : g_assert (filter->data_transpose.trans_order[i] <
2067 : NNS_TENSOR_RANK_LIMIT);
2068 576 : out_info->dimension[filter->data_transpose.trans_order[i]] =
2069 576 : in_info->dimension[i];
2070 : }
2071 : }
2072 287 : break;
2073 :
2074 40 : case GTT_STAND:
2075 : /** For both directions, dimension does not change */
2076 40 : if (direction == GST_PAD_SINK) {
2077 24 : if (filter->data_stand.out_type != _NNS_END)
2078 12 : out_info->type = filter->data_stand.out_type;
2079 : } else {
2080 : /* cannot get the incoming data type on sink pad */
2081 16 : out_info->type = _NNS_END;
2082 : }
2083 40 : break;
2084 :
2085 176 : case GTT_CLAMP:
2086 : /* same tensors info, do nothing. */
2087 176 : break;
2088 :
2089 44 : case GTT_PADDING:
2090 44 : if (direction == GST_PAD_SINK) {
2091 30 : out_info->dimension[0] +=
2092 30 : filter->data_padding.pad[PADDING_LEFT] +
2093 30 : filter->data_padding.pad[PADDING_RIGHT];
2094 30 : out_info->dimension[1] +=
2095 30 : filter->data_padding.pad[PADDING_TOP] +
2096 30 : filter->data_padding.pad[PADDING_BOTTOM];
2097 30 : out_info->dimension[2] +=
2098 30 : filter->data_padding.pad[PADDING_FRONT] +
2099 30 : filter->data_padding.pad[PADDING_BACK];
2100 : }
2101 44 : break;
2102 154 : default:
2103 154 : return FALSE;
2104 : }
2105 :
2106 4194 : return TRUE;
2107 : }
2108 :
2109 : /**
2110 : * @brief configure srcpad cap from "proposed" cap. (required vmethod for BaseTransform)
2111 : *
2112 : * @param trans ("this" pointer)
2113 : * @param direction (why do we need this?)
2114 : * @param caps sinkpad cap
2115 : * @param filtercap this element's cap (don't know specifically.)
2116 : *
2117 : * @todo Get to know what the heck is @filtercap and use it!
2118 : */
2119 : static GstCaps *
2120 3289 : gst_tensor_transform_transform_caps (GstBaseTransform * trans,
2121 : GstPadDirection direction, GstCaps * caps, GstCaps * filtercap)
2122 : {
2123 : GstTensorTransform *filter;
2124 3289 : GstCaps *result = NULL;
2125 : GstStructure *structure;
2126 : GstPad *pad;
2127 : guint i, j;
2128 :
2129 3289 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2130 :
2131 3289 : silent_debug (filter, "Calling TransformCaps, direction = %d\n", direction);
2132 3289 : silent_debug_caps (filter, caps, "from");
2133 3289 : silent_debug_caps (filter, filtercap, "filter");
2134 :
2135 3289 : if (direction == GST_PAD_SRC) {
2136 1529 : pad = GST_BASE_TRANSFORM_SINK_PAD (trans);
2137 : } else {
2138 1760 : pad = GST_BASE_TRANSFORM_SRC_PAD (trans);
2139 : }
2140 :
2141 3289 : result = gst_caps_new_empty ();
2142 10457 : for (i = 0; i < gst_caps_get_size (caps); i++) {
2143 : GstTensorsConfig in_config, out_config;
2144 : GstTensorInfo *in_info, *out_info;
2145 : GstCaps *out_caps;
2146 :
2147 7168 : structure = gst_caps_get_structure (caps, i);
2148 7168 : gst_tensors_config_from_structure (&in_config, structure);
2149 :
2150 7168 : gst_tensors_config_copy (&out_config, &in_config);
2151 :
2152 7168 : if (out_config.info.format == _NNS_TENSOR_FORMAT_END) {
2153 1511 : out_caps = gst_pad_get_pad_template_caps (pad);
2154 : } else {
2155 5657 : if (gst_tensors_config_is_static (&out_config)) {
2156 8770 : for (j = 0; j < in_config.info.num_tensors; j++) {
2157 4121 : in_info = gst_tensors_info_get_nth_info (&in_config.info, j);
2158 4121 : out_info = gst_tensors_info_get_nth_info (&out_config.info, j);
2159 :
2160 4121 : gst_tensor_transform_convert_dimension (filter, direction,
2161 : j, in_info, out_info);
2162 : }
2163 : }
2164 :
2165 5657 : out_caps = gst_tensor_pad_possible_caps_from_config (pad, &out_config);
2166 : }
2167 :
2168 7168 : if (out_caps) {
2169 7168 : result = gst_caps_merge (result, out_caps);
2170 : }
2171 :
2172 7168 : gst_tensors_config_free (&in_config);
2173 7168 : gst_tensors_config_free (&out_config);
2174 : }
2175 :
2176 3289 : if (filtercap && gst_caps_get_size (filtercap) > 0) {
2177 : GstCaps *intersection;
2178 :
2179 : intersection =
2180 690 : gst_caps_intersect_full (result, filtercap, GST_CAPS_INTERSECT_FIRST);
2181 :
2182 690 : gst_caps_unref (result);
2183 690 : result = intersection;
2184 : }
2185 :
2186 3289 : silent_debug_caps (filter, result, "to");
2187 3289 : return result;
2188 : }
2189 :
2190 : /**
2191 : * @brief fixate caps. required vmethod of BaseTransform
2192 : */
2193 : static GstCaps *
2194 238 : gst_tensor_transform_fixate_caps (GstBaseTransform * trans,
2195 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
2196 : {
2197 : GstTensorTransform *filter;
2198 : GstCaps *result;
2199 :
2200 238 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2201 :
2202 238 : silent_debug (filter, "Calling FixateCaps, direction = %d\n", direction);
2203 238 : silent_debug_caps (filter, caps, "caps");
2204 238 : silent_debug_caps (filter, othercaps, "othercaps");
2205 :
2206 : result =
2207 238 : gst_tensor_transform_transform_caps (trans, direction, caps, othercaps);
2208 238 : gst_caps_unref (othercaps);
2209 :
2210 238 : result = gst_caps_make_writable (result);
2211 238 : result = gst_caps_fixate (result);
2212 :
2213 238 : silent_debug_caps (filter, result, "result");
2214 238 : return result;
2215 : }
2216 :
2217 : /**
2218 : * @brief set caps. required vmethod of BaseTransform
2219 : */
2220 : static gboolean
2221 220 : gst_tensor_transform_set_caps (GstBaseTransform * trans,
2222 : GstCaps * incaps, GstCaps * outcaps)
2223 : {
2224 : GstTensorTransform *filter;
2225 : GstTensorsConfig in_config, out_config;
2226 : GstTensorsConfig config;
2227 : GstTensorInfo *in_info, *out_info;
2228 : gboolean in_flexible, out_flexible;
2229 220 : gboolean allowed = FALSE;
2230 : guint i;
2231 :
2232 220 : filter = GST_TENSOR_TRANSFORM_CAST (trans);
2233 :
2234 220 : silent_debug (filter, "Calling SetCaps\n");
2235 220 : silent_debug_caps (filter, incaps, "incaps");
2236 220 : silent_debug_caps (filter, outcaps, "outcaps");
2237 :
2238 220 : if (!gst_tensors_config_from_caps (&in_config, incaps, TRUE)) {
2239 0 : GST_ERROR_OBJECT (filter, "Cannot read cap of incaps\n");
2240 0 : goto error;
2241 : }
2242 :
2243 220 : if (!gst_tensors_config_from_caps (&out_config, outcaps, TRUE)) {
2244 0 : GST_ERROR_OBJECT (filter, "Cannot read cap of outcaps\n");
2245 0 : goto error;
2246 : }
2247 :
2248 220 : in_flexible = gst_tensors_config_is_flexible (&in_config);
2249 220 : out_flexible = gst_tensors_config_is_flexible (&out_config);
2250 :
2251 : /* compare type and dimension */
2252 220 : gst_tensors_config_init (&config);
2253 220 : config.info.format = out_config.info.format;
2254 :
2255 220 : config.rate_n = in_config.rate_n;
2256 220 : config.rate_d = in_config.rate_d;
2257 220 : config.info.num_tensors = in_config.info.num_tensors;
2258 :
2259 220 : if (!in_flexible) {
2260 436 : for (i = 0; i < in_config.info.num_tensors; i++) {
2261 226 : in_info = gst_tensors_info_get_nth_info (&in_config.info, i);
2262 226 : out_info = gst_tensors_info_get_nth_info (&config.info, i);
2263 :
2264 226 : if (!gst_tensor_transform_convert_dimension (filter, GST_PAD_SINK,
2265 : i, in_info, out_info)) {
2266 0 : GST_ERROR_OBJECT (filter,
2267 : "Tensor info is not matched with given properties.");
2268 0 : goto error;
2269 : }
2270 : }
2271 : }
2272 :
2273 220 : if (out_flexible) {
2274 15 : GST_INFO_OBJECT (filter, "Output tensor is flexible.");
2275 :
2276 : /* set output configuration if input is static */
2277 15 : if (!in_flexible)
2278 5 : out_config = config;
2279 205 : } else if (!gst_tensors_config_is_equal (&out_config, &config)) {
2280 0 : GST_ERROR_OBJECT (filter,
2281 : "Tensor info is not matched with given properties.\n");
2282 0 : goto error;
2283 : }
2284 :
2285 : /* set in/out tensor info */
2286 220 : filter->in_config = in_config;
2287 220 : filter->out_config = out_config;
2288 220 : allowed = TRUE;
2289 :
2290 220 : error:
2291 220 : if (!allowed)
2292 0 : GST_ERROR_OBJECT (filter, "Set Caps Failed!\n");
2293 :
2294 220 : return allowed;
2295 : }
2296 :
2297 : /**
2298 : * @brief Tell the framework the required size of buffer based on the info of the other side pad. Note that this is always the same with the input. optional vmethod of BaseTransform
2299 : */
2300 : static gboolean
2301 22121 : gst_tensor_transform_transform_size (GstBaseTransform * trans,
2302 : GstPadDirection direction, GstCaps * caps, gsize size, GstCaps * othercaps,
2303 : gsize * othersize)
2304 : {
2305 : UNUSED (trans);
2306 : UNUSED (direction);
2307 : UNUSED (caps);
2308 : UNUSED (size);
2309 : UNUSED (othercaps);
2310 : /**
2311 : * Consider multi-tensors.
2312 : * Set each memory block in transform()
2313 : */
2314 22121 : *othersize = 0;
2315 :
2316 22121 : return TRUE;
2317 : }
|
