LCOV - code coverage report
Current view: top level - nnstreamer-2.4.2/gst/nnstreamer/elements - gsttensor_converter.c (source / functions) Coverage Total Hit
Test: nnstreamer 2.4.2-0 nnstreamer/nnstreamer#eca68b8d050408568af95d831a8eef62aaee7784 Lines: 85.8 % 1095 939
Test Date: 2025-03-14 05:36:58 Functions: 97.6 % 41 40

            Line data    Source code
       1              : /**
       2              :  * GStreamer
       3              :  * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
       4              :  * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
       5              :  * Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>
       6              :  *
       7              :  * This library is free software; you can redistribute it and/or
       8              :  * modify it under the terms of the GNU Library General Public
       9              :  * License as published by the Free Software Foundation;
      10              :  * version 2.1 of the License.
      11              :  *
      12              :  * This library is distributed in the hope that it will be useful,
      13              :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      14              :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      15              :  * Library General Public License for more details.
      16              :  */
      17              : 
      18              : /**
      19              :  * @file        gsttensor_converter.c
      20              :  * @date        26 Mar 2018
      21              :  * @brief       GStreamer plugin to convert media types to tensors (as a filter for other general neural network filters)
      22              :  * @see         https://github.com/nnstreamer/nnstreamer
      23              :  * @author      MyungJoo Ham <myungjoo.ham@samsung.com>
      24              :  * @bug         No known bugs except for NYI items
      25              :  * @todo        For flatbuffers, support other/tensors with properties
      26              :  * @todo        Subplugins are not tested, yet.
      27              :  */
      28              : 
      29              : /**
      30              :  * SECTION:element-tensor_converter
      31              :  *
      32              :  * A filter that converts media stream to tensor stream for NN frameworks.
      33              :  * The output is always in the format of other/tensor or other/tensors.
      34              :  *
      35              :  * <refsect2>
      36              :  * <title>Example launch line</title>
      37              :  * |[
      38              :  * gst-launch-1.0 videotestsrc ! video/x-raw,format=RGB,width=640,height=480 ! tensor_converter ! tensor_sink
      39              :  * ]|
      40              :  * </refsect2>
      41              :  */
      42              : 
      43              : #ifdef HAVE_CONFIG_H
      44              : #include <config.h>
      45              : #endif
      46              : 
      47              : #include <string.h>
      48              : #include "gsttensor_converter.h"
      49              : #include "tensor_meta.h"
      50              : 
      51              : #ifdef NO_VIDEO
      52              : #include "gsttensor_converter_media_no_video.h"
      53              : #else
      54              : #include "gsttensor_converter_media_info_video.h"
      55              : #endif
      56              : 
      57              : #ifdef NO_AUDIO
      58              : #include "gsttensor_converter_media_no_audio.h"
      59              : #else
      60              : #include "gsttensor_converter_media_info_audio.h"
      61              : #endif
      62              : #include <nnstreamer_log.h>
      63              : #include <nnstreamer_subplugin.h>
      64              : #include <nnstreamer_util.h>
      65              : 
      66              : /**
      67              :  * @brief Caps string for text input
      68              :  */
      69              : #define TEXT_CAPS_STR "text/x-raw, format = (string) utf8"
      70              : 
      71              : #define append_text_caps_template(caps) \
      72              :     gst_caps_append (caps, gst_caps_from_string (TEXT_CAPS_STR))
      73              : 
      74              : /**
      75              :  * @brief Caps string for binary stream
      76              :  */
      77              : #define OCTET_CAPS_STR "application/octet-stream"
      78              : 
      79              : #define append_octet_caps_template(caps) \
      80              :     gst_caps_append (caps, gst_caps_from_string (OCTET_CAPS_STR))
      81              : 
      82              : /**
      83              :  * @brief Macro to append template caps for flexible tensor
      84              :  */
      85              : #define append_flex_tensor_caps_template(caps) \
      86              :     gst_caps_append (caps, gst_caps_from_string (GST_TENSORS_FLEX_CAP_DEFAULT))
      87              : 
      88              : /**
      89              :  * @brief Macro for debug mode.
      90              :  */
      91              : #ifndef DBG
      92              : #define DBG (!self->silent)
      93              : #endif
      94              : 
      95              : #define silent_debug_timestamp(self, buf) do { \
      96              :   if (DBG) { \
      97              :     GST_DEBUG_OBJECT (self, "pts = %" GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_PTS (buf))); \
      98              :     GST_DEBUG_OBJECT (self, "dts = %" GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_DTS (buf))); \
      99              :     GST_DEBUG_OBJECT (self, "duration = %" GST_TIME_FORMAT "\n", GST_TIME_ARGS (GST_BUFFER_DURATION (buf))); \
     100              :   } \
     101              : } while (0)
     102              : 
     103              : GST_DEBUG_CATEGORY_STATIC (gst_tensor_converter_debug);
     104              : #define GST_CAT_DEFAULT gst_tensor_converter_debug
     105              : 
     106              : #define STRING_CUSTOM_MODE(self) \
     107              :     (((self)->mode == _CONVERTER_MODE_CUSTOM_CODE) ? \
     108              :         "custom_code (function)" : \
     109              :         (((self)->mode == _CONVERTER_MODE_CUSTOM_SCRIPT) ? \
     110              :             "custom_script (py)" : \
     111              :             "unknown custom mode (internal error!)"))
     112              : 
     113              : /**
     114              :  * @brief tensor_converter properties
     115              :  * @todo For flatbuffers, support other/tensors.
     116              :  */
     117              : enum
     118              : {
     119              :   PROP_0,
     120              :   PROP_INPUT_DIMENSION,
     121              :   PROP_INPUT_TYPE,
     122              :   PROP_FRAMES_PER_TENSOR,
     123              :   PROP_SET_TIMESTAMP,
     124              :   PROP_SUBPLUGINS,
     125              :   PROP_SILENT,
     126              :   PROP_MODE
     127              : };
     128              : 
     129              : /**
     130              :  * @brief Flag to set timestamp when received a buffer with invalid timestamp.
     131              :  */
     132              : #define DEFAULT_SET_TIMESTAMP TRUE
     133              : 
     134              : /**
     135              :  * @brief Flag to print minimized log.
     136              :  */
     137              : #define DEFAULT_SILENT TRUE
     138              : 
     139              : /**
     140              :  * @brief Frames in output tensor.
     141              :  */
     142              : #define DEFAULT_FRAMES_PER_TENSOR 1
     143              : 
     144              : #define gst_tensor_converter_parent_class parent_class
     145        56915 : G_DEFINE_TYPE (GstTensorConverter, gst_tensor_converter, GST_TYPE_ELEMENT);
     146              : 
     147              : static void gst_tensor_converter_finalize (GObject * object);
     148              : static void gst_tensor_converter_set_property (GObject * object,
     149              :     guint prop_id, const GValue * value, GParamSpec * pspec);
     150              : static void gst_tensor_converter_get_property (GObject * object,
     151              :     guint prop_id, GValue * value, GParamSpec * pspec);
     152              : 
     153              : static gboolean gst_tensor_converter_sink_event (GstPad * pad,
     154              :     GstObject * parent, GstEvent * event);
     155              : static gboolean gst_tensor_converter_sink_query (GstPad * pad,
     156              :     GstObject * parent, GstQuery * query);
     157              : static gboolean gst_tensor_converter_src_query (GstPad * pad,
     158              :     GstObject * parent, GstQuery * query);
     159              : static GstFlowReturn gst_tensor_converter_chain (GstPad * pad,
     160              :     GstObject * parent, GstBuffer * buf);
     161              : static GstStateChangeReturn
     162              : gst_tensor_converter_change_state (GstElement * element,
     163              :     GstStateChange transition);
     164              : 
     165              : static void gst_tensor_converter_reset (GstTensorConverter * self);
     166              : static GstCaps *gst_tensor_converter_query_caps (GstTensorConverter * self,
     167              :     GstPad * pad, GstCaps * filter);
     168              : static gboolean gst_tensor_converter_parse_caps (GstTensorConverter * self,
     169              :     const GstCaps * caps);
     170              : static void gst_tensor_converter_update_caps (GstTensorConverter * self);
     171              : static const NNStreamerExternalConverter *findExternalConverter (const char
     172              :     *media_type_name);
     173              : 
     174              : /**
     175              :  * @brief Initialize the tensor_converter's class.
     176              :  */
     177              : static void
     178          420 : gst_tensor_converter_class_init (GstTensorConverterClass * klass)
     179              : {
     180              :   GObjectClass *object_class;
     181              :   GstElementClass *element_class;
     182              :   GstPadTemplate *pad_template;
     183              :   GstCaps *pad_caps;
     184              :   gchar **str_array;
     185              :   guint total, i;
     186              :   const NNStreamerExternalConverter *ex;
     187              : 
     188          420 :   GST_DEBUG_CATEGORY_INIT (gst_tensor_converter_debug, "tensor_converter", 0,
     189              :       "Element to convert media stream to tensor stream");
     190              : 
     191          420 :   object_class = (GObjectClass *) klass;
     192          420 :   element_class = (GstElementClass *) klass;
     193              : 
     194              :   /* GObjectClass vmethods */
     195          420 :   object_class->set_property = gst_tensor_converter_set_property;
     196          420 :   object_class->get_property = gst_tensor_converter_get_property;
     197          420 :   object_class->finalize = gst_tensor_converter_finalize;
     198              : 
     199              :   /**
     200              :    * GstTensorConverter::input-dim:
     201              :    *
     202              :    * Input tensor dimension from inner array.
     203              :    * Generally this property is used to set tensor configuration for byte-stream (application/octet-stream).
     204              :    * When setting this property and input media type is video or audio stream, GstTensorConverter will compare the media info with this.
     205              :    * (If it is different, it will be failed.)
     206              :    */
     207          420 :   g_object_class_install_property (object_class, PROP_INPUT_DIMENSION,
     208              :       g_param_spec_string ("input-dim", "Input tensor dimension",
     209              :           "Input tensor dimension from inner array", "",
     210              :           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
     211              : 
     212              :   /**
     213              :    * GstTensorConverter::input-type:
     214              :    *
     215              :    * Type of each element of the input tensor.
     216              :    * Generally this property is used to set tensor configuration for byte-stream (application/octet-stream).
     217              :    * When setting this property and input media type is video or audio stream, GstTensorConverter will compare the media info with this.
     218              :    * (If it is different, it will be failed.)
     219              :    */
     220          420 :   g_object_class_install_property (object_class, PROP_INPUT_TYPE,
     221              :       g_param_spec_string ("input-type", "Input tensor type",
     222              :           "Type of each element of the input tensor", "",
     223              :           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
     224              : 
     225              :   /**
     226              :    * GstTensorConverter::frames-per-tensor:
     227              :    *
     228              :    * The number of frames in outgoing buffer. (buffer is a single tensor instance)
     229              :    * GstTensorConverter can push a buffer with multiple media frames.
     230              :    */
     231          420 :   g_object_class_install_property (object_class, PROP_FRAMES_PER_TENSOR,
     232              :       g_param_spec_uint ("frames-per-tensor", "Frames per tensor",
     233              :           "The number of frames in output tensor", 1, G_MAXUINT,
     234              :           DEFAULT_FRAMES_PER_TENSOR,
     235              :           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
     236              : 
     237              :   /**
     238              :    * GstTensorConverter::set-timestamp:
     239              :    *
     240              :    * The flag to set timestamp when received a buffer with invalid timestamp.
     241              :    */
     242          420 :   g_object_class_install_property (object_class, PROP_SET_TIMESTAMP,
     243              :       g_param_spec_boolean ("set-timestamp", "Set timestamp",
     244              :           "The flag to set timestamp when received a buffer with invalid timestamp",
     245              :           DEFAULT_SET_TIMESTAMP, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
     246              : 
     247              :   /**
     248              :    * GstTensorConverter::sub-plugins:
     249              :    *
     250              :    * Registrable sub-plugins list of tensor-converter.
     251              :    */
     252          420 :   g_object_class_install_property (object_class, PROP_SUBPLUGINS,
     253              :       g_param_spec_string ("sub-plugins", "Sub-plugins",
     254              :           "Registrable sub-plugins list", "",
     255              :           G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
     256              : 
     257              :   /**
     258              :    * GstTensorConverter::silent:
     259              :    *
     260              :    * The flag to enable/disable debugging messages.
     261              :    */
     262          420 :   g_object_class_install_property (object_class, PROP_SILENT,
     263              :       g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
     264              :           DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
     265              : 
     266              :   /**
     267              :    * GstTensorConverter::mode:
     268              :    *
     269              :    * Generally this property is used to set tensor converter custom mode.
     270              :    */
     271          420 :   g_object_class_install_property (object_class, PROP_MODE,
     272              :       g_param_spec_string ("mode", "Mode",
     273              :           "Converter mode. e.g., mode=custom-code:<registered callback name>. For detail, refer to https://github.com/nnstreamer/nnstreamer/blob/main/gst/nnstreamer/elements/gsttensor_converter.md#custom-converter",
     274              :           "", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
     275              : 
     276              :   /* set src pad template */
     277              :   pad_caps =
     278          420 :       gst_caps_from_string (GST_TENSOR_CAP_DEFAULT ";"
     279              :       GST_TENSORS_CAP_MAKE ("{ static, flexible }"));
     280              : 
     281          420 :   pad_template = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
     282              :       pad_caps);
     283          420 :   gst_element_class_add_pad_template (element_class, pad_template);
     284              : 
     285          420 :   gst_caps_unref (pad_caps);
     286              : 
     287              :   /* set sink pad template */
     288          420 :   pad_caps = gst_caps_new_empty ();
     289              : 
     290              :   /* append caps string for all media types */
     291          420 :   append_video_caps_template (pad_caps);
     292          420 :   append_audio_caps_template (pad_caps);
     293          420 :   append_text_caps_template (pad_caps);
     294          420 :   append_octet_caps_template (pad_caps);
     295          420 :   append_flex_tensor_caps_template (pad_caps);
     296              : 
     297              :   /* append sub-plugin template caps */
     298          420 :   str_array = get_all_subplugins (NNS_SUBPLUGIN_CONVERTER);
     299          420 :   if (str_array) {
     300          420 :     total = g_strv_length (str_array);
     301              : 
     302         2100 :     for (i = 0; i < total; i++) {
     303         1680 :       ex = nnstreamer_converter_find (str_array[i]);
     304         1680 :       if (ex && ex->query_caps)
     305         1680 :         gst_caps_append (pad_caps, ex->query_caps (NULL));
     306              :     }
     307              : 
     308          420 :     g_strfreev (str_array);
     309              :   }
     310              : 
     311          420 :   pad_template = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
     312              :       pad_caps);
     313          420 :   gst_element_class_add_pad_template (element_class, pad_template);
     314              : 
     315          420 :   gst_caps_unref (pad_caps);
     316              : 
     317          420 :   gst_element_class_set_static_metadata (element_class,
     318              :       "TensorConverter",
     319              :       "Converter/Tensor",
     320              :       "Converts an audio, video, text, or arbitrary stream to a tensor stream of C-Array for neural network framework filters",
     321              :       "MyungJoo Ham <myungjoo.ham@samsung.com>");
     322              : 
     323              :   /* GstElementClass vmethods */
     324          420 :   element_class->change_state = gst_tensor_converter_change_state;
     325          420 : }
     326              : 
     327              : /**
     328              :  * @brief Initialize tensor_converter element.
     329              :  */
     330              : static void
     331          902 : gst_tensor_converter_init (GstTensorConverter * self)
     332              : {
     333              :   /** setup sink pad */
     334          902 :   self->sinkpad =
     335          902 :       gst_pad_new_from_template (gst_element_class_get_pad_template
     336          902 :       (GST_ELEMENT_GET_CLASS (self), "sink"), "sink");
     337          902 :   gst_pad_set_event_function (self->sinkpad,
     338              :       GST_DEBUG_FUNCPTR (gst_tensor_converter_sink_event));
     339          902 :   gst_pad_set_query_function (self->sinkpad,
     340              :       GST_DEBUG_FUNCPTR (gst_tensor_converter_sink_query));
     341          902 :   gst_pad_set_chain_function (self->sinkpad,
     342              :       GST_DEBUG_FUNCPTR (gst_tensor_converter_chain));
     343          902 :   GST_PAD_SET_PROXY_CAPS (self->sinkpad);
     344          902 :   gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);
     345              : 
     346              :   /** setup src pad */
     347          902 :   self->srcpad =
     348          902 :       gst_pad_new_from_template (gst_element_class_get_pad_template
     349          902 :       (GST_ELEMENT_GET_CLASS (self), "src"), "src");
     350          902 :   gst_pad_set_query_function (self->srcpad,
     351              :       GST_DEBUG_FUNCPTR (gst_tensor_converter_src_query));
     352          902 :   GST_PAD_SET_PROXY_CAPS (self->srcpad);
     353          902 :   gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
     354              : 
     355              :   /** init properties */
     356          902 :   self->silent = DEFAULT_SILENT;
     357          902 :   self->set_timestamp = DEFAULT_SET_TIMESTAMP;
     358          902 :   self->frames_per_tensor = DEFAULT_FRAMES_PER_TENSOR;
     359          902 :   self->in_media_type = _NNS_MEDIA_INVALID;
     360          902 :   self->frame_size = 0;
     361          902 :   self->remove_padding = FALSE;
     362          902 :   self->externalConverter = NULL;
     363          902 :   self->priv_data = NULL;
     364          902 :   self->mode = _CONVERTER_MODE_NONE;
     365          902 :   self->mode_option = NULL;
     366          902 :   self->custom.func = NULL;
     367          902 :   self->custom.data = NULL;
     368          902 :   self->do_not_append_header = FALSE;
     369          902 :   gst_tensors_info_init (&self->tensors_info);
     370          902 :   gst_tensors_config_init (&self->tensors_config);
     371          902 :   self->tensors_configured = FALSE;
     372              : 
     373          902 :   self->adapter_table = gst_tensor_aggregation_init ();
     374          902 :   gst_tensor_converter_reset (self);
     375          902 : }
     376              : 
     377              : /**
     378              :  * @brief Function to finalize instance.
     379              :  */
     380              : static void
     381          845 : gst_tensor_converter_finalize (GObject * object)
     382              : {
     383              :   GstTensorConverter *self;
     384              : 
     385          845 :   self = GST_TENSOR_CONVERTER (object);
     386              : 
     387          845 :   gst_tensor_converter_reset (self);
     388              : 
     389          845 :   gst_tensors_config_free (&self->tensors_config);
     390          845 :   gst_tensors_info_free (&self->tensors_info);
     391          845 :   g_hash_table_destroy (self->adapter_table);
     392              : 
     393          845 :   g_free (self->mode_option);
     394          845 :   g_free (self->ext_fw);
     395          845 :   self->custom.func = NULL;
     396          845 :   self->custom.data = NULL;
     397          845 :   if (self->externalConverter && self->externalConverter->close)
     398           12 :     self->externalConverter->close (&self->priv_data);
     399          845 :   G_OBJECT_CLASS (parent_class)->finalize (object);
     400          845 : }
     401              : 
     402              : /**
     403              :  * @brief Setter for tensor_converter properties.
     404              :  */
     405              : static void
     406          285 : gst_tensor_converter_set_property (GObject * object, guint prop_id,
     407              :     const GValue * value, GParamSpec * pspec)
     408              : {
     409              :   GstTensorConverter *self;
     410              :   GstTensorsInfo *info;
     411              :   GstTensorInfo *_info;
     412              :   guint i, j, num;
     413              :   const gchar *value_str;
     414              : 
     415          285 :   self = GST_TENSOR_CONVERTER (object);
     416          285 :   info = &self->tensors_info;
     417              : 
     418          285 :   switch (prop_id) {
     419          108 :     case PROP_INPUT_DIMENSION:
     420          108 :       value_str = g_value_get_string (value);
     421          108 :       num = gst_tensors_info_parse_dimensions_string (info, value_str);
     422              : 
     423          108 :       if (num == 0) {
     424            0 :         GST_WARNING ("%s is invalid dimension string.", value_str);
     425          108 :       } else if (info->num_tensors > 0 && info->num_tensors != num) {
     426            1 :         GST_WARNING ("%s, the number of tensor is %u.", value_str, num);
     427              :       }
     428              : 
     429              :       /* prevent invalid value, init dimensions. */
     430        27633 :       for (i = num; i < NNS_TENSOR_SIZE_LIMIT; ++i) {
     431        27525 :         _info = gst_tensors_info_get_nth_info (info, i);
     432              : 
     433       467925 :         for (j = 0; j < NNS_TENSOR_RANK_LIMIT; ++j)
     434       440400 :           _info->dimension[j] = 0;
     435              :       }
     436              : 
     437          108 :       info->num_tensors = num;
     438          108 :       break;
     439           98 :     case PROP_INPUT_TYPE:
     440           98 :       value_str = g_value_get_string (value);
     441           98 :       num = gst_tensors_info_parse_types_string (info, value_str);
     442              : 
     443           98 :       if (num == 0) {
     444            0 :         GST_WARNING ("%s is invalid type string.", value_str);
     445           98 :       } else if (info->num_tensors > 0 && info->num_tensors != num) {
     446            2 :         GST_WARNING ("%s, the number of tensor is %u.", value_str, num);
     447              :       }
     448              : 
     449              :       /* prevent invalid value, init types. */
     450        25074 :       for (i = num; i < NNS_TENSOR_SIZE_LIMIT; ++i) {
     451        24976 :         _info = gst_tensors_info_get_nth_info (info, i);
     452        24976 :         _info->type = _NNS_END;
     453              :       }
     454              : 
     455           98 :       info->num_tensors = num;
     456           98 :       break;
     457           51 :     case PROP_FRAMES_PER_TENSOR:
     458           51 :       self->frames_per_tensor = g_value_get_uint (value);
     459           51 :       silent_debug (self, "Set frames in output = %d", self->frames_per_tensor);
     460           51 :       break;
     461            2 :     case PROP_SET_TIMESTAMP:
     462            2 :       self->set_timestamp = g_value_get_boolean (value);
     463            2 :       silent_debug (self, "Set timestamp = %d", self->set_timestamp);
     464            2 :       break;
     465           12 :     case PROP_SILENT:
     466           12 :       self->silent = g_value_get_boolean (value);
     467           12 :       silent_debug (self, "Set silent = %d", self->silent);
     468           12 :       break;
     469           14 :     case PROP_MODE:
     470              :     {
     471           14 :       const gchar *param = g_value_get_string (value);
     472           14 :       const converter_custom_cb_s *ptr = NULL;
     473           14 :       gchar **strv = g_strsplit_set (param, ":", -1);
     474           14 :       self->custom.func = NULL;
     475              : 
     476           14 :       if (g_strv_length (strv) < 2) {
     477            0 :         nns_logw
     478              :             ("Tensor converter mode option is incorrect. Please specify mode option as <MODE>:<MODE_OPTION>. Refer to https://github.com/nnstreamer/nnstreamer/blob/main/gst/nnstreamer/elements/gsttensor_converter.md#custom-converter for detail.");
     479            0 :         g_strfreev (strv);
     480            0 :         break;
     481              :       }
     482              : 
     483           14 :       self->mode_option = g_strdup (strv[1]);
     484           14 :       if (g_ascii_strcasecmp (strv[0], "custom-code") == 0) {
     485            1 :         self->mode = _CONVERTER_MODE_CUSTOM_CODE;
     486            1 :         ptr = get_subplugin (NNS_CUSTOM_CONVERTER, self->mode_option);
     487            1 :         if (!ptr) {
     488            0 :           nns_logw
     489              :               ("Failed to find custom subplugin of the tensor_converter. The custom-code for tensor_converter, \"%s\" is not registered by nnstreamer_converter_custom_register() function. Refer to https://github.com/nnstreamer/nnstreamer/blob/main/gst/nnstreamer/elements/gsttensor_converter.md#custom-converter for detail.",
     490              :               strv[1]);
     491            0 :           return;
     492              :         }
     493            1 :         self->custom.func = ptr->func;
     494            1 :         self->custom.data = ptr->data;
     495           13 :       } else if (g_ascii_strcasecmp (strv[0], "custom-script") == 0) {
     496           13 :         self->mode = _CONVERTER_MODE_CUSTOM_SCRIPT;
     497              :         /** @todo detects framework based on the script extension */
     498           13 :         self->ext_fw = g_strdup ("python3");
     499              :       }
     500           14 :       g_strfreev (strv);
     501              : 
     502           14 :       break;
     503              :     }
     504            0 :     default:
     505            0 :       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     506            0 :       break;
     507              :   }
     508              : }
     509              : 
     510              : /**
     511              :  * @brief Getter for tensor_converter properties.
     512              :  */
     513              : static void
     514           37 : gst_tensor_converter_get_property (GObject * object, guint prop_id,
     515              :     GValue * value, GParamSpec * pspec)
     516              : {
     517              :   GstTensorConverter *self;
     518              :   GstTensorsInfo *info;
     519              : 
     520           37 :   self = GST_TENSOR_CONVERTER (object);
     521           37 :   info = &self->tensors_info;
     522              : 
     523           37 :   switch (prop_id) {
     524            7 :     case PROP_INPUT_DIMENSION:
     525            7 :       if (info->num_tensors > 0) {
     526            5 :         g_value_take_string (value,
     527              :             gst_tensors_info_get_dimensions_string (info));
     528              :       } else {
     529            2 :         g_value_set_string (value, "");
     530              :       }
     531            7 :       break;
     532            7 :     case PROP_INPUT_TYPE:
     533            7 :       if (info->num_tensors > 0) {
     534            5 :         g_value_take_string (value, gst_tensors_info_get_types_string (info));
     535              :       } else {
     536            2 :         g_value_set_string (value, "");
     537              :       }
     538            7 :       break;
     539            7 :     case PROP_FRAMES_PER_TENSOR:
     540            7 :       g_value_set_uint (value, self->frames_per_tensor);
     541            7 :       break;
     542            6 :     case PROP_SET_TIMESTAMP:
     543            6 :       g_value_set_boolean (value, self->set_timestamp);
     544            6 :       break;
     545            2 :     case PROP_SUBPLUGINS:
     546              :     {
     547            2 :       gchar **str_array = get_all_subplugins (NNS_SUBPLUGIN_CONVERTER);
     548              : 
     549            2 :       if (str_array) {
     550            2 :         g_value_take_string (value, g_strjoinv (",", str_array));
     551            2 :         g_strfreev (str_array);
     552              :       } else {
     553            0 :         g_value_set_string (value, "");
     554              :       }
     555            2 :       break;
     556              :     }
     557            6 :     case PROP_SILENT:
     558            6 :       g_value_set_boolean (value, self->silent);
     559            6 :       break;
     560            2 :     case PROP_MODE:
     561              :     {
     562            2 :       gchar *mode_str = NULL;
     563            2 :       if (self->mode_option == NULL)
     564            2 :         mode_str = g_strdup ("");
     565              :       else {
     566            0 :         if (self->mode == _CONVERTER_MODE_CUSTOM_CODE)
     567              :           mode_str =
     568            0 :               g_strdup_printf ("%s:%s", "custom-code", self->mode_option);
     569            0 :         else if (self->mode == _CONVERTER_MODE_CUSTOM_SCRIPT)
     570              :           mode_str =
     571            0 :               g_strdup_printf ("%s:%s", "custom-script", self->mode_option);
     572              :       }
     573            2 :       g_value_take_string (value, mode_str);
     574            2 :       break;
     575              :     }
     576            0 :     default:
     577            0 :       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     578            0 :       break;
     579              :   }
     580           37 : }
     581              : 
     582              : /**
     583              :  * @brief This function handles sink event.
     584              :  */
     585              : static gboolean
     586         3385 : gst_tensor_converter_sink_event (GstPad * pad, GstObject * parent,
     587              :     GstEvent * event)
     588              : {
     589              :   GstTensorConverter *self;
     590              : 
     591         3385 :   self = GST_TENSOR_CONVERTER (parent);
     592              : 
     593         3385 :   GST_DEBUG_OBJECT (self, "Received %s event: %" GST_PTR_FORMAT,
     594              :       GST_EVENT_TYPE_NAME (event), event);
     595              : 
     596         3385 :   switch (GST_EVENT_TYPE (event)) {
     597          864 :     case GST_EVENT_CAPS:
     598              :     {
     599              :       GstCaps *in_caps;
     600              : 
     601          864 :       gst_event_parse_caps (event, &in_caps);
     602          864 :       silent_debug_caps (self, in_caps, "in-caps");
     603              : 
     604          864 :       if (gst_tensor_converter_parse_caps (self, in_caps)) {
     605          834 :         gst_tensor_converter_update_caps (self);
     606          834 :         gst_event_unref (event);
     607          864 :         return TRUE;
     608              :       } else {
     609           30 :         gst_event_unref (event);
     610           30 :         return FALSE;
     611              :       }
     612              :       break;
     613              :     }
     614            0 :     case GST_EVENT_FLUSH_STOP:
     615            0 :       gst_tensor_converter_reset (self);
     616            0 :       break;
     617          829 :     case GST_EVENT_SEGMENT:
     618              :     {
     619              :       GstSegment seg;
     620              : 
     621          829 :       gst_event_copy_segment (event, &seg);
     622          829 :       silent_debug (self, "received seg %s", gst_format_get_name (seg.format));
     623              : 
     624          829 :       self->segment = seg;
     625          829 :       self->have_segment = TRUE;
     626              : 
     627          829 :       if (seg.format == GST_FORMAT_TIME) {
     628          829 :         return gst_pad_push_event (self->srcpad, event);
     629              :       }
     630              : 
     631          100 :       if (seg.format == GST_FORMAT_BYTES) {
     632              :         /* handle seg event in chain function */
     633          100 :         self->need_segment = TRUE;
     634          100 :         gst_event_unref (event);
     635          100 :         return TRUE;
     636              :       }
     637              : 
     638            0 :       GST_ERROR_OBJECT (self, "Unsupported format = %s\n",
     639              :           gst_format_get_name (seg.format));
     640            0 :       gst_event_unref (event);
     641            0 :       return FALSE;
     642              :     }
     643         1692 :     default:
     644         1692 :       break;
     645              :   }
     646              : 
     647         1692 :   return gst_pad_event_default (pad, parent, event);
     648              : }
     649              : 
     650              : /**
     651              :  * @brief This function handles sink pad query.
     652              :  */
     653              : static gboolean
     654         6821 : gst_tensor_converter_sink_query (GstPad * pad, GstObject * parent,
     655              :     GstQuery * query)
     656              : {
     657              :   GstTensorConverter *self;
     658              : 
     659         6821 :   self = GST_TENSOR_CONVERTER (parent);
     660         6821 :   GST_DEBUG_OBJECT (self, "Received %s query: %" GST_PTR_FORMAT,
     661              :       GST_QUERY_TYPE_NAME (query), query);
     662              : 
     663         6821 :   switch (GST_QUERY_TYPE (query)) {
     664         4178 :     case GST_QUERY_CAPS:
     665              :     {
     666              :       GstCaps *caps;
     667              :       GstCaps *filter;
     668              : 
     669         4178 :       gst_query_parse_caps (query, &filter);
     670         4178 :       caps = gst_tensor_converter_query_caps (self, pad, filter);
     671              : 
     672         4178 :       gst_query_set_caps_result (query, caps);
     673         4178 :       gst_caps_unref (caps);
     674         4178 :       return TRUE;
     675              :     }
     676         1669 :     case GST_QUERY_ACCEPT_CAPS:
     677              :     {
     678              :       GstCaps *caps;
     679              :       GstCaps *template_caps;
     680         1669 :       gboolean res = FALSE;
     681              : 
     682         1669 :       gst_query_parse_accept_caps (query, &caps);
     683         1669 :       silent_debug_caps (self, caps, "accept-caps");
     684              : 
     685         1669 :       if (gst_caps_is_fixed (caps)) {
     686         1669 :         template_caps = gst_pad_get_pad_template_caps (pad);
     687              : 
     688         1669 :         res = gst_caps_can_intersect (template_caps, caps);
     689         1669 :         gst_caps_unref (template_caps);
     690              :       }
     691              : 
     692         1669 :       gst_query_set_accept_caps_result (query, res);
     693         1669 :       return TRUE;
     694              :     }
     695          974 :     default:
     696          974 :       break;
     697              :   }
     698              : 
     699          974 :   return gst_pad_query_default (pad, parent, query);
     700              : }
     701              : 
     702              : /**
     703              :  * @brief Internal function to get adapter.
     704              :  */
     705              : static GstAdapter *
     706          121 : gst_tensor_converter_get_adapter (GstTensorConverter * self, GstBuffer * buf)
     707              : {
     708              :   GstMetaQuery *meta;
     709          121 :   guint32 key = 0;
     710              : 
     711          121 :   meta = gst_buffer_get_meta_query (buf);
     712          121 :   if (meta)
     713            0 :     key = meta->client_id;
     714              : 
     715          121 :   return gst_tensor_aggregation_get_adapter (self->adapter_table, key);
     716              : }
     717              : 
     718              : /**
     719              :  * @brief This function handles src pad query.
     720              :  */
     721              : static gboolean
     722         5896 : gst_tensor_converter_src_query (GstPad * pad, GstObject * parent,
     723              :     GstQuery * query)
     724              : {
     725              :   GstTensorConverter *self;
     726              : 
     727         5896 :   self = GST_TENSOR_CONVERTER (parent);
     728              : 
     729         5896 :   GST_DEBUG_OBJECT (self, "Received %s query: %" GST_PTR_FORMAT,
     730              :       GST_QUERY_TYPE_NAME (query), query);
     731              : 
     732         5896 :   switch (GST_QUERY_TYPE (query)) {
     733         3416 :     case GST_QUERY_CAPS:
     734              :     {
     735              :       GstCaps *caps;
     736              :       GstCaps *filter;
     737              : 
     738         3416 :       gst_query_parse_caps (query, &filter);
     739         3416 :       caps = gst_tensor_converter_query_caps (self, pad, filter);
     740              : 
     741         3416 :       gst_query_set_caps_result (query, caps);
     742         3416 :       gst_caps_unref (caps);
     743         3416 :       return TRUE;
     744              :     }
     745         2480 :     default:
     746         2480 :       break;
     747              :   }
     748              : 
     749         2480 :   return gst_pad_query_default (pad, parent, query);
     750              : }
     751              : 
     752              : /** @brief Chain function's private routine */
     753              : static void
     754        33390 : _gst_tensor_converter_chain_segment (GstTensorConverter * self,
     755              :     gsize frame_size)
     756              : {
     757        33390 :   if (self->need_segment) {
     758              :     GstTensorsConfig *config;
     759              :     GstSegment seg;
     760              :     guint64 start;
     761              :     gboolean have_framerate;
     762              : 
     763          100 :     config = &self->tensors_config;
     764          100 :     have_framerate = (config->rate_n > 0 && config->rate_d > 0);
     765              : 
     766              :     /** This is an internal logic error. */
     767          100 :     g_assert (self->have_segment);
     768          100 :     start = self->segment.start;
     769              : 
     770          100 :     gst_segment_init (&seg, GST_FORMAT_TIME);
     771              : 
     772          100 :     if (have_framerate && start > 0) {
     773            0 :       start = gst_util_uint64_scale_int (start * config->rate_d, GST_SECOND,
     774            0 :           frame_size * config->rate_n);
     775            0 :       seg.start = seg.time = start;
     776              :     }
     777              : 
     778          100 :     self->segment = seg;
     779          100 :     self->need_segment = FALSE;
     780              : 
     781          100 :     gst_pad_push_event (self->srcpad, gst_event_new_segment (&seg));
     782              :   }
     783        33390 : }
     784              : 
     785              : /** @brief Chain function's private routine */
     786              : static void
     787        33390 : _gst_tensor_converter_chain_timestamp (GstTensorConverter * self,
     788              :     GstBuffer * inbuf, guint frames_in)
     789              : {
     790        33390 :   if (self->set_timestamp) {
     791              :     GstTensorsConfig *config;
     792              :     GstClockTime pts, duration;
     793              :     gboolean have_framerate;
     794              : 
     795        33370 :     config = &self->tensors_config;
     796        33370 :     have_framerate = (config->rate_n > 0 && config->rate_d > 0);
     797              : 
     798              :     /* set duration */
     799        33370 :     duration = GST_BUFFER_DURATION (inbuf);
     800              : 
     801        33370 :     if (!GST_CLOCK_TIME_IS_VALID (duration)) {
     802          371 :       if (have_framerate) {
     803              :         duration =
     804           78 :             gst_util_uint64_scale_int ((guint64) frames_in * config->rate_d,
     805              :             GST_SECOND, config->rate_n);
     806              : 
     807           78 :         GST_BUFFER_DURATION (inbuf) = duration;
     808              :       }
     809              :     }
     810              : 
     811              :     /* set timestamp if buffer has invalid timestamp */
     812        33370 :     pts = GST_BUFFER_TIMESTAMP (inbuf);
     813              : 
     814        33370 :     if (!GST_CLOCK_TIME_IS_VALID (pts)) {
     815          194 :       pts = self->segment.start;
     816              : 
     817          194 :       if (have_framerate) {
     818           46 :         if (GST_CLOCK_TIME_IS_VALID (self->old_timestamp)) {
     819           32 :           pts = self->old_timestamp + duration;
     820              :         }
     821              :       } else {
     822              :         GstClock *clock;
     823              : 
     824          148 :         clock = gst_element_get_clock (GST_ELEMENT (self));
     825              : 
     826          148 :         if (clock) {
     827              :           GstClockTime now, base;
     828              : 
     829           70 :           base = gst_element_get_base_time (GST_ELEMENT (self));
     830           70 :           now = gst_clock_get_time (clock);
     831              : 
     832           70 :           pts = (base < now) ? (now - base) : 0;
     833           70 :           gst_object_unref (clock);
     834              :         }
     835              :       }
     836              : 
     837          194 :       GST_BUFFER_TIMESTAMP (inbuf) = pts;
     838              :     }
     839              :   }
     840              : 
     841              :   /* update old timestamp */
     842        33390 :   self->old_timestamp = GST_BUFFER_TIMESTAMP (inbuf);
     843        33390 : }
     844              : 
     845              : /** @brief Chain function's private routine to process octet stream */
     846              : static GstBuffer *
     847          234 : _gst_tensor_converter_chain_octet (GstTensorConverter * self, GstBuffer * buf)
     848              : {
     849          234 :   GstBuffer *buffer = buf;
     850          234 :   GstTensorsInfo *info = &self->tensors_config.info;
     851          234 :   gboolean multi = (info->num_tensors > 1);
     852              : 
     853              :   /* configure multi tensors */
     854          234 :   if (multi || gst_buffer_n_memory (buf) > 1) {
     855              :     GstTensorInfo *_info;
     856              :     GstMemory *mem, *new_mem;
     857              :     gsize offset, size;
     858              :     guint i;
     859              : 
     860           18 :     g_assert (self->frames_per_tensor == 1);
     861              : 
     862           18 :     offset = 0;
     863           18 :     buffer = gst_buffer_new ();
     864           18 :     mem = gst_buffer_get_all_memory (buf);
     865              : 
     866           18 :     if (multi) {
     867           53 :       for (i = 0; i < info->num_tensors; ++i) {
     868           36 :         _info = gst_tensors_info_get_nth_info (info, i);
     869              : 
     870           36 :         size = gst_tensor_info_get_size (_info);
     871           36 :         new_mem = gst_memory_share (mem, offset, size);
     872           36 :         offset += size;
     873              : 
     874           36 :         gst_tensor_buffer_append_memory (buffer, new_mem, _info);
     875              :       }
     876              : 
     877           17 :       gst_memory_unref (mem);
     878              :     } else {
     879            1 :       _info = gst_tensors_info_get_nth_info (info, 0);
     880            1 :       gst_tensor_buffer_append_memory (buffer, mem, _info);
     881              :     }
     882              : 
     883              :     /* copy timestamps */
     884           18 :     gst_buffer_copy_into (buffer, buf, GST_BUFFER_COPY_METADATA, 0, -1);
     885           18 :     gst_buffer_unref (buf);
     886              :   }
     887              : 
     888          234 :   return buffer;
     889              : }
     890              : 
     891              : /** @brief Chain function's private routine to process flex tensor */
     892              : static GstBuffer *
     893          220 : _gst_tensor_converter_chain_flex_tensor (GstTensorConverter * self,
     894              :     GstBuffer * buf)
     895              : {
     896              :   GstBuffer *buffer;
     897              :   GstMemory *mem, *new_mem;
     898              :   GstTensorsInfo *info;
     899              :   GstTensorInfo *_info;
     900              :   GstTensorMetaInfo meta;
     901              :   guint i;
     902              : 
     903          220 :   info = &self->tensors_config.info;
     904          220 :   buffer = gst_buffer_new ();
     905              : 
     906          445 :   for (i = 0; i < info->num_tensors; i++) {
     907          225 :     _info = gst_tensors_info_get_nth_info (info, i);
     908          225 :     gst_tensor_info_convert_to_meta (_info, &meta);
     909              : 
     910              :     /* set media type */
     911          225 :     switch (self->in_media_type) {
     912          225 :       case _NNS_VIDEO:
     913              :       case _NNS_AUDIO:
     914              :       case _NNS_TEXT:
     915              :       case _NNS_OCTET:
     916          225 :         meta.media_type = self->in_media_type;
     917          225 :         break;
     918            0 :       default:
     919              :         /* default output type is tensor */
     920            0 :         meta.media_type = _NNS_TENSOR;
     921            0 :         break;
     922              :     }
     923              : 
     924          225 :     mem = gst_tensor_buffer_get_nth_memory (buf, i);
     925          225 :     new_mem = gst_tensor_meta_info_append_header (&meta, mem);
     926          225 :     gst_memory_unref (mem);
     927              : 
     928          225 :     gst_tensor_buffer_append_memory (buffer, new_mem, _info);
     929              :   }
     930              : 
     931          220 :   gst_buffer_copy_into (buffer, buf, GST_BUFFER_COPY_METADATA, 0, -1);
     932          220 :   gst_buffer_unref (buf);
     933          220 :   return buffer;
     934              : }
     935              : 
     936              : /** @brief Chain function's private routine to push buffer into src pad */
     937              : static GstFlowReturn
     938        33381 : _gst_tensor_converter_chain_push (GstTensorConverter * self, GstBuffer * buf)
     939              : {
     940        33381 :   GstBuffer *buffer = buf;
     941              : 
     942        33381 :   if (self->in_media_type == _NNS_OCTET) {
     943              :     /* configure multi tensors */
     944          234 :     buffer = _gst_tensor_converter_chain_octet (self, buffer);
     945              :   }
     946              : 
     947              :   /* if output is flexible, add header. */
     948        33381 :   if (!self->do_not_append_header
     949        33363 :       && gst_tensor_pad_caps_is_flexible (self->srcpad)) {
     950          220 :     buffer = _gst_tensor_converter_chain_flex_tensor (self, buffer);
     951              :   }
     952              : 
     953        33381 :   silent_debug_timestamp (self, buffer);
     954        33381 :   return gst_pad_push (self->srcpad, buffer);
     955              : }
     956              : 
     957              : /** @brief Chain function's private routine to push multiple buffers */
     958              : static GstFlowReturn
     959          121 : _gst_tensor_converter_chain_chunk (GstTensorConverter * self,
     960              :     GstBuffer * inbuf, guint frames_in, guint frames_out, gsize frame_size)
     961              : {
     962              :   GstAdapter *adapter;
     963              :   GstTensorsConfig *config;
     964          121 :   GstFlowReturn ret = GST_FLOW_OK;
     965              :   GstClockTime pts, dts, duration;
     966              :   gsize avail, out_size;
     967              :   gboolean have_framerate;
     968              : 
     969          121 :   config = &self->tensors_config;
     970          121 :   adapter = gst_tensor_converter_get_adapter (self, inbuf);
     971          121 :   g_assert (adapter != NULL);
     972              : 
     973          121 :   have_framerate = (config->rate_n > 0 && config->rate_d > 0);
     974              : 
     975          121 :   duration = GST_BUFFER_DURATION (inbuf);
     976          121 :   if (GST_CLOCK_TIME_IS_VALID (duration)) {
     977              :     /** supposed same duration for incoming buffer */
     978          119 :     duration = gst_util_uint64_scale_int (duration, frames_out, frames_in);
     979              :   }
     980              : 
     981          121 :   gst_adapter_push (adapter, inbuf);
     982              : 
     983          121 :   out_size = frames_out * frame_size;
     984          233 :   while ((avail = gst_adapter_available (adapter)) >= out_size &&
     985              :       ret == GST_FLOW_OK) {
     986              :     GstBuffer *outbuf;
     987              :     guint64 pts_dist, dts_dist;
     988              : 
     989          112 :     pts = gst_adapter_prev_pts (adapter, &pts_dist);
     990          112 :     dts = gst_adapter_prev_dts (adapter, &dts_dist);
     991              : 
     992              :     /**
     993              :      * Update timestamp.
     994              :      * If frames-in is larger then frames-out, the same timestamp (pts and dts) would be returned.
     995              :      */
     996          112 :     if (frames_in > 1 && have_framerate) {
     997           47 :       if (GST_CLOCK_TIME_IS_VALID (pts)) {
     998           47 :         pts +=
     999           47 :             gst_util_uint64_scale_int (pts_dist * config->rate_d, GST_SECOND,
    1000           47 :             config->rate_n * frame_size);
    1001              :       }
    1002              : 
    1003           47 :       if (GST_CLOCK_TIME_IS_VALID (dts)) {
    1004           20 :         dts +=
    1005           20 :             gst_util_uint64_scale_int (dts_dist * config->rate_d, GST_SECOND,
    1006           20 :             config->rate_n * frame_size);
    1007              :       }
    1008              :     }
    1009              : 
    1010          112 :     outbuf = gst_adapter_take_buffer (adapter, out_size);
    1011          112 :     outbuf = gst_buffer_make_writable (outbuf);
    1012              : 
    1013              :     /** set timestamp */
    1014          112 :     GST_BUFFER_PTS (outbuf) = pts;
    1015          112 :     GST_BUFFER_DTS (outbuf) = dts;
    1016          112 :     GST_BUFFER_DURATION (outbuf) = duration;
    1017              : 
    1018          112 :     ret = _gst_tensor_converter_chain_push (self, outbuf);
    1019              :   }
    1020              : 
    1021          121 :   return ret;
    1022              : }
    1023              : 
    1024              : /**
    1025              :  * @brief Chain function, this function does the actual processing.
    1026              :  */
    1027              : static GstFlowReturn
    1028        33392 : gst_tensor_converter_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
    1029              : {
    1030              :   GstTensorConverter *self;
    1031              :   GstTensorsConfig *config;
    1032              :   GstTensorsConfig new_config;
    1033              :   GstTensorInfo *_info;
    1034              :   GstBuffer *inbuf;
    1035              :   gsize buf_size, frame_size;
    1036              :   guint frames_in, frames_out;
    1037              :   UNUSED (pad);
    1038              : 
    1039        33392 :   buf_size = gst_buffer_get_size (buf);
    1040        66783 :   g_return_val_if_fail (buf_size > 0, GST_FLOW_ERROR);
    1041              : 
    1042        33392 :   self = GST_TENSOR_CONVERTER (parent);
    1043              : 
    1044              :   /** This is an internal logic error. */
    1045        33392 :   g_assert (self->tensors_configured);
    1046        33392 :   config = &self->tensors_config;
    1047        33392 :   gst_tensors_config_init (&new_config);
    1048              : 
    1049        33392 :   frames_out = self->frames_per_tensor;
    1050        33392 :   inbuf = buf;
    1051              : 
    1052              :   /**
    1053              :    * Supposed 1 frame in buffer (default).
    1054              :    * Update frame size for each media type.
    1055              :    */
    1056        33392 :   frame_size = self->frame_size;
    1057        33392 :   frames_in = 1;
    1058              : 
    1059        33392 :   switch (self->in_media_type) {
    1060        32773 :     case _NNS_VIDEO:
    1061              :     {
    1062              :       guint color, width, height;
    1063              :       gsize type;
    1064              : 
    1065        32773 :       color = config->info.info[0].dimension[0];
    1066        32773 :       width = config->info.info[0].dimension[1];
    1067        32773 :       height = config->info.info[0].dimension[2];
    1068        32773 :       type = gst_tensor_get_element_size (config->info.info[0].type);
    1069              : 
    1070              :       /** type * colorspace * width * height */
    1071        32773 :       frame_size = type * color * width * height;
    1072              : 
    1073              :       /** supposed 1 frame in buffer */
    1074        32773 :       g_assert ((buf_size / self->frame_size) == 1);
    1075              : 
    1076        32773 :       if (self->remove_padding) {
    1077              :         GstMapInfo src_info, dest_info;
    1078              :         guint d0, d1;
    1079          110 :         unsigned int src_idx = 0, dest_idx = 0;
    1080              :         size_t size, offset;
    1081              : 
    1082          110 :         if (!gst_buffer_map (buf, &src_info, GST_MAP_READ)) {
    1083            0 :           ml_logf
    1084              :               ("tensor_converter: Cannot map src buffer at tensor_converter/video. The incoming buffer (GstBuffer) for the sinkpad of tensor_converter cannot be mapped for reading.\n");
    1085            0 :           goto error;
    1086              :         }
    1087              : 
    1088          110 :         inbuf = gst_buffer_new_and_alloc (frame_size);
    1089          110 :         gst_buffer_memset (inbuf, 0, 0, frame_size);
    1090          110 :         if (!gst_buffer_map (inbuf, &dest_info, GST_MAP_WRITE)) {
    1091            0 :           ml_logf
    1092              :               ("tensor_converter: Cannot map dest buffer at tensor_converter/video. The outgoing buffer (GstBuffer) for the srcpad of tensor_converter cannot be mapped for writing.\n");
    1093            0 :           gst_buffer_unmap (buf, &src_info);
    1094            0 :           gst_buffer_unref (inbuf);     /* the new buffer is wasted. */
    1095            0 :           goto error;
    1096              :         }
    1097              : 
    1098              :         /**
    1099              :          * Refer: https://gstreamer.freedesktop.org/documentation/design/mediatype-video-raw.html
    1100              :          */
    1101          110 :         size = offset = type * color * width;
    1102              : 
    1103          110 :         g_assert (offset % 4); /** Internal logic error! */
    1104          110 :         if (offset % 4) {
    1105          110 :           offset += 4 - (offset % 4);
    1106              :         }
    1107              : 
    1108          220 :         for (d0 = 0; d0 < frames_in; d0++) {
    1109        11095 :           for (d1 = 0; d1 < height; d1++) {
    1110        10985 :             memcpy (dest_info.data + dest_idx, src_info.data + src_idx, size);
    1111        10985 :             dest_idx += size;
    1112        10985 :             src_idx += offset;
    1113              :           }
    1114              :         }
    1115              : 
    1116          110 :         gst_buffer_unmap (buf, &src_info);
    1117          110 :         gst_buffer_unmap (inbuf, &dest_info);
    1118              : 
    1119              :         /** copy timestamps */
    1120          110 :         gst_buffer_copy_into (inbuf, buf, GST_BUFFER_COPY_METADATA, 0, -1);
    1121              :       }
    1122        32773 :       break;
    1123              :     }
    1124          134 :     case _NNS_AUDIO:
    1125              :       /* number of bytes for one frame */
    1126          134 :       frames_in = buf_size / frame_size;
    1127          134 :       break;
    1128           36 :     case _NNS_TEXT:
    1129           36 :       if (buf_size != frame_size) {
    1130              :         GstMapInfo src_info, dest_info;
    1131           20 :         gsize block_size = MIN (buf_size, frame_size);
    1132              : 
    1133           20 :         if (!gst_buffer_map (buf, &src_info, GST_MAP_READ)) {
    1134            0 :           ml_logf
    1135              :               ("tensor_converter: Cannot map src buffer at tensor_converter/text. The incoming buffer (GstBuffer) for the sinkpad of tensor_converter cannot be mapped for reading.\n");
    1136            0 :           goto error;
    1137              :         }
    1138              : 
    1139           20 :         inbuf = gst_buffer_new_and_alloc (frame_size);
    1140           20 :         gst_buffer_memset (inbuf, 0, 0, frame_size);
    1141           20 :         if (!gst_buffer_map (inbuf, &dest_info, GST_MAP_WRITE)) {
    1142            0 :           ml_logf
    1143              :               ("tensor_converter: Cannot map dest buffer at tensor_converter/text. The outgoing buffer (GstBuffer) for the srcpad of tensor_converter cannot be mapped for writing.\n");
    1144            0 :           gst_buffer_unmap (buf, &src_info);
    1145            0 :           gst_buffer_unref (inbuf);     /* the new buffer is wasted. */
    1146            0 :           goto error;
    1147              :         }
    1148              : 
    1149           20 :         memcpy (dest_info.data, src_info.data, block_size);
    1150              : 
    1151           20 :         gst_buffer_unmap (buf, &src_info);
    1152           20 :         gst_buffer_unmap (inbuf, &dest_info);
    1153              : 
    1154              :         /** copy timestamps */
    1155           20 :         gst_buffer_copy_into (inbuf, buf, GST_BUFFER_COPY_METADATA, 0, -1);
    1156              :       }
    1157           36 :       break;
    1158          206 :     case _NNS_OCTET:
    1159          206 :       if (gst_tensors_config_is_flexible (config)) {
    1160              :         /* update dimension with buffer size */
    1161            3 :         config->info.info[0].dimension[0] = buf_size;
    1162            3 :         frame_size = buf_size;
    1163              :       } else {
    1164              :         /* get frame size from the properties */
    1165          203 :         g_assert ((buf_size % frame_size) == 0); /** @todo need rewrite. do not use assert */
    1166          203 :         frames_in = buf_size / frame_size;
    1167              :       }
    1168          206 :       break;
    1169            9 :     case _NNS_TENSOR:
    1170              :     {
    1171              :       GstTensorMetaInfo meta;
    1172              :       GstTensorsConfig tmp;
    1173              :       GstMemory *mem, *new_mem;
    1174              :       gsize s1, s2, hsize;
    1175              :       guint n;
    1176              : 
    1177            9 :       gst_tensors_config_init (&tmp);
    1178            9 :       tmp.rate_n = config->rate_n;
    1179            9 :       tmp.rate_d = config->rate_d;
    1180              : 
    1181            9 :       tmp.info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
    1182            9 :       buf = gst_tensor_buffer_from_config (buf, &tmp);
    1183              : 
    1184              :       /* type and dimension from buffer */
    1185            9 :       tmp.info.format = _NNS_TENSOR_FORMAT_STATIC;
    1186            9 :       tmp.info.num_tensors = gst_tensor_buffer_get_count (buf);
    1187              : 
    1188              :       /* compare data size and append memory */
    1189            9 :       inbuf = gst_buffer_new ();
    1190              : 
    1191           19 :       for (n = 0; n < tmp.info.num_tensors; n++) {
    1192           11 :         _info = gst_tensors_info_get_nth_info (&tmp.info, n);
    1193           11 :         mem = gst_tensor_buffer_get_nth_memory (buf, n);
    1194           11 :         s1 = gst_memory_get_sizes (mem, NULL, NULL);
    1195              : 
    1196              :         /* flex-tensor has header in each mem block */
    1197           11 :         gst_tensor_meta_info_parse_memory (&meta, mem);
    1198           11 :         gst_tensor_meta_info_convert (&meta, _info);
    1199           11 :         hsize = gst_tensor_meta_info_get_header_size (&meta);
    1200           11 :         s1 -= hsize;
    1201              : 
    1202           11 :         s2 = gst_tensor_info_get_size (_info);
    1203              : 
    1204              :         /**
    1205              :          * @todo expand mem if given property is larger than mem size.
    1206              :          * Now compare same size, later we should modify mem block if developer sets different dimension.
    1207              :          */
    1208           11 :         if (s1 != s2) {
    1209            1 :           nns_loge
    1210              :               ("Cannot process an incoming buffer frame for tensor_converter (chain function). It appears that it is trying to convert other/tensors,format=flexible to other/tensors,format=static. Incoming buffer has invalid data size %zd, expected size is %zd (%u/%u).",
    1211              :               s1, s2, (n + 1), tmp.info.num_tensors);
    1212            1 :           gst_memory_unref (mem);
    1213            1 :           gst_buffer_unref (inbuf);
    1214            2 :           goto error;
    1215              :         }
    1216              : 
    1217           10 :         new_mem = gst_memory_share (mem, hsize, s1);
    1218           10 :         gst_memory_unref (mem);
    1219           10 :         gst_tensor_buffer_append_memory (inbuf, new_mem, _info);
    1220              :       }
    1221              : 
    1222            8 :       gst_buffer_copy_into (inbuf, buf, GST_BUFFER_COPY_METADATA, 0, -1);
    1223              : 
    1224            8 :       if (!gst_tensors_config_is_equal (config, &tmp)) {
    1225            3 :         if (gst_tensors_info_validate (&self->tensors_info)) {
    1226            1 :           nns_loge
    1227              :               ("Incoming buffer does not match with given tensors info. It appears that it is trying to convert other/tensors,format=flexible to other/tensors,format=static. The converted output appears not compatible with the given configuration.");
    1228            1 :           gst_buffer_unref (inbuf);
    1229            1 :           goto error;
    1230              :         }
    1231              : 
    1232              :         /* update caps with new configuration */
    1233            2 :         *config = tmp;
    1234            2 :         gst_tensor_converter_update_caps (self);
    1235              :       }
    1236            7 :       break;
    1237              :     }
    1238          234 :     case _NNS_MEDIA_ANY:
    1239              :     {
    1240          234 :       if (self->mode == _CONVERTER_MODE_CUSTOM_CODE) {
    1241            1 :         if (self->custom.func == NULL) {
    1242            0 :           nns_loge
    1243              :               ("Tensor converter is in custom/code mode (mode=custom-code:${funcname}), where a user code as a callback function is required. However, the required information to configure the tensor converter is not given or incorrectly given. For detail, please refer to https://github.com/nnstreamer/nnstreamer/blob/main/gst/nnstreamer/elements/gsttensor_converter.md#custom-converter. The given ${funcname} is \"%s\", which is an invalid/unregistered name.",
    1244              :               self->mode_option);
    1245            0 :           goto error;
    1246              :         }
    1247            1 :         inbuf = self->custom.func (buf, self->custom.data, &new_config);
    1248              : 
    1249            1 :         if (inbuf == NULL) {
    1250            0 :           nns_loge
    1251              :               ("Failed to convert input streams to tensors: the converted result of the incoming buffer is NULL. The converter is custom-func with %s function, which is available and loaded, but has returned NULL buffer after the conversion.",
    1252              :               self->mode_option);
    1253            0 :           goto error;
    1254              :         }
    1255          233 :       } else if (self->externalConverter && self->externalConverter->convert) {
    1256              :         inbuf =
    1257          233 :             self->externalConverter->convert (buf, &new_config,
    1258              :             self->priv_data);
    1259              : 
    1260          233 :         if (inbuf == NULL) {
    1261            0 :           nns_loge
    1262              :               ("Failed to convert input streams to tensors: the converted result of the incoming buffer is NULL. The converter is using external tensor_converter subplugin (%s), which is available and loaded, but has returned NULL buffer after the conversion.",
    1263              :               self->externalConverter->name);
    1264            0 :           goto error;
    1265              :         }
    1266            0 :       } else if (self->mode == _CONVERTER_MODE_CUSTOM_SCRIPT) {
    1267              :         /* self->externalConverter->converter should've been available! */
    1268            0 :         GST_ERROR_OBJECT (self,
    1269              :             "Tensor converter is in custom/script mode (mode=custom-script:${scriptpath}), where a path to a script file is required. However, it is not properly configured. The given ${scriptpath} is \"%s\".",
    1270              :             self->mode_option);
    1271            0 :         goto error;
    1272              :       } else {
    1273              :         /** @todo identify and printout the given input stream caps. */
    1274            0 :         GST_ERROR_OBJECT (self,
    1275              :             "Tensor converter has an undefined behavior with type _NNS_MEDIA_ANY. It should've been custom-code or custom-script mode or a corresponding external converter should've been registered (tensor_converter subplugin). However, nothing is available for the given input stream.");
    1276            0 :         goto error;
    1277              :       }
    1278          234 :       self->do_not_append_header =
    1279          234 :           (new_config.info.format == _NNS_TENSOR_FORMAT_FLEXIBLE);
    1280              : 
    1281          234 :       frames_in = 1;
    1282          234 :       frame_size = gst_buffer_get_size (inbuf);
    1283              : 
    1284          234 :       if (!gst_tensors_config_is_equal (config, &new_config)) {
    1285           72 :         gst_tensors_config_free (config);
    1286           72 :         *config = new_config;
    1287              : 
    1288           72 :         gst_tensor_converter_update_caps (self);
    1289              :       } else {
    1290          162 :         gst_tensors_config_free (&new_config);
    1291              :       }
    1292              : 
    1293          234 :       break;
    1294              :     }
    1295            0 :     case _NNS_MEDIA_INVALID:
    1296            0 :       GST_ERROR_OBJECT (self,
    1297              :           "The incoming tensor to be converted has no type defined (INVALID). This is an internal unknown error. Please report the case to https://github.com/nnstreamer/issues with the pipeline description reproducing the error.");
    1298            0 :       goto error;
    1299              :       break;
    1300            0 :     default:
    1301            0 :       GST_ERROR_OBJECT (self,
    1302              :           "The incoming tensor to be converted has unknown type (type value not recognized: %d). This is an internal unknown error. Please report the case to https://github.com/nnstreamer/issues with the pipeline description reproducing the error.",
    1303              :           self->in_media_type);
    1304            0 :       goto error;
    1305              :   }
    1306              : 
    1307        33390 :   if (inbuf != buf)
    1308          371 :     gst_buffer_unref (buf);
    1309              : 
    1310              :   /** convert format (bytes > time) and push segment event.
    1311              :     * It will push event if needed (self->need_segment is true). */
    1312        33390 :   _gst_tensor_converter_chain_segment (self, frame_size);
    1313              : 
    1314              :   /** configures timestamp if required (self->set_timestamp is true) */
    1315        33390 :   _gst_tensor_converter_chain_timestamp (self, inbuf, frames_in);
    1316              : 
    1317        33390 :   if (frames_in == frames_out) {
    1318              :     /** do nothing, push the incoming buffer */
    1319        33269 :     return _gst_tensor_converter_chain_push (self, inbuf);
    1320              :   }
    1321              : 
    1322              :   /* push multiple buffers */
    1323          121 :   return _gst_tensor_converter_chain_chunk (self, inbuf, frames_in,
    1324              :       frames_out, frame_size);
    1325              : 
    1326            2 : error:
    1327            2 :   gst_buffer_unref (buf);
    1328            2 :   gst_tensors_config_free (&new_config);
    1329            2 :   return GST_FLOW_ERROR;
    1330              : }
    1331              : 
    1332              : /**
    1333              :  * @brief Called to perform state change.
    1334              :  */
    1335              : static GstStateChangeReturn
    1336         4970 : gst_tensor_converter_change_state (GstElement * element,
    1337              :     GstStateChange transition)
    1338              : {
    1339              :   GstTensorConverter *self;
    1340              :   GstStateChangeReturn ret;
    1341              : 
    1342         4970 :   self = GST_TENSOR_CONVERTER (element);
    1343              : 
    1344         4970 :   switch (transition) {
    1345          856 :     case GST_STATE_CHANGE_READY_TO_PAUSED:
    1346          856 :       gst_tensor_converter_reset (self);
    1347          856 :       break;
    1348         4114 :     default:
    1349         4114 :       break;
    1350              :   }
    1351              : 
    1352         4970 :   ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
    1353              : 
    1354         4970 :   switch (transition) {
    1355          804 :     case GST_STATE_CHANGE_PAUSED_TO_READY:
    1356          804 :       gst_tensor_converter_reset (self);
    1357          804 :       break;
    1358         4166 :     default:
    1359         4166 :       break;
    1360              :   }
    1361              : 
    1362         4970 :   return ret;
    1363              : }
    1364              : 
    1365              : /**
    1366              :  * @brief Clear and reset data.
    1367              :  */
    1368              : static void
    1369         3407 : gst_tensor_converter_reset (GstTensorConverter * self)
    1370              : {
    1371              :   /* remove all buffers from adapter */
    1372         3407 :   gst_tensor_aggregation_clear_all (self->adapter_table);
    1373              : 
    1374         3407 :   self->have_segment = FALSE;
    1375         3407 :   self->need_segment = FALSE;
    1376         3407 :   gst_segment_init (&self->segment, GST_FORMAT_TIME);
    1377              : 
    1378         3407 :   self->old_timestamp = GST_CLOCK_TIME_NONE;
    1379         3407 : }
    1380              : 
    1381              : /**
    1382              :  * @brief Get supported format list.
    1383              :  */
    1384              : static void
    1385          317 : gst_tensor_converter_get_format_list (GValue * list, ...)
    1386              : {
    1387          317 :   GValue item = G_VALUE_INIT;
    1388              :   gchar *str;
    1389              :   va_list args;
    1390              : 
    1391          317 :   g_value_init (list, GST_TYPE_LIST);
    1392              : 
    1393          317 :   va_start (args, list);
    1394          960 :   while ((str = va_arg (args, gchar *))) {
    1395          643 :     g_value_init (&item, G_TYPE_STRING);
    1396          643 :     g_value_set_string (&item, str);
    1397              : 
    1398          643 :     gst_value_list_append_value (list, &item);
    1399          643 :     g_value_unset (&item);
    1400              :   }
    1401          317 :   va_end (args);
    1402          317 : }
    1403              : 
    1404              : /**
    1405              :  * @brief Determine if we need zero-padding
    1406              :  * @return TRUE if we need to add (or remove) stride per row from the stream data.
    1407              :  */
    1408              : static gboolean
    1409          620 : gst_tensor_converter_video_stride (GstVideoFormat format, gint width)
    1410              : {
    1411              :   /**
    1412              :    * @todo The actual list is much longer, fill them.
    1413              :    * (read https://gstreamer.freedesktop.org/documentation/design/mediatype-video-raw.html)
    1414              :    */
    1415          620 :   switch (format) {
    1416          530 :     case GST_VIDEO_FORMAT_GRAY8:
    1417              :     case GST_VIDEO_FORMAT_RGB:
    1418              :     case GST_VIDEO_FORMAT_BGR:
    1419              :     case GST_VIDEO_FORMAT_I420:
    1420              : #if GST_CHECK_VERSION(1, 20, 0)
    1421              :     case GST_VIDEO_FORMAT_RGBP:
    1422              :     case GST_VIDEO_FORMAT_BGRP:
    1423              : #endif
    1424          530 :       if (width % 4) {
    1425           32 :         return TRUE;
    1426              :       }
    1427          498 :       break;
    1428           90 :     default:
    1429           90 :       break;
    1430              :   }
    1431              : 
    1432          588 :   return FALSE;
    1433              : }
    1434              : 
    1435              : /**
    1436              :  * @brief Set the tensors config structure from video info (internal static function)
    1437              :  * @param self this pointer to GstTensorConverter
    1438              :  * @param caps caps for media stream
    1439              :  * @param config tensors config structure to be filled
    1440              :  * @note Change dimension if tensor contains N frames.
    1441              :  * @return TRUE if supported type
    1442              :  */
    1443              : static gboolean
    1444          620 : gst_tensor_converter_parse_video (GstTensorConverter * self,
    1445              :     const GstCaps * caps, GstTensorsConfig * config)
    1446              : {
    1447              :   /**
    1448              :    * Refer: https://www.tensorflow.org/api_docs/python/tf/summary/image
    1449              :    * A 4-D uint8 or float32 Tensor of shape [batch_size, height, width, channels]
    1450              :    * where channels is 1, 3, or 4.
    1451              :    */
    1452              :   GstVideoInfo vinfo;
    1453              :   GstVideoFormat format;
    1454              :   gint width, height, views;
    1455              :   guint i;
    1456              : 
    1457         1240 :   g_return_val_if_fail (config != NULL, FALSE);
    1458              : 
    1459          620 :   gst_tensors_config_init (config);
    1460              : 
    1461          620 :   gst_video_info_init (&vinfo);
    1462          620 :   if (!gst_video_info_from_caps (&vinfo, caps)) {
    1463            0 :     char *capstr = gst_caps_to_string (caps);
    1464            0 :     GST_ERROR_OBJECT (self,
    1465              :         "Failed to get video info from caps; gst_video_info_from_caps (&info, \"%s\") has returned FALSE, which means the given caps cannot be parsed as a video.",
    1466              :         capstr);
    1467            0 :     g_free (capstr);
    1468            0 :     return FALSE;
    1469              :   }
    1470              : 
    1471          620 :   format = GST_VIDEO_INFO_FORMAT (&vinfo);
    1472          620 :   width = GST_VIDEO_INFO_WIDTH (&vinfo);
    1473          620 :   height = GST_VIDEO_INFO_HEIGHT (&vinfo);
    1474          620 :   views = GST_VIDEO_INFO_VIEWS (&vinfo);
    1475              : 
    1476          620 :   if (views > 1) {
    1477            0 :     GST_WARNING_OBJECT (self,
    1478              :         "Incoming video caps should have 'views=(int)1 but has views=(int)%d - ignoring all but view #0. \n",
    1479              :         views);
    1480              :   }
    1481              : 
    1482          620 :   config->info.num_tensors = 1;
    1483              : 
    1484              :   /* [color-space][width][height][frames] */
    1485          620 :   switch (format) {
    1486           18 :     case GST_VIDEO_FORMAT_GRAY8:
    1487           18 :       config->info.info[0].type = _NNS_UINT8;
    1488           18 :       config->info.info[0].dimension[0] = 1;
    1489           18 :       config->info.info[0].dimension[1] = width;
    1490           18 :       config->info.info[0].dimension[2] = height;
    1491           18 :       break;
    1492            6 :     case GST_VIDEO_FORMAT_GRAY16_BE:
    1493              :     case GST_VIDEO_FORMAT_GRAY16_LE:
    1494            6 :       config->info.info[0].type = _NNS_UINT16;
    1495            6 :       config->info.info[0].dimension[0] = 1;
    1496            6 :       config->info.info[0].dimension[1] = width;
    1497            6 :       config->info.info[0].dimension[2] = height;
    1498            6 :       break;
    1499          510 :     case GST_VIDEO_FORMAT_RGB:
    1500              :     case GST_VIDEO_FORMAT_BGR:
    1501          510 :       config->info.info[0].type = _NNS_UINT8;
    1502          510 :       config->info.info[0].dimension[0] = 3;
    1503          510 :       config->info.info[0].dimension[1] = width;
    1504          510 :       config->info.info[0].dimension[2] = height;
    1505          510 :       break;
    1506           84 :     case GST_VIDEO_FORMAT_RGBx:
    1507              :     case GST_VIDEO_FORMAT_BGRx:
    1508              :     case GST_VIDEO_FORMAT_xRGB:
    1509              :     case GST_VIDEO_FORMAT_xBGR:
    1510              :     case GST_VIDEO_FORMAT_RGBA:
    1511              :     case GST_VIDEO_FORMAT_BGRA:
    1512              :     case GST_VIDEO_FORMAT_ARGB:
    1513              :     case GST_VIDEO_FORMAT_ABGR:
    1514           84 :       config->info.info[0].type = _NNS_UINT8;
    1515           84 :       config->info.info[0].dimension[0] = 4;
    1516           84 :       config->info.info[0].dimension[1] = width;
    1517           84 :       config->info.info[0].dimension[2] = height;
    1518           84 :       break;
    1519              : #if GST_CHECK_VERSION(1, 20, 0)
    1520            2 :     case GST_VIDEO_FORMAT_RGBP:
    1521              :     case GST_VIDEO_FORMAT_BGRP:
    1522            2 :       config->info.info[0].type = _NNS_UINT8;
    1523            2 :       config->info.info[0].dimension[0] = width;
    1524            2 :       config->info.info[0].dimension[1] = height;
    1525            2 :       config->info.info[0].dimension[2] = 3;
    1526            2 :       break;
    1527              : #endif
    1528            0 :     default:
    1529            0 :       GST_WARNING_OBJECT (self,
    1530              :           "The given video caps with format \"%s\" is not supported. Please use " NNS_VIDEO_FORMAT,
    1531              :           GST_STR_NULL (gst_video_format_to_string (format)));
    1532            0 :       break;
    1533              :   }
    1534              : 
    1535              :   /* Supposed 1 frame in tensor, change dimension[3] if tensor contains N frames. */
    1536          620 :   config->info.info[0].dimension[3] = 1;
    1537         8060 :   for (i = 4; i < NNS_TENSOR_RANK_LIMIT; i++)
    1538         7440 :     config->info.info[0].dimension[i] = 0;
    1539              : 
    1540          620 :   config->rate_n = GST_VIDEO_INFO_FPS_N (&vinfo);
    1541          620 :   config->rate_d = GST_VIDEO_INFO_FPS_D (&vinfo);
    1542              : 
    1543              :   /**
    1544              :    * Emit Warning if RSTRIDE = RU4 (3BPP) && Width % 4 > 0
    1545              :    * @todo Add more conditions!
    1546              :    */
    1547          620 :   if (gst_tensor_converter_video_stride (format, width)) {
    1548           32 :     self->remove_padding = TRUE;
    1549           32 :     silent_debug (self, "Set flag to remove padding, width = %d", width);
    1550              : 
    1551              : #if GST_CHECK_VERSION(1, 20, 0)
    1552           32 :     if (format == GST_VIDEO_FORMAT_RGBP || format == GST_VIDEO_FORMAT_BGRP) {
    1553            0 :       if (self->remove_padding) {
    1554            0 :         GST_ERROR_OBJECT (self,
    1555              :             "Padding removal is not supported for RGBP and BGRP formats. Please use width as multiple of 4. Given width: %d",
    1556              :             width);
    1557            0 :         return FALSE;
    1558              :       }
    1559              :     }
    1560              : #endif
    1561              : 
    1562              :     /** @todo need rewrite. */
    1563           32 :     GST_WARNING_OBJECT (self,
    1564              :         "\nYOUR STREAM CONFIGURATION INCURS PERFORMANCE DETERIORATION!\n"
    1565              :         "Please use 4 x n as image width for inputs; the width of your input is %d.\n",
    1566              :         width);
    1567              :   }
    1568              : 
    1569          620 :   self->frame_size = GST_VIDEO_INFO_SIZE (&vinfo);
    1570          620 :   return (config->info.info[0].type != _NNS_END);
    1571              : }
    1572              : 
    1573              : /**
    1574              :  * @brief Set the tensors config structure from audio info (internal static function)
    1575              :  * @param self this pointer to GstTensorConverter
    1576              :  * @param caps caps for media stream
    1577              :  * @param config tensors config structure to be filled
    1578              :  * @note Change dimension if tensor contains N frames.
    1579              :  * @return TRUE if supported type
    1580              :  */
    1581              : static gboolean
    1582           37 : gst_tensor_converter_parse_audio (GstTensorConverter * self,
    1583              :     const GstCaps * caps, GstTensorsConfig * config)
    1584              : {
    1585              :   /**
    1586              :    * Refer: https://www.tensorflow.org/api_docs/python/tf/summary/audio
    1587              :    * A 3-D float32 Tensor of shape [batch_size, frames, channels]
    1588              :    * or a 2-D float32 Tensor of shape [batch_size, frames].
    1589              :    */
    1590              :   GstAudioInfo ainfo;
    1591              :   GstAudioFormat format;
    1592              :   gint channels;
    1593              :   guint i;
    1594              : 
    1595           74 :   g_return_val_if_fail (config != NULL, FALSE);
    1596              : 
    1597           37 :   gst_tensors_config_init (config);
    1598              : 
    1599           37 :   gst_audio_info_init (&ainfo);
    1600           37 :   if (!gst_audio_info_from_caps (&ainfo, caps)) {
    1601            0 :     char *capstr = gst_caps_to_string (caps);
    1602            0 :     GST_ERROR_OBJECT (self,
    1603              :         "Failed to get audio info from caps; gst_audio_info_from_caps(&info, \"%s\") has returned FALSE.\n",
    1604              :         capstr);
    1605            0 :     g_free (capstr);
    1606            0 :     return FALSE;
    1607              :   }
    1608              : 
    1609           37 :   format = GST_AUDIO_INFO_FORMAT (&ainfo);
    1610           37 :   channels = GST_AUDIO_INFO_CHANNELS (&ainfo);
    1611              : 
    1612           37 :   config->info.num_tensors = 1;
    1613              : 
    1614              :   /* [channels][frames] */
    1615           37 :   switch (format) {
    1616            1 :     case GST_AUDIO_FORMAT_S8:
    1617            1 :       config->info.info[0].type = _NNS_INT8;
    1618            1 :       break;
    1619            7 :     case GST_AUDIO_FORMAT_U8:
    1620            7 :       config->info.info[0].type = _NNS_UINT8;
    1621            7 :       break;
    1622           16 :     case GST_AUDIO_FORMAT_S16:
    1623           16 :       config->info.info[0].type = _NNS_INT16;
    1624           16 :       break;
    1625            8 :     case GST_AUDIO_FORMAT_U16:
    1626            8 :       config->info.info[0].type = _NNS_UINT16;
    1627            8 :       break;
    1628            2 :     case GST_AUDIO_FORMAT_S32:
    1629            2 :       config->info.info[0].type = _NNS_INT32;
    1630            2 :       break;
    1631            1 :     case GST_AUDIO_FORMAT_U32:
    1632            1 :       config->info.info[0].type = _NNS_UINT32;
    1633            1 :       break;
    1634            1 :     case GST_AUDIO_FORMAT_F32:
    1635            1 :       config->info.info[0].type = _NNS_FLOAT32;
    1636            1 :       break;
    1637            1 :     case GST_AUDIO_FORMAT_F64:
    1638            1 :       config->info.info[0].type = _NNS_FLOAT64;
    1639            1 :       break;
    1640            0 :     default:
    1641            0 :       GST_WARNING_OBJECT (self,
    1642              :           "Audio format \"%s\" is not supported. Please use S8, U8, S16, U16, S32, U32, F32, or F64.\n",
    1643              :           GST_STR_NULL (gst_audio_format_to_string (format)));
    1644            0 :       break;
    1645              :   }
    1646              : 
    1647           37 :   config->info.info[0].dimension[0] = channels;
    1648              : 
    1649              :   /* Supposed 1 frame in tensor, change dimension[1] if tensor contains N frames. */
    1650           37 :   config->info.info[0].dimension[1] = 1;
    1651          555 :   for (i = 2; i < NNS_TENSOR_RANK_LIMIT; i++)
    1652          518 :     config->info.info[0].dimension[i] = 0;
    1653              : 
    1654           37 :   config->rate_n = GST_AUDIO_INFO_RATE (&ainfo);
    1655           37 :   config->rate_d = 1;
    1656              : 
    1657           37 :   self->frame_size = GST_AUDIO_INFO_BPF (&ainfo);
    1658           37 :   return (config->info.info[0].type != _NNS_END);
    1659              : }
    1660              : 
    1661              : /**
    1662              :  * @brief Set the tensors config structure from text info (internal static function)
    1663              :  * @param self this pointer to GstTensorConverter
    1664              :  * @param config tensors config structure to be filled
    1665              :  * @param structure caps structure
    1666              :  * @note Change dimension if tensors contains N frames.
    1667              :  * @return TRUE if supported type
    1668              :  */
    1669              : static gboolean
    1670           10 : gst_tensor_converter_parse_text (GstTensorConverter * self,
    1671              :     GstTensorsConfig * config, const GstStructure * structure)
    1672              : {
    1673              :   /**
    1674              :    * Refer: https://www.tensorflow.org/api_docs/python/tf/summary/text
    1675              :    * A string-type Tensor
    1676              :    */
    1677              :   const gchar *format_string;
    1678              :   guint i, text_size;
    1679              : 
    1680           10 :   g_return_val_if_fail (config != NULL, FALSE);
    1681           10 :   g_return_val_if_fail (structure != NULL, FALSE);
    1682              : 
    1683           10 :   gst_tensors_config_init (config);
    1684              : 
    1685              :   /* get fixed size of text string from property */
    1686           10 :   text_size = self->tensors_info.info[0].dimension[0];
    1687           10 :   if (text_size == 0) {
    1688            0 :     GST_ERROR_OBJECT (self,
    1689              :         "Failed to get tensor info, need to update string size.");
    1690              : 
    1691            0 :     ml_loge
    1692              :         ("tensor_converter: Please set the property input-dim to convert stream manually for text streams unlike video streams. For example, input-dim=30 to handle up to 30 bytes of string per frame.");
    1693            0 :     return FALSE;
    1694              :   }
    1695              : 
    1696           10 :   format_string = gst_structure_get_string (structure, "format");
    1697           10 :   if (format_string) {
    1698           10 :     if (g_ascii_strcasecmp (format_string, "utf8") == 0) {
    1699           10 :       config->info.info[0].type = _NNS_UINT8;
    1700              :     } else {
    1701            0 :       GST_WARNING_OBJECT (self,
    1702              :           "For text streams, only utf8 streams are supported; format = \"%s\" is not supported.\n",
    1703              :           format_string);
    1704            0 :       return FALSE;
    1705              :     }
    1706              :   }
    1707              : 
    1708           10 :   config->info.num_tensors = 1;
    1709              : 
    1710              :   /* [size][frames] */
    1711              :   /* Fixed size of string, we cannot get the size from caps. */
    1712           10 :   config->info.info[0].dimension[0] = text_size;
    1713              : 
    1714              :   /* Supposed 1 frame in tensor, change dimension[1] if tensor contains N frames. */
    1715           10 :   config->info.info[0].dimension[1] = 1;
    1716          150 :   for (i = 2; i < NNS_TENSOR_RANK_LIMIT; i++)
    1717          140 :     config->info.info[0].dimension[i] = 0;
    1718              : 
    1719           10 :   if (gst_structure_has_field (structure, "framerate")) {
    1720            1 :     gst_structure_get_fraction (structure, "framerate", &config->rate_n,
    1721            1 :         &config->rate_d);
    1722              :   } else {
    1723              :     /* cannot get the framerate for text type */
    1724            9 :     config->rate_n = 0;
    1725            9 :     config->rate_d = 1;
    1726              :   }
    1727              : 
    1728           10 :   self->frame_size = gst_tensor_info_get_size (&config->info.info[0]);
    1729           10 :   return (config->info.info[0].type != _NNS_END);
    1730              : }
    1731              : 
    1732              : /**
    1733              :  * @brief Set the tensors configs structure from octet stream (internal static function)
    1734              :  * @param self this pointer to GstTensorConverter
    1735              :  * @param config tensors config structure to be filled
    1736              :  * @param structure caps structure
    1737              :  * @note Change tensors dimension and type.
    1738              :  * @return TRUE if supported type
    1739              :  */
    1740              : static gboolean
    1741          116 : gst_tensor_converter_parse_octet (GstTensorConverter * self,
    1742              :     GstTensorsConfig * config, const GstStructure * structure)
    1743              : {
    1744          116 :   GstTensorsInfo *info = &self->tensors_info;
    1745              :   GstTensorsConfig peer;
    1746              :   gboolean flexible, configured;
    1747              :   guint i;
    1748              : 
    1749          232 :   g_return_val_if_fail (config != NULL, FALSE);
    1750          116 :   g_return_val_if_fail (structure != NULL, FALSE);
    1751              : 
    1752          116 :   gst_tensors_config_init (config);
    1753          116 :   flexible = configured = FALSE;
    1754              : 
    1755              :   /* get possible tensors info from peer if no property is given */
    1756          116 :   if (!gst_tensors_info_validate (info)) {
    1757           22 :     if (gst_tensors_config_from_peer (self->srcpad, &peer, NULL)) {
    1758            6 :       flexible = gst_tensors_config_is_flexible (&peer);
    1759            6 :       configured = gst_tensors_info_validate (&peer.info);
    1760              : 
    1761            6 :       if (configured)
    1762            6 :         info = &peer.info;
    1763              :     }
    1764              : 
    1765           22 :     if (!flexible && !configured) {
    1766           16 :       GST_ERROR_OBJECT (self,
    1767              :           "Failed to get tensor info, need to update dimension and type.");
    1768              : 
    1769           16 :       ml_loge
    1770              :           ("tensor_converter: Please set the properties input-dim and input-type to convert application/stream to non-flexible other/tensors. Use other/tensors,format=flexible if you want flexible dimensions. For static (non-flexible) tensors, you may, for example, use input-dim=30,input-type=uint8 to handle 30 bytes of bin data as a single frame.");
    1771           16 :       return FALSE;
    1772              :     }
    1773              :   }
    1774              : 
    1775          100 :   if (self->frames_per_tensor > 1) {
    1776              :     /**
    1777              :      * Failure case when octet-stream has multi tensors and multi frames.
    1778              :      */
    1779            8 :     if (info->num_tensors > 1) {
    1780            4 :       ml_loge
    1781              :           ("tensor_converter: Cannot configure multiple tensors (num_tensors = %u) from an application/octet stream with frames_per_tensor (= %u)> 1. Please set the property frames-per-tensor 1 to convert stream to multiple-tensors (num_tensors > 1).",
    1782              :           info->num_tensors, self->frames_per_tensor);
    1783            4 :       return FALSE;
    1784              :     }
    1785            4 :     if (flexible) {
    1786            4 :       ml_loge
    1787              :           ("tensor_converter: Cannot configure flexible tensors from an application/octet stream with frames_per_tensor (%u) > 1. Please set the property frames-per-tensor 1 to convert stream to flexible tensors.",
    1788              :           self->frames_per_tensor);
    1789            4 :       return FALSE;
    1790              :     }
    1791              :   }
    1792              : 
    1793           92 :   if (gst_structure_has_field (structure, "framerate")) {
    1794            4 :     gst_structure_get_fraction (structure, "framerate", &config->rate_n,
    1795            4 :         &config->rate_d);
    1796              :   } else {
    1797              :     /* cannot get the framerate */
    1798           88 :     config->rate_n = 0;
    1799           88 :     config->rate_d = 1;
    1800              :   }
    1801              : 
    1802              :   /**
    1803              :    * Raw byte-stream (application/octet-stream)
    1804              :    * We cannot get the exact tensors info from caps.
    1805              :    * All tensors info should be updated.
    1806              :    * If output is flexible, dimension should be updated in chain function with buffer size.
    1807              :    * (data format for tensor: [size])
    1808              :    */
    1809           92 :   if (flexible) {
    1810            1 :     config->info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
    1811              : 
    1812            1 :     config->info.num_tensors = 1;
    1813            1 :     config->info.info[0].type = _NNS_UINT8;
    1814            1 :     config->info.info[0].dimension[0] = 1;
    1815           16 :     for (i = 1; i < NNS_TENSOR_RANK_LIMIT; i++)
    1816           15 :       config->info.info[0].dimension[i] = 0;
    1817              :   } else {
    1818           91 :     gst_tensors_info_copy (&config->info, info);
    1819           91 :     self->frame_size = gst_tensors_info_get_size (&config->info, -1);
    1820              :   }
    1821              : 
    1822           92 :   return TRUE;
    1823              : }
    1824              : 
    1825              : /**
    1826              :  * @brief Set the tensors configs structure from fliex tensor stream (internal static function)
    1827              :  * @param self this pointer to GstTensorConverter
    1828              :  * @param config tensors config structure to be filled
    1829              :  * @param structure caps structure
    1830              :  * @note Change tensors dimension and type.
    1831              :  * @return TRUE if supported type
    1832              :  */
    1833              : static gboolean
    1834            5 : gst_tensor_converter_parse_tensor (GstTensorConverter * self,
    1835              :     GstTensorsConfig * config, const GstStructure * structure)
    1836              : {
    1837            5 :   GstTensorsInfo *info = &self->tensors_info;
    1838              :   guint i;
    1839              : 
    1840            5 :   g_return_val_if_fail (config != NULL, FALSE);
    1841            5 :   g_return_val_if_fail (structure != NULL, FALSE);
    1842              : 
    1843            5 :   gst_tensors_config_init (config);
    1844              : 
    1845            5 :   if (self->frames_per_tensor > 1) {
    1846            0 :     ml_loge
    1847              :         ("tensor_converter: Cannot configure multiple tensors. Please set the property frames-per-tensor (%u != 1) 1 to convert stream into tensor stream with num_tensor > 1.",
    1848              :         self->frames_per_tensor);
    1849            0 :     return FALSE;
    1850              :   }
    1851              : 
    1852              :   /* update tensor info from properties */
    1853            5 :   if (gst_tensors_info_validate (info)) {
    1854            3 :     gst_tensors_info_copy (&config->info, info);
    1855            3 :     self->frame_size = gst_tensors_info_get_size (&config->info, -1);
    1856              :   } else {
    1857              :     /**
    1858              :      * We cannot get the exact tensors info from caps.
    1859              :      * All tensors info should be updated in chain function.
    1860              :      * (data format for tensor: [size])
    1861              :      */
    1862            2 :     config->info.num_tensors = 1;
    1863            2 :     config->info.info[0].type = _NNS_UINT8;
    1864            2 :     config->info.info[0].dimension[0] = 1;
    1865           32 :     for (i = 1; i < NNS_TENSOR_RANK_LIMIT; i++)
    1866           30 :       config->info.info[0].dimension[i] = 0;
    1867              :   }
    1868              : 
    1869            5 :   if (gst_structure_has_field (structure, "framerate")) {
    1870            5 :     gst_structure_get_fraction (structure, "framerate", &config->rate_n,
    1871            5 :         &config->rate_d);
    1872              :   } else {
    1873              :     /* cannot get the framerate */
    1874            0 :     config->rate_n = 0;
    1875            0 :     config->rate_d = 1;
    1876              :   }
    1877              : 
    1878            5 :   return TRUE;
    1879              : }
    1880              : 
    1881              : /**
    1882              :  * @brief Set the tensors config structure from caps (internal static function for custom mode)
    1883              :  * @param self this pointer to GstTensorConverter
    1884              :  * @param config tensors config structure to be filled
    1885              :  * @param caps incoming caps
    1886              :  * @return TRUE if supported type
    1887              :  */
    1888              : static gboolean
    1889           76 : gst_tensor_converter_parse_custom (GstTensorConverter * self,
    1890              :     GstTensorsConfig * config, const GstCaps * caps)
    1891              : {
    1892              :   GstStructure *structure;
    1893              :   const gchar *mimetype;
    1894           76 :   gboolean is_fixed = FALSE;
    1895              : 
    1896          152 :   g_return_val_if_fail (config != NULL, FALSE);
    1897           76 :   g_return_val_if_fail (gst_caps_is_fixed (caps), FALSE);
    1898              : 
    1899           76 :   gst_tensors_config_from_peer (self->srcpad, config, &is_fixed);
    1900              : 
    1901           76 :   structure = gst_caps_get_structure (caps, 0);
    1902           76 :   mimetype = gst_structure_get_name (structure);
    1903              : 
    1904           76 :   if (self->mode == _CONVERTER_MODE_CUSTOM_CODE) {
    1905            1 :     if (!is_fixed) {
    1906            1 :       gst_tensors_config_init (config);
    1907              :       /* All tensor info should be updated later in chain function. */
    1908            1 :       config->info.num_tensors = 1;
    1909            1 :       config->info.info[0].type = _NNS_UINT8;
    1910            1 :       gst_tensor_parse_dimension ("1:1:1:1", config->info.info[0].dimension);
    1911              : 
    1912            1 :       if (gst_structure_has_field (structure, "framerate")) {
    1913            0 :         gst_structure_get_fraction (structure, "framerate", &config->rate_n,
    1914            0 :             &config->rate_d);
    1915              :       } else {
    1916              :         /* cannot get the framerate */
    1917            1 :         config->rate_n = 0;
    1918            1 :         config->rate_d = 1;
    1919              :       }
    1920              :     }
    1921           75 :   } else if (!self->externalConverter) {
    1922              :     const NNStreamerExternalConverter *ex;
    1923           75 :     if (self->mode == _CONVERTER_MODE_CUSTOM_SCRIPT) {
    1924           18 :       mimetype = self->ext_fw;
    1925              :     }
    1926           75 :     if (!(ex = findExternalConverter (mimetype))) {
    1927            0 :       ml_loge
    1928              :           ("tensor_converter: Failed to get external converter for %s. Cannot find a corresponding external converter for the given type. The custom converter mode is %s with \"%s\"",
    1929              :           mimetype, STRING_CUSTOM_MODE (self), self->mode_option);
    1930            0 :       return FALSE;
    1931              :     }
    1932              : 
    1933           75 :     if (!is_fixed) {
    1934           75 :       if (!ex->get_out_config) {
    1935            0 :         ml_loge
    1936              :             ("tensor_converter: Failed to get tensors info from %s (%s:%s). Its corresponding external converter is found. but its mandatory callback, get_out_config is not available.",
    1937              :             mimetype, STRING_CUSTOM_MODE (self), self->mode_option);
    1938            0 :         return FALSE;
    1939              :       }
    1940           75 :       if (!ex->get_out_config (caps, config)) {
    1941            0 :         char *capstr = gst_caps_to_string (caps);
    1942            0 :         ml_loge
    1943              :             ("tensor_converter: Failed to get tensors info from %s (%s:%s). Its corresponding external converter is found. but its mandatory callback, get_out_config(\"%s\", config) has returned FALSE (cannot get config from the caps).",
    1944              :             mimetype, STRING_CUSTOM_MODE (self), self->mode_option, capstr);
    1945            0 :         g_free (capstr);
    1946            0 :         return FALSE;
    1947              :       }
    1948              :     }
    1949              : 
    1950           75 :     self->externalConverter = ex;
    1951           75 :     if (self->mode == _CONVERTER_MODE_CUSTOM_SCRIPT) {
    1952           18 :       int ret = 0;
    1953           18 :       if (self->externalConverter->open &&
    1954           18 :           (ret = self->externalConverter->open (self->mode_option,
    1955              :                   &self->priv_data)) < 0) {
    1956            6 :         ml_loge
    1957              :             ("tensor_converter: Failed to open tensor converter custom subplugin: custom-script mode with \"%s\" for \"%s\" (%s) has 'open' callback; however, it has returned %d error.\n",
    1958              :             self->mode_option, mimetype, self->externalConverter->name, ret);
    1959            6 :         self->externalConverter = NULL;
    1960            6 :         return FALSE;
    1961              :       }
    1962              :     }
    1963              :   }
    1964              : 
    1965           70 :   return TRUE;
    1966              : }
    1967              : 
    1968              : /**
    1969              :  * @brief Get possible media-caps from downstream element.
    1970              :  */
    1971              : static GstCaps *
    1972         4178 : gst_tensor_converter_get_possible_media_caps (GstTensorConverter * self)
    1973              : {
    1974         4178 :   GstCaps *media_caps = NULL;
    1975              :   GstTensorsConfig config;
    1976              : 
    1977              :   /* get possible caps from downstream element */
    1978         4178 :   if (gst_tensors_config_from_peer (self->srcpad, &config, NULL)) {
    1979              :     GstStructure *st;
    1980              :     guint i, caps_len;
    1981              :     media_type type;
    1982              : 
    1983              :     /* convert peer caps to possible media caps */
    1984         2137 :     media_caps = gst_pad_get_pad_template_caps (self->sinkpad);
    1985         2137 :     media_caps = gst_caps_make_writable (media_caps);
    1986              : 
    1987         2137 :     caps_len = gst_caps_get_size (media_caps);
    1988              : 
    1989        21370 :     for (i = 0; i < caps_len; ++i) {
    1990        19233 :       st = gst_caps_get_structure (media_caps, i);
    1991        19233 :       type = gst_structure_get_media_type (st);
    1992              : 
    1993        19233 :       switch (type) {
    1994         2137 :         case _NNS_VIDEO:
    1995              :           /* video caps from tensor info */
    1996              :           if (is_video_supported (self)) {
    1997         2137 :             GValue supported_formats = G_VALUE_INIT;
    1998              :             gint colorspace, width, height;
    1999              : 
    2000         2137 :             colorspace = config.info.info[0].dimension[0];
    2001         2137 :             width = config.info.info[0].dimension[1];
    2002         2137 :             height = config.info.info[0].dimension[2];
    2003              : 
    2004         2137 :             switch (colorspace) {
    2005            9 :               case 1:
    2006            9 :                 gst_tensor_converter_get_format_list (&supported_formats,
    2007              :                     "GRAY8", "GRAY16_BE", "GRAY16_LE", NULL);
    2008            9 :                 break;
    2009          303 :               case 3:
    2010          303 :                 gst_tensor_converter_get_format_list (&supported_formats,
    2011              :                     "RGB", "BGR", NULL);
    2012          303 :                 break;
    2013            0 :               case 4:
    2014            0 :                 gst_tensor_converter_get_format_list (&supported_formats,
    2015              :                     "RGBx", "BGRx", "xRGB", "xBGR", "RGBA", "BGRA", "ARGB",
    2016              :                     "ABGR", NULL);
    2017            0 :                 break;
    2018         1825 :               default:
    2019              :                 /* unsupported format, set default video formats */
    2020         1825 :                 break;
    2021              :             }
    2022              : 
    2023         2137 :             if (G_VALUE_TYPE (&supported_formats) == GST_TYPE_LIST &&
    2024          312 :                 gst_value_list_get_size (&supported_formats) > 0) {
    2025          312 :               gst_structure_set_value (st, "format", &supported_formats);
    2026              :             }
    2027         2137 :             g_value_unset (&supported_formats);
    2028              : 
    2029         2137 :             if (width > 0) {
    2030          317 :               gst_structure_set (st, "width", G_TYPE_INT, width, NULL);
    2031              :             }
    2032              : 
    2033         2137 :             if (height > 0) {
    2034          308 :               gst_structure_set (st, "height", G_TYPE_INT, height, NULL);
    2035              :             }
    2036              : 
    2037         2137 :             if (config.rate_n >= 0 && config.rate_d > 0) {
    2038          315 :               gst_structure_set (st, "framerate", GST_TYPE_FRACTION,
    2039              :                   config.rate_n, config.rate_d, NULL);
    2040              :             }
    2041              : 
    2042              :             /* add new structure for NCHW formats */
    2043              : #if GST_CHECK_VERSION(1, 20, 0)
    2044         2137 :             width = config.info.info[0].dimension[0];
    2045         2137 :             height = config.info.info[0].dimension[1];
    2046         2137 :             colorspace = config.info.info[0].dimension[2];
    2047              : 
    2048         2137 :             if (colorspace == 3) {
    2049            5 :               GValue nchw_format = G_VALUE_INIT;
    2050            5 :               GstStructure *nchw_st = gst_structure_copy (st);
    2051              : 
    2052            5 :               gst_tensor_converter_get_format_list (&nchw_format,
    2053              :                   "RGBP", "BGRP", NULL);
    2054              : 
    2055            5 :               if (G_VALUE_TYPE (&nchw_format) == GST_TYPE_LIST &&
    2056            5 :                   gst_value_list_get_size (&nchw_format) > 0) {
    2057            5 :                 gst_structure_set_value (nchw_st, "format", &nchw_format);
    2058              :               }
    2059            5 :               g_value_unset (&nchw_format);
    2060              : 
    2061            5 :               if (width > 0) {
    2062            5 :                 gst_structure_set (nchw_st, "width", G_TYPE_INT, width, NULL);
    2063              :               }
    2064              : 
    2065            5 :               if (height > 0) {
    2066            5 :                 gst_structure_set (nchw_st, "height", G_TYPE_INT, height, NULL);
    2067              :               }
    2068            5 :               gst_caps_append_structure (media_caps, nchw_st);
    2069              :             }
    2070              : #endif
    2071              :           }
    2072         2137 :           break;
    2073         2137 :         case _NNS_AUDIO:
    2074              :           /* audio caps from tensor info */
    2075         2137 :           if (is_audio_supported (self)
    2076         2137 :               && config.info.info[0].type != _NNS_END) {
    2077              :             gint ch, rate;
    2078              :             GstAudioFormat aformat;
    2079              : 
    2080          320 :             switch (config.info.info[0].type) {
    2081            0 :               case _NNS_INT8:
    2082            0 :                 aformat = GST_AUDIO_FORMAT_S8;
    2083            0 :                 break;
    2084          263 :               case _NNS_UINT8:
    2085          263 :                 aformat = GST_AUDIO_FORMAT_U8;
    2086          263 :                 break;
    2087            0 :               case _NNS_INT16:
    2088            0 :                 aformat = GST_AUDIO_FORMAT_S16;
    2089            0 :                 break;
    2090            0 :               case _NNS_UINT16:
    2091            0 :                 aformat = GST_AUDIO_FORMAT_U16;
    2092            0 :                 break;
    2093            0 :               case _NNS_INT32:
    2094            0 :                 aformat = GST_AUDIO_FORMAT_S32;
    2095            0 :                 break;
    2096            0 :               case _NNS_UINT32:
    2097            0 :                 aformat = GST_AUDIO_FORMAT_U32;
    2098            0 :                 break;
    2099            9 :               case _NNS_FLOAT16:
    2100            9 :                 aformat = GST_AUDIO_FORMAT_UNKNOWN;
    2101            9 :                 ml_loge
    2102              :                     ("tensor_converter: Audio stream cannot be converted to float16 stream directly because GStreamer's standard audio streams do not support float16. Try Float32 or Float64 instead and 'transform' it to Float16 later.\n");
    2103            9 :                 break;
    2104           48 :               case _NNS_FLOAT32:
    2105           48 :                 aformat = GST_AUDIO_FORMAT_F32;
    2106           48 :                 break;
    2107            0 :               case _NNS_FLOAT64:
    2108            0 :                 aformat = GST_AUDIO_FORMAT_F64;
    2109            0 :                 break;
    2110            0 :               default:
    2111              :                 /* unsupported format */
    2112            0 :                 aformat = GST_AUDIO_FORMAT_UNKNOWN;
    2113            0 :                 break;
    2114              :             }
    2115              : 
    2116          320 :             if (aformat != GST_AUDIO_FORMAT_UNKNOWN) {
    2117          311 :               gst_structure_set (st, "format", G_TYPE_STRING,
    2118              :                   gst_audio_format_to_string (aformat), NULL);
    2119              : 
    2120          311 :               if ((ch = config.info.info[0].dimension[0]) > 0) {
    2121          306 :                 gst_structure_set (st, "channels", G_TYPE_INT, ch, NULL);
    2122              :               }
    2123              : 
    2124          311 :               if ((rate = config.rate_n) > 0) {
    2125            3 :                 gst_structure_set (st, "rate", G_TYPE_INT, rate, NULL);
    2126              :               }
    2127              :             }
    2128              :           }
    2129         2137 :           break;
    2130        14959 :         default:
    2131              :           /* do nothing for text and octet stream */
    2132        14959 :           break;
    2133              :       }
    2134              :     }
    2135              :   }
    2136              : 
    2137         4178 :   return media_caps;
    2138              : }
    2139              : 
    2140              : /**
    2141              :  * @brief Get pad caps for caps negotiation.
    2142              :  */
    2143              : static GstCaps *
    2144         7594 : gst_tensor_converter_query_caps (GstTensorConverter * self, GstPad * pad,
    2145              :     GstCaps * filter)
    2146              : {
    2147              :   GstCaps *caps;
    2148              : 
    2149         7594 :   caps = gst_pad_get_current_caps (pad);
    2150         7594 :   if (!caps) {
    2151         7074 :     caps = gst_pad_get_pad_template_caps (pad);
    2152              :   }
    2153              : 
    2154         7594 :   if (pad == self->sinkpad) {
    2155              :     GstCaps *media_caps;
    2156              : 
    2157         4178 :     media_caps = gst_tensor_converter_get_possible_media_caps (self);
    2158         4178 :     if (media_caps) {
    2159              :       /* intersect with pad caps */
    2160         2137 :       GstCaps *tmp = gst_caps_intersect_full (media_caps, caps,
    2161              :           GST_CAPS_INTERSECT_FIRST);
    2162         2137 :       gst_caps_unref (caps);
    2163         2137 :       caps = tmp;
    2164              : 
    2165         2137 :       gst_caps_unref (media_caps);
    2166              :     }
    2167              :   }
    2168              : 
    2169         7594 :   silent_debug_caps (self, caps, "caps");
    2170         7594 :   silent_debug_caps (self, filter, "filter");
    2171              : 
    2172         7594 :   if (filter) {
    2173              :     GstCaps *intersection;
    2174              : 
    2175              :     intersection =
    2176         2173 :         gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
    2177              : 
    2178         2173 :     gst_caps_unref (caps);
    2179         2173 :     caps = intersection;
    2180              :   }
    2181              : 
    2182         7594 :   silent_debug_caps (self, caps, "result");
    2183         7594 :   return caps;
    2184              : }
    2185              : 
    2186              : /**
    2187              :  * @brief Parse caps and set tensors info.
    2188              :  */
    2189              : static gboolean
    2190          864 : gst_tensor_converter_parse_caps (GstTensorConverter * self,
    2191              :     const GstCaps * caps)
    2192              : {
    2193              :   GstStructure *structure;
    2194              :   GstTensorsConfig config;
    2195              :   media_type in_type;
    2196          864 :   gint frames_dim = -1; /** dimension index of frames in configured tensors */
    2197              : 
    2198         1728 :   g_return_val_if_fail (caps != NULL, FALSE);
    2199          864 :   g_return_val_if_fail (gst_caps_is_fixed (caps), FALSE);
    2200              : 
    2201          864 :   structure = gst_caps_get_structure (caps, 0);
    2202          864 :   if (self->mode != _CONVERTER_MODE_NONE) {
    2203           19 :     in_type = _NNS_MEDIA_ANY;
    2204              :   } else {
    2205          845 :     in_type = gst_structure_get_media_type (structure);
    2206              :   }
    2207              : 
    2208          864 :   switch (in_type) {
    2209          620 :     case _NNS_VIDEO:
    2210              :       if (is_video_supported (self)) {
    2211          620 :         if (!gst_tensor_converter_parse_video (self, caps, &config)) {
    2212            0 :           char *capstr = gst_caps_to_string (caps);
    2213            0 :           GST_ERROR_OBJECT (self,
    2214              :               "Failed to configure tensor from gst cap \"%s\" for video streams.",
    2215              :               capstr);
    2216            0 :           g_free (capstr);
    2217            0 :           return FALSE;
    2218              :         }
    2219              : 
    2220          620 :         frames_dim = 3;
    2221              :       } else {
    2222              :         ml_loge
    2223              :             ("tensor_converter: This binary does not support video type. Please build NNStreamer with -Dvideo-support=enabled (default). You have configured -Dvideo-support=disabled when you build this binary.\n");
    2224              :         return FALSE;
    2225              :       }
    2226          620 :       break;
    2227           37 :     case _NNS_AUDIO:
    2228              :       if (is_audio_supported (self)) {
    2229           37 :         if (!gst_tensor_converter_parse_audio (self, caps, &config)) {
    2230            0 :           char *capstr = gst_caps_to_string (caps);
    2231            0 :           GST_ERROR_OBJECT (self,
    2232              :               "Failed to configure tensor from gst cap \"%s\" for audio streams.",
    2233              :               capstr);
    2234            0 :           g_free (capstr);
    2235            0 :           return FALSE;
    2236              :         }
    2237              : 
    2238           37 :         frames_dim = 1;
    2239              :       } else {
    2240              :         ml_loge
    2241              :             ("tensor_converter: This binary does not support audio type. Please build NNStreamer with -Daudio-support=enabled (default). You have configured -Daudio-support=disabled when you build this binary.\n");
    2242              :         return FALSE;
    2243              :       }
    2244           37 :       break;
    2245           10 :     case _NNS_TEXT:
    2246           10 :       if (!gst_tensor_converter_parse_text (self, &config, structure)) {
    2247            0 :         char *capstr = gst_caps_to_string (caps);
    2248            0 :         GST_ERROR_OBJECT (self,
    2249              :             "Failed to configure tensor from gst cap \"%s\" for text streams.",
    2250              :             capstr);
    2251            0 :         g_free (capstr);
    2252            0 :         return FALSE;
    2253              :       }
    2254              : 
    2255           10 :       frames_dim = 1;
    2256           10 :       break;
    2257          116 :     case _NNS_OCTET:
    2258          116 :       if (!gst_tensor_converter_parse_octet (self, &config, structure)) {
    2259           24 :         char *capstr = gst_caps_to_string (caps);
    2260           24 :         GST_ERROR_OBJECT (self,
    2261              :             "Failed to configure tensors from gst cap \"%s\" for octet streams.",
    2262              :             capstr);
    2263           24 :         g_free (capstr);
    2264           24 :         return FALSE;
    2265              :       }
    2266           92 :       break;
    2267            5 :     case _NNS_TENSOR:
    2268              :       /* flexible tensor to static tensor stream */
    2269            5 :       if (!gst_tensor_converter_parse_tensor (self, &config, structure)) {
    2270            0 :         char *capstr = gst_caps_to_string (caps);
    2271            0 :         GST_ERROR_OBJECT (self,
    2272              :             "Failed to configure tensor from flexible tensor (%s); trying to convert to static tensor.",
    2273              :             capstr);
    2274            0 :         g_free (capstr);
    2275            0 :         return FALSE;
    2276              :       }
    2277            5 :       break;
    2278           76 :     default:
    2279           76 :       if (!gst_tensor_converter_parse_custom (self, &config, caps)) {
    2280            6 :         char *capstr = gst_caps_to_string (caps);
    2281            6 :         GST_ERROR_OBJECT (self,
    2282              :             "Failed to configure tensors with custom mode from streams of gst cap (%s) with custom converter subplugins.",
    2283              :             capstr);
    2284            6 :         g_free (capstr);
    2285            6 :         return FALSE;
    2286              :       }
    2287           70 :       in_type = _NNS_MEDIA_ANY;
    2288           70 :       break;
    2289              :   }
    2290              : 
    2291              :   /** set the number of frames in dimension */
    2292          834 :   if (frames_dim >= 0) {
    2293          667 :     config.info.info[0].dimension[frames_dim] = self->frames_per_tensor;
    2294              :   }
    2295              : 
    2296          834 :   if (!gst_tensors_config_validate (&config)) {
    2297              :     /** not fully configured. the resulting config is weird. */
    2298            0 :     char *capstr = gst_caps_to_string (caps);
    2299            0 :     char *cfgstr = gst_tensors_config_to_string (&config);
    2300            0 :     GST_ERROR_OBJECT (self,
    2301              :         "Failed to configure tensors info with gst cap (%s). Cannot validate tensor configuration acquired from the given gst cap. The resulting invalid tensor config is: %s\n",
    2302              :         capstr, cfgstr);
    2303            0 :     g_free (capstr);
    2304            0 :     g_free (cfgstr);
    2305            0 :     return FALSE;
    2306              :   }
    2307              : 
    2308          834 :   if (gst_tensors_info_validate (&self->tensors_info)) {
    2309              :     /** compare tensor info */
    2310           93 :     if (!gst_tensors_info_is_equal (&self->tensors_info, &config.info)) {
    2311            0 :       gchar *str1 = gst_tensors_info_to_string (&self->tensors_info);
    2312            0 :       gchar *str2 = gst_tensors_info_to_string (&config.info);
    2313            0 :       GST_ERROR_OBJECT (self,
    2314              :           "Failed, mismatched tensor info. The two tensor configuration should match: \"configured by properties and tensor output\": %s and \"configured by input stream\": %s\n",
    2315              :           str1, str2);
    2316            0 :       g_free (str1);
    2317            0 :       g_free (str2);
    2318            0 :       return FALSE;
    2319              :     }
    2320              :   }
    2321              : 
    2322          834 :   self->in_media_type = in_type;
    2323          834 :   self->tensors_configured = TRUE;
    2324          834 :   self->tensors_config = config;
    2325              : 
    2326          834 :   return TRUE;
    2327              : }
    2328              : 
    2329              : /**
    2330              :  * @brief Update src pad caps from tensors config.
    2331              :  */
    2332              : static void
    2333          908 : gst_tensor_converter_update_caps (GstTensorConverter * self)
    2334              : {
    2335              :   GstTensorsConfig *config;
    2336              :   GstCaps *curr_caps, *out_caps;
    2337              : 
    2338          908 :   config = &self->tensors_config;
    2339          908 :   out_caps = gst_tensor_pad_caps_from_config (self->srcpad, config);
    2340              : 
    2341              :   /* Update src pad caps if it is different. */
    2342          908 :   curr_caps = gst_pad_get_current_caps (self->srcpad);
    2343          908 :   if (curr_caps == NULL || !gst_caps_is_equal (curr_caps, out_caps)) {
    2344          908 :     silent_debug_caps (self, out_caps, "set out-caps");
    2345          908 :     gst_pad_set_caps (self->srcpad, out_caps);
    2346              :   }
    2347              : 
    2348          908 :   if (curr_caps)
    2349           74 :     gst_caps_unref (curr_caps);
    2350              : 
    2351          908 :   gst_caps_unref (out_caps);
    2352          908 : }
    2353              : 
    2354              : /**
    2355              :  * @brief Find converter sub-plugin with the name.
    2356              :  * @param[in] name The name of converter sub-plugin.
    2357              :  * @return NULL if not found or the sub-plugin object has an error.
    2358              :  */
    2359              : const NNStreamerExternalConverter *
    2360         1884 : nnstreamer_converter_find (const char *name)
    2361              : {
    2362         1884 :   return get_subplugin (NNS_SUBPLUGIN_CONVERTER, name);
    2363              : }
    2364              : 
    2365              : /**
    2366              :  * @brief Validate converter sub-plugin's data.
    2367              :  */
    2368              : static gboolean
    2369         1687 : nnstreamer_converter_validate (const NNStreamerExternalConverter * converter)
    2370              : {
    2371         1687 :   if (!converter || !converter->name) {
    2372              :     /* invalid name */
    2373            2 :     return FALSE;
    2374              :   }
    2375              : 
    2376         1685 :   if (!converter->query_caps || !converter->get_out_config
    2377         1683 :       || !converter->convert) {
    2378              :     /* invalid methods in converter sub-plugin */
    2379            3 :     return FALSE;
    2380              :   }
    2381              : 
    2382         1682 :   return TRUE;
    2383              : }
    2384              : 
    2385              : /**
    2386              :  * @brief Converter's external subplugins should call this at init.
    2387              :  */
    2388              : int
    2389         1687 : registerExternalConverter (NNStreamerExternalConverter * ex)
    2390              : {
    2391         1687 :   g_return_val_if_fail (nnstreamer_converter_validate (ex), FALSE);
    2392         1682 :   return register_subplugin (NNS_SUBPLUGIN_CONVERTER, ex->name, ex);
    2393              : }
    2394              : 
    2395              : /**
    2396              :  * @brief Converter's external subplugins should call this at exit.
    2397              :  */
    2398              : void
    2399         1681 : unregisterExternalConverter (const char *name)
    2400              : {
    2401         1681 :   unregister_subplugin (NNS_SUBPLUGIN_CONVERTER, name);
    2402         1681 : }
    2403              : 
    2404              : /**
    2405              :  * @brief Internal static function to find registered subplugins.
    2406              :  */
    2407              : static const NNStreamerExternalConverter *
    2408           75 : findExternalConverter (const char *media_type)
    2409              : {
    2410              :   gchar **str_array;
    2411              :   guint total, i, j, caps_size;
    2412              :   GstCaps *caps;
    2413              :   const gchar *caps_name;
    2414              :   const NNStreamerExternalConverter *ex;
    2415              : 
    2416           75 :   str_array = get_all_subplugins (NNS_SUBPLUGIN_CONVERTER);
    2417           75 :   if (str_array) {
    2418           75 :     total = g_strv_length (str_array);
    2419              : 
    2420          188 :     for (i = 0; i < total; i++) {
    2421          188 :       ex = nnstreamer_converter_find (str_array[i]);
    2422              : 
    2423          188 :       if (g_strcmp0 (media_type, str_array[i]) == 0) {
    2424              :         /* found matched media type */
    2425           18 :         g_strfreev (str_array);
    2426           18 :         return ex;
    2427              :       }
    2428              : 
    2429          170 :       if (ex && ex->query_caps) {
    2430          170 :         caps = ex->query_caps (NULL);
    2431          170 :         caps_size = gst_caps_get_size (caps);
    2432              : 
    2433          283 :         for (j = 0; j < caps_size; j++) {
    2434          170 :           caps_name = gst_structure_get_name (gst_caps_get_structure (caps, j));
    2435          170 :           if (g_strcmp0 (media_type, caps_name) == 0) {
    2436              :             /* found matched media type */
    2437           57 :             gst_caps_unref (caps);
    2438           57 :             g_strfreev (str_array);
    2439           57 :             return ex;
    2440              :           }
    2441              :         }
    2442              : 
    2443          113 :         gst_caps_unref (caps);
    2444              :       }
    2445              :     }
    2446              : 
    2447            0 :     g_strfreev (str_array);
    2448              :   }
    2449              : 
    2450            0 :   return NULL;
    2451              : }
    2452              : 
    2453              : /**
    2454              :  * @brief set custom property description for tensor converter sub-plugin
    2455              :  */
    2456              : void
    2457            0 : nnstreamer_converter_set_custom_property_desc (const char *name,
    2458              :     const char *prop, ...)
    2459              : {
    2460              :   va_list varargs;
    2461              : 
    2462            0 :   va_start (varargs, prop);
    2463            0 :   subplugin_set_custom_property_desc (NNS_SUBPLUGIN_CONVERTER, name, prop,
    2464              :       varargs);
    2465            0 :   va_end (varargs);
    2466            0 : }
    2467              : 
    2468              : /**
    2469              :  * @brief Registers a callback for tensor_converter custom condition
    2470              :  * @return 0 if success. -ERRNO if error.
    2471              :  */
    2472              : int
    2473            5 : nnstreamer_converter_custom_register (const gchar * name,
    2474              :     tensor_converter_custom func, void *data)
    2475              : {
    2476              :   converter_custom_cb_s *ptr;
    2477              : 
    2478            5 :   g_return_val_if_fail (name && strlen (name), -EINVAL);
    2479            4 :   g_return_val_if_fail (func, -EINVAL);
    2480              : 
    2481            3 :   ptr = g_new0 (converter_custom_cb_s, 1);
    2482            3 :   ptr->func = func;
    2483            3 :   ptr->data = data;
    2484              : 
    2485            3 :   if (register_subplugin (NNS_CUSTOM_CONVERTER, name, ptr))
    2486            2 :     return 0;
    2487              : 
    2488            1 :   g_free (ptr);
    2489            1 :   ml_loge
    2490              :       ("tensor_converter: cannot register a converter subplugin, \"%s\" function. register_subplugin () has failed to register \"%s\".",
    2491              :       name, name);
    2492            1 :   return -EINVAL;
    2493              : }
    2494              : 
    2495              : /**
    2496              :  * @brief Unregisters a callback for tensor_converter custom condition
    2497              :  * @return 0 if success. -ERRNO if error.
    2498              :  */
    2499              : int
    2500            4 : nnstreamer_converter_custom_unregister (const gchar * name)
    2501              : {
    2502              :   converter_custom_cb_s *ptr;
    2503              : 
    2504            4 :   ptr = (converter_custom_cb_s *) get_subplugin (NNS_CUSTOM_CONVERTER, name);
    2505            4 :   if (!unregister_subplugin (NNS_CUSTOM_CONVERTER, name)) {
    2506            2 :     ml_loge ("tensor_converter: Failed to unregister custom callback %s.",
    2507              :         name);
    2508            2 :     return -EINVAL;
    2509              :   }
    2510            2 :   g_free (ptr);
    2511              : 
    2512            2 :   return 0;
    2513              : }
        

Generated by: LCOV version 2.0-1