LCOV - code coverage report
Current view: top level - nnstreamer-2.4.2/gst/nnstreamer/elements - gsttensor_merge.c (source / functions) Coverage Total Hit
Test: nnstreamer 2.4.2-0 nnstreamer/nnstreamer#eca68b8d050408568af95d831a8eef62aaee7784 Lines: 85.4 % 396 338
Test Date: 2025-03-14 05:36:58 Functions: 100.0 % 21 21

            Line data    Source code
       1              : /**
       2              :  * GStreamer
       3              :  * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
       4              :  * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
       5              :  * Copyright (C) 2018 Jijoong Moon <jijoong.moon@samsung.com>
       6              :  *
       7              :  *
       8              :  * This library is free software; you can redistribute it and/or
       9              :  * modify it under the terms of the GNU Library General Public
      10              :  * License as published by the Free Software Foundation;
      11              :  * version 2.1 of the License.
      12              :  *
      13              :  * This library is distributed in the hope that it will be useful,
      14              :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      15              :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      16              :  * Library General Public License for more details.
      17              :  *
      18              :  */
      19              : /**
      20              :  * @file        gsttensor_merge.c
      21              :  * @date        03 July 2018
      22              :  * @brief       GStreamer plugin to merge tensors (as a filter for other general neural network filters)
      23              :  * @see         https://github.com/nnstreamer/nnstreamer
      24              :  * @author      Jijoong Moon <jijoong.moon@samsung.com>
      25              :  * @bug         No known bugs except for NYI items
      26              :  *
      27              :  */
      28              : 
      29              : /**
      30              :  * SECTION:element-tensormerge
      31              :  *
      32              :  * A Merger that merge tensor stream to tensor stream for NN frameworks.
      33              :  * The output is always in the format of other/tensor
      34              :  *
      35              :  * <refsect2>
      36              :  * <title>Example launch line</title>
      37              :  * |[
      38              :  * gst-launch -v -m tensor_merge name=merge ! fakesink
      39              :  * filesrc location=b.png ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,width=100,height=100,framerate=0/1  ! tensor_converter ! merge.sink_0
      40              :  * filesrc location=b.png ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,width=100,height=100,framerate=0/1  ! tensor_converter ! merge.sink_1
      41              :  * filesrc location=b.png ! pngdec ! videoscale ! imagefreeze ! videoconvert ! video/x-raw,format=RGB,width=100,height=100,framerate=0/1  ! tensor_converter ! merge.sink_2
      42              :  * ]|
      43              :  *
      44              :  * |[
      45              :  * gst-launch -v -m tensor_merge name=merge ! filesink location=merge.log
      46              :  * multifilesrc location="testsequence_%1d.png" index=0 caps="image/png, framerate=(fraction)30/1" ! pngdec ! tensor_converter ! merge.sink_0
      47              :  * multifilesrc location="testsequence_%1d.png" index=0 caps="image/png, framerate=(fraction)30/1" ! pngdec ! tensor_converter ! merge.sink_1
      48              :  * multifilesrc location="testsequence_%1d.png" index=0 caps="image/png, framerate=(fraction)30/1" ! pngdec ! tensor_converter ! merge.sink_2
      49              :  *
      50              :  * </refsect2>
      51              :  *
      52              :  */
      53              : 
      54              : 
      55              : #ifdef HAVE_CONFIG_H
      56              : #include <config.h>
      57              : #endif
      58              : 
      59              : #include <string.h>
      60              : #include <gst/gst.h>
      61              : #include <glib.h>
      62              : #include <nnstreamer_util.h>
      63              : 
      64              : #include "gsttensor_merge.h"
      65              : 
      66              : GST_DEBUG_CATEGORY_STATIC (gst_tensor_merge_debug);
      67              : #define GST_CAT_DEFAULT gst_tensor_merge_debug
      68              : 
      69              : /**
      70              :  * @brief Macro for debug mode.
      71              :  */
      72              : #ifndef DBG
      73              : #define DBG (!tensor_merge->silent)
      74              : #endif
      75              : 
      76              : enum
      77              : {
      78              :   PROP_0,
      79              :   PROP_MODE,
      80              :   PROP_OPTION,
      81              :   PROP_SYNC_MODE,
      82              :   PROP_SYNC_OPTION,
      83              :   PROP_SILENT,
      84              : };
      85              : 
      86              : static const gchar *gst_tensor_merge_mode_string[] = {
      87              :   [GTT_LINEAR] = "linear",
      88              :   [GTT_END] = "error",
      89              : };
      90              : 
      91              : static const gchar *gst_tensor_merge_linear_string[] = {
      92              :   [LINEAR_FIRST] = "0",
      93              :   [LINEAR_SECOND] = "1",
      94              :   [LINEAR_THIRD] = "2",
      95              :   [LINEAR_FOURTH] = "3",
      96              :   [LINEAR_END] = NULL,
      97              : };
      98              : 
      99              : /**
     100              :  * @brief Template caps string.
     101              :  */
     102              : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT ";" GST_TENSORS_CAP_WITH_NUM ("1")
     103              : 
     104              : /**
     105              :  * @brief the capabilities of the inputs and outputs.
     106              :  * describe the real formats here.
     107              :  */
     108              : static GstStaticPadTemplate src_templ = GST_STATIC_PAD_TEMPLATE ("src",
     109              :     GST_PAD_SRC,
     110              :     GST_PAD_ALWAYS,
     111              :     GST_STATIC_CAPS (CAPS_STRING));
     112              : 
     113              : static GstStaticPadTemplate sink_templ = GST_STATIC_PAD_TEMPLATE ("sink_%u",
     114              :     GST_PAD_SINK,
     115              :     GST_PAD_REQUEST,
     116              :     GST_STATIC_CAPS (CAPS_STRING));
     117              : 
     118              : static gboolean gst_tensor_merge_src_event (GstPad * pad, GstObject * parent,
     119              :     GstEvent * event);
     120              : static GstPad *gst_tensor_merge_request_new_pad (GstElement * element,
     121              :     GstPadTemplate * templ, const gchar * name, const GstCaps * caps);
     122              : static GstStateChangeReturn gst_tensor_merge_change_state (GstElement * element,
     123              :     GstStateChange transition);
     124              : static gboolean gst_tensor_merge_sink_event (GstCollectPads * pads,
     125              :     GstCollectData * data, GstEvent * event, GstTensorMerge * tensor_merge);
     126              : static GstFlowReturn gst_tensor_merge_collected (GstCollectPads * pads,
     127              :     GstTensorMerge * tensor_merge);
     128              : 
     129              : static void gst_tensor_merge_set_property (GObject * object, guint prop_id,
     130              :     const GValue * value, GParamSpec * pspec);
     131              : static void gst_tensor_merge_get_property (GObject * object, guint prop_id,
     132              :     GValue * value, GParamSpec * pspec);
     133              : static void gst_tensor_merge_finalize (GObject * object);
     134              : 
     135              : #define gst_tensor_merge_parent_class parent_class
     136         1316 : G_DEFINE_TYPE (GstTensorMerge, gst_tensor_merge, GST_TYPE_ELEMENT);
     137              : 
     138              : /**
     139              :  * @brief initialize the tensor_merge's class
     140              :  */
     141              : static void
     142           22 : gst_tensor_merge_class_init (GstTensorMergeClass * klass)
     143              : {
     144              :   GObjectClass *gobject_class;
     145              :   GstElementClass *gstelement_class;
     146              : 
     147           22 :   GST_DEBUG_CATEGORY_INIT (gst_tensor_merge_debug, "tensor_merge", 0,
     148              :       "Element to merge multiple tensor stream to tensor stream");
     149              : 
     150           22 :   gobject_class = (GObjectClass *) klass;
     151           22 :   gstelement_class = (GstElementClass *) klass;
     152              : 
     153           22 :   parent_class = g_type_class_peek_parent (klass);
     154              : 
     155           22 :   gobject_class->finalize = gst_tensor_merge_finalize;
     156           22 :   gobject_class->get_property = gst_tensor_merge_get_property;
     157           22 :   gobject_class->set_property = gst_tensor_merge_set_property;
     158              : 
     159           22 :   g_object_class_install_property (gobject_class, PROP_SILENT,
     160              :       g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
     161              :           TRUE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
     162              : 
     163           22 :   g_object_class_install_property (gobject_class, PROP_MODE,
     164              :       g_param_spec_string ("mode", "Mode",
     165              :           "Tensor Merge mode. Currently, `linear` is available only.",
     166              :           gst_tensor_merge_mode_string[GTT_LINEAR], G_PARAM_READWRITE));
     167           22 :   g_object_class_install_property (gobject_class, PROP_OPTION,
     168              :       g_param_spec_string ("option", "Option",
     169              :           "Option for the tensor Merge mode.\n"
     170              :           "\t\t\t  1) linear mode: it will become the index of the rank.\n"
     171              :           "\t\t\t     (e.g. want to merge 3:640:480 & 3:640:480 to 3:1280:480, option=1)\n"
     172              :           "\t\t\t  2) TBU", "", G_PARAM_READWRITE));
     173              : 
     174           22 :   g_object_class_install_property (gobject_class, PROP_SYNC_MODE,
     175              :       g_param_spec_string ("sync-mode", "Sync Mode",
     176              :           "Time synchronization mode\n"
     177              :           "\t\t\tSee also: https://github.com/nnstreamer/nnstreamer/blob/main/Documentation/synchronization-policies-at-mux-merge.md",
     178              :           "", G_PARAM_READWRITE));
     179              : 
     180           22 :   g_object_class_install_property (gobject_class, PROP_SYNC_OPTION,
     181              :       g_param_spec_string ("sync-option", "Sync Option",
     182              :           "Option for the time synchronization mode", "", G_PARAM_READWRITE));
     183              : 
     184           22 :   gstelement_class->request_new_pad =
     185           22 :       GST_DEBUG_FUNCPTR (gst_tensor_merge_request_new_pad);
     186           22 :   gstelement_class->change_state =
     187           22 :       GST_DEBUG_FUNCPTR (gst_tensor_merge_change_state);
     188              : 
     189           22 :   gst_element_class_add_pad_template (gstelement_class,
     190              :       gst_static_pad_template_get (&sink_templ));
     191           22 :   gst_element_class_add_pad_template (gstelement_class,
     192              :       gst_static_pad_template_get (&src_templ));
     193              : 
     194           22 :   gst_element_class_set_details_simple (gstelement_class,
     195              :       "TensorMerge",
     196              :       "Muxer/Tensor",
     197              :       "Merge multiple tensor stream to tensor stream",
     198              :       "Jijoong Moon <jijoong.moon@samsung.com>");
     199              : 
     200           22 : }
     201              : 
     202              : /**
     203              :  * @brief initialize the new element
     204              :  * instantiate pads and add them to element
     205              :  * set pad callback functions
     206              :  * initialize instance structure
     207              :  */
     208              : static void
     209           27 : gst_tensor_merge_init (GstTensorMerge * tensor_merge)
     210              : {
     211           27 :   GstElementClass *klass = GST_ELEMENT_GET_CLASS (tensor_merge);
     212              : 
     213           27 :   tensor_merge->srcpad =
     214           27 :       gst_pad_new_from_template (gst_element_class_get_pad_template (klass,
     215              :           "src"), "src");
     216           27 :   gst_pad_set_event_function (tensor_merge->srcpad, gst_tensor_merge_src_event);
     217              : 
     218           27 :   gst_element_add_pad (GST_ELEMENT (tensor_merge), tensor_merge->srcpad);
     219              : 
     220           27 :   tensor_merge->collect = gst_collect_pads_new ();
     221           27 :   gst_collect_pads_set_event_function (tensor_merge->collect,
     222              :       (GstCollectPadsEventFunction)
     223           27 :       GST_DEBUG_FUNCPTR (gst_tensor_merge_sink_event), tensor_merge);
     224           27 :   gst_collect_pads_set_function (tensor_merge->collect,
     225           27 :       (GstCollectPadsFunction) GST_DEBUG_FUNCPTR (gst_tensor_merge_collected),
     226              :       tensor_merge);
     227              : 
     228           27 :   tensor_merge->silent = TRUE;
     229           27 :   tensor_merge->sync.mode = SYNC_NOSYNC;
     230           27 :   tensor_merge->sync.option = NULL;
     231           27 :   gst_tensors_config_init (&tensor_merge->tensors_config);
     232           27 :   tensor_merge->mode = GTT_LINEAR;
     233           27 :   tensor_merge->loaded = FALSE;
     234           27 :   tensor_merge->current_time = 0;
     235           27 :   tensor_merge->need_set_time = TRUE;
     236           27 : }
     237              : 
     238              : /**
     239              :  * @brief Get the corresponding mode from the string value
     240              :  * @param[in] str The string value for the mode
     241              :  * @return corresponding mode for the string. GTT_END for errors
     242              :  */
     243              : static tensor_merge_mode
     244           27 : gst_tensor_merge_get_mode (const gchar * str)
     245              : {
     246              :   int i;
     247           27 :   for (i = 0; i < GTT_END; i++) {
     248           27 :     if (g_ascii_strcasecmp (gst_tensor_merge_mode_string[i], str) == 0)
     249           27 :       return i;
     250              :   }
     251            0 :   return GTT_END;
     252              : }
     253              : 
     254              : /**
     255              :  * @brief finalize vmethod
     256              :  */
     257              : static void
     258           27 : gst_tensor_merge_finalize (GObject * object)
     259              : {
     260              :   GstTensorMerge *tensor_merge;
     261              : 
     262           27 :   tensor_merge = GST_TENSOR_MERGE (object);
     263              : 
     264           27 :   if (tensor_merge->collect) {
     265           27 :     gst_tensor_time_sync_flush (tensor_merge->collect);
     266           27 :     gst_object_unref (tensor_merge->collect);
     267           27 :     tensor_merge->collect = NULL;
     268              :   }
     269              : 
     270           27 :   if (tensor_merge->option) {
     271           27 :     g_free (tensor_merge->option);
     272           27 :     tensor_merge->option = NULL;
     273              :   }
     274              : 
     275           27 :   if (tensor_merge->sync.option) {
     276           12 :     g_free (tensor_merge->sync.option);
     277           12 :     tensor_merge->sync.option = NULL;
     278              :   }
     279              : 
     280           27 :   G_OBJECT_CLASS (parent_class)->finalize (object);
     281           27 : }
     282              : 
     283              : /**
     284              :  * @brief making new request pad (gst element vmethod)
     285              :  */
     286              : static GstPad *
     287           76 : gst_tensor_merge_request_new_pad (GstElement * element, GstPadTemplate * templ,
     288              :     const gchar * req_name, const GstCaps * caps)
     289              : {
     290              :   GstPad *newpad;
     291              :   GstTensorMerge *tensor_merge;
     292              :   GSList *walk;
     293              :   guint length;
     294              :   gchar *name;
     295              :   UNUSED (req_name);
     296              :   UNUSED (caps);
     297              : 
     298           76 :   g_return_val_if_fail (templ != NULL, NULL);
     299           76 :   g_return_val_if_fail (GST_IS_TENSOR_MERGE (element), NULL);
     300              : 
     301           76 :   tensor_merge = GST_TENSOR_MERGE (element);
     302           76 :   walk = tensor_merge->collect->data;
     303           76 :   length = g_slist_length (walk);
     304              : 
     305           76 :   if (length >= NNS_TENSOR_SIZE_LIMIT) {
     306            0 :     GST_ERROR_OBJECT (tensor_merge,
     307              :         "supposed max number of tensors is " NNS_TENSOR_SIZE_LIMIT_STR);
     308            0 :     return NULL;
     309              :   }
     310              : 
     311           76 :   name = g_strdup_printf ("sink_%u", length);
     312           76 :   newpad = gst_pad_new_from_template (templ, name);
     313           76 :   g_free (name);
     314              : 
     315           76 :   if (newpad) {
     316              :     GstTensorCollectPadData *tensormergepad;
     317              : 
     318              :     tensormergepad = (GstTensorCollectPadData *)
     319           76 :         gst_collect_pads_add_pad (tensor_merge->collect, newpad,
     320              :         sizeof (GstTensorCollectPadData), NULL, TRUE);
     321              : 
     322           76 :     gst_pad_set_element_private (newpad, tensormergepad);
     323           76 :     gst_element_add_pad (element, newpad);
     324              :   } else {
     325            0 :     GST_WARNING_OBJECT (tensor_merge, "failed to create request pad");
     326              :   }
     327           76 :   return newpad;
     328              : }
     329              : 
     330              : /**
     331              :  * @brief src event vmethod
     332              :  */
     333              : static gboolean
     334          139 : gst_tensor_merge_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
     335              : {
     336          139 :   g_return_val_if_fail (event != NULL, FALSE);
     337              : 
     338          139 :   switch (GST_EVENT_TYPE (event)) {
     339            0 :     case GST_EVENT_SEEK:
     340            0 :       gst_event_unref (event);
     341            0 :       return FALSE;
     342          139 :     default:
     343          139 :       break;
     344              :   }
     345              : 
     346          139 :   return gst_pad_event_default (pad, parent, event);
     347              : }
     348              : 
     349              : /**
     350              :  * @brief sink event vmethod
     351              :  */
     352              : static gboolean
     353          286 : gst_tensor_merge_sink_event (GstCollectPads * pads, GstCollectData * data,
     354              :     GstEvent * event, GstTensorMerge * tensor_merge)
     355              : {
     356          286 :   g_return_val_if_fail (event != NULL, FALSE);
     357              : 
     358          286 :   switch (GST_EVENT_TYPE (event)) {
     359            0 :     case GST_EVENT_FLUSH_STOP:
     360            0 :       tensor_merge->need_segment = TRUE;
     361            0 :       tensor_merge->need_set_time = TRUE;
     362            0 :       gst_tensor_time_sync_flush (tensor_merge->collect);
     363            0 :       break;
     364          286 :     default:
     365          286 :       break;
     366              :   }
     367              : 
     368          286 :   return gst_collect_pads_event_default (pads, data, event, FALSE);
     369              : }
     370              : 
     371              : /**
     372              :  * @brief Generate out TensorsConfig with in TensorsConfig
     373              :  * @param tensor_merge tensor merger
     374              :  * @param in_config in tensors config data (multi tensors)
     375              :  * @param out_config out tensors config data (single tensor)
     376              :  * @return true / false
     377              :  */
     378              : static gboolean
     379           25 : gst_tensor_merge_get_merged_config (GstTensorMerge * tensor_merge,
     380              :     const GstTensorsConfig * in_config, GstTensorsConfig * out_config)
     381              : {
     382              :   GstTensorsInfo *in_info;
     383              :   GstTensorInfo *_info;
     384           25 :   gboolean ret = FALSE;
     385              :   guint i;
     386              :   gint j;
     387              :   tensor_dim dim;
     388              :   tensor_type type;
     389              : 
     390           25 :   gst_tensors_config_init (out_config);
     391              : 
     392           25 :   in_info = (GstTensorsInfo *) &in_config->info;
     393           25 :   _info = gst_tensors_info_get_nth_info (in_info, 0);
     394              : 
     395           25 :   type = _info->type;
     396           25 :   memcpy (&dim, &_info->dimension, sizeof (tensor_dim));
     397              : 
     398           72 :   for (i = 1; i < in_config->info.num_tensors; i++) {
     399           47 :     _info = gst_tensors_info_get_nth_info (in_info, i);
     400              : 
     401           47 :     if (type != _info->type)
     402            0 :       GST_ELEMENT_ERROR (tensor_merge, CORE, NEGOTIATION, (NULL), (NULL));
     403              :   }
     404              : 
     405           25 :   switch (tensor_merge->mode) {
     406           25 :     case GTT_LINEAR:
     407              :     {
     408           25 :       int targetIdx = tensor_merge->data_linear.direction;
     409           72 :       for (i = 1; i < in_config->info.num_tensors; i++) {
     410           47 :         _info = gst_tensors_info_get_nth_info (in_info, i);
     411              : 
     412          799 :         for (j = 0; j < NNS_TENSOR_RANK_LIMIT; j++) {
     413          752 :           if (j == targetIdx) {
     414           47 :             dim[j] += _info->dimension[j];
     415              :           } else {
     416          705 :             if (dim[j] != _info->dimension[j])
     417            0 :               GST_ELEMENT_ERROR (tensor_merge, CORE, NEGOTIATION, (NULL),
     418              :                   (NULL));
     419              :           }
     420              :         }
     421              :       }
     422              : 
     423           25 :       out_config->info.num_tensors = 1;
     424           25 :       out_config->rate_d = in_config->rate_d;
     425           25 :       out_config->rate_n = in_config->rate_n;
     426              : 
     427           25 :       _info = gst_tensors_info_get_nth_info (&out_config->info, 0);
     428           25 :       _info->type = type;
     429           25 :       memcpy (&_info->dimension, &dim, sizeof (tensor_dim));
     430           25 :       ret = TRUE;
     431              :     }
     432           25 :       break;
     433            0 :     default:
     434            0 :       ret = FALSE;
     435              :   }
     436              : 
     437           25 :   return ret;
     438              : }
     439              : 
     440              : /**
     441              :  * @brief Looping to generete output buffer for srcpad
     442              :  * @param tensor_merge tensor merger
     443              :  * @param tensor_buf output buffer for srcpad
     444              :  * @param is_eos boolean EOS ( End of Stream )
     445              :  * @return TRUE to push buffer to src pad
     446              :  */
     447              : static gboolean
     448          386 : gst_tensor_merge_collect_buffer (GstTensorMerge * tensor_merge,
     449              :     GstBuffer * tensors_buf, gboolean * is_eos)
     450              : {
     451          386 :   if (tensor_merge->need_set_time) {
     452          248 :     if (gst_tensor_time_sync_get_current_time (tensor_merge->collect,
     453              :             &tensor_merge->sync, &tensor_merge->current_time, tensors_buf)) {
     454              :       /* end-of-stream */
     455           37 :       *is_eos = TRUE;
     456           37 :       return FALSE;
     457              :     }
     458              : 
     459          211 :     tensor_merge->need_set_time = FALSE;
     460              :   }
     461              : 
     462          349 :   return gst_tensor_time_sync_buffer_from_collectpad (tensor_merge->collect,
     463              :       &tensor_merge->sync, tensor_merge->current_time, tensors_buf,
     464              :       &tensor_merge->tensors_config, is_eos);
     465              : }
     466              : 
     467              : /**
     468              :  * @brief Generate Output GstMemory
     469              :  * @param tensor_merge tensor merger
     470              :  * @param tensors_buf collected tensors buffer
     471              :  * @param tensor_buf output tensor buffer
     472              :  * @return boolean
     473              :  */
     474              : static GstFlowReturn
     475          211 : gst_tensor_merge_generate_mem (GstTensorMerge * tensor_merge,
     476              :     GstBuffer * tensors_buf, GstBuffer * tensor_buf)
     477              : {
     478          211 :   GstFlowReturn ret = GST_FLOW_OK;
     479              :   GstMapInfo mInfo[NNS_TENSOR_SIZE_LIMIT];
     480              :   GstMemory *mem[NNS_TENSOR_SIZE_LIMIT];
     481              :   GstMapInfo outInfo;
     482              :   GstMemory *outMem;
     483              :   uint8_t *inptr, *outptr;
     484          211 :   guint num_mem = tensor_merge->tensors_config.info.num_tensors;
     485              :   GstTensorsInfo *info;
     486              :   GstTensorInfo *_info;
     487              :   guint i, j, k, l;
     488              :   size_t c, s;
     489          211 :   gsize outSize = 0;
     490              :   gsize element_size;
     491              :   tensor_dim dim;
     492              :   tensor_type type;
     493              : 
     494          211 :   info = &tensor_merge->tensors_config.info;
     495          211 :   _info = gst_tensors_info_get_nth_info (info, 0);
     496              : 
     497          211 :   memcpy (&dim, &_info->dimension, sizeof (tensor_dim));
     498          211 :   type = _info->type;
     499          211 :   element_size = gst_tensor_get_element_size (type);
     500              : 
     501          750 :   for (i = 0; i < num_mem; i++) {
     502          539 :     mem[i] = gst_tensor_buffer_get_nth_memory (tensors_buf, i);
     503          539 :     if (!gst_memory_map (mem[i], &mInfo[i], GST_MAP_READ)) {
     504            0 :       ml_logf ("Cannot map input memory buffers (%d)\n", i);
     505            0 :       gst_memory_unref (mem[i]);
     506            0 :       num_mem = i;
     507            0 :       ret = GST_FLOW_ERROR;
     508            0 :       goto error_ret;
     509              :     }
     510          539 :     outSize += mInfo[i].size;
     511              :   }
     512              : 
     513          211 :   outMem = gst_allocator_alloc (NULL, outSize, NULL);
     514          211 :   if (!gst_memory_map (outMem, &outInfo, GST_MAP_WRITE)) {
     515            0 :     gst_memory_unref (outMem);
     516            0 :     ml_logf ("Cannot map output memory buffer\n");
     517            0 :     ret = GST_FLOW_ERROR;
     518            0 :     goto error_ret;
     519              :   }
     520          211 :   outptr = outInfo.data;
     521              : 
     522          211 :   switch (tensor_merge->mode) {
     523          211 :     case GTT_LINEAR:
     524              :     {
     525          211 :       switch (tensor_merge->data_linear.direction) {
     526            1 :         case LINEAR_FIRST:
     527              :         {
     528            2 :           for (l = 0; l < dim[3]; l++) {
     529          101 :             for (i = 0; i < dim[2]; i++) {
     530         5100 :               for (j = 0; j < dim[1]; j++) {
     531        20000 :                 for (k = 0; k < num_mem; k++) {
     532        15000 :                   _info = gst_tensors_info_get_nth_info (info, k);
     533              : 
     534        15000 :                   c = _info->dimension[0];
     535        15000 :                   s = element_size * c;
     536        15000 :                   inptr =
     537        15000 :                       mInfo[k].data + (l * dim[2] * dim[1] + i * dim[1] +
     538        15000 :                       j) * s;
     539        15000 :                   memcpy (outptr, inptr, s);
     540        15000 :                   outptr += s;
     541              :                 }
     542              :               }
     543              :             }
     544              :           }
     545            1 :           break;
     546              :         }
     547            1 :         case LINEAR_SECOND:
     548              :         {
     549            2 :           for (l = 0; l < dim[3]; l++) {
     550           51 :             for (i = 0; i < dim[2]; i++) {
     551          200 :               for (k = 0; k < num_mem; k++) {
     552          150 :                 _info = gst_tensors_info_get_nth_info (info, k);
     553              : 
     554          150 :                 c = 1;
     555          450 :                 for (j = 0; j < LINEAR_SECOND + 1; j++)
     556          300 :                   c *= _info->dimension[j];
     557              : 
     558          150 :                 s = element_size * c;
     559              : 
     560          150 :                 inptr = mInfo[k].data + (l * dim[2] + i) * s;
     561              : 
     562          150 :                 memcpy (outptr, inptr, s);
     563          150 :                 outptr += s;
     564              :               }
     565              :             }
     566              :           }
     567            1 :           break;
     568              :         }
     569          119 :         case LINEAR_THIRD:
     570              :         {
     571          238 :           for (l = 0; l < dim[3]; l++) {
     572          471 :             for (k = 0; k < num_mem; k++) {
     573          352 :               _info = gst_tensors_info_get_nth_info (info, k);
     574              : 
     575          352 :               c = 1;
     576         1408 :               for (j = 0; j < LINEAR_THIRD + 1; j++)
     577         1056 :                 c *= _info->dimension[j];
     578              : 
     579          352 :               s = element_size * c;
     580              : 
     581          352 :               inptr = mInfo[k].data + l * s;
     582              : 
     583          352 :               memcpy (outptr, inptr, s);
     584          352 :               outptr += s;
     585              :             }
     586              :           }
     587          119 :           break;
     588              :         }
     589           90 :         case LINEAR_FOURTH:
     590              :         {
     591          271 :           for (k = 0; k < num_mem; k++) {
     592          181 :             _info = gst_tensors_info_get_nth_info (info, k);
     593              : 
     594          181 :             c = 1;
     595          905 :             for (j = 0; j < LINEAR_FOURTH + 1; j++)
     596          724 :               c *= _info->dimension[j];
     597              : 
     598          181 :             s = element_size * c;
     599              : 
     600          181 :             inptr = mInfo[k].data;
     601          181 :             memcpy (outptr, inptr, s);
     602          181 :             outptr += s;
     603              :           }
     604           90 :           break;
     605              :         }
     606            0 :         default:
     607            0 :           ret = GST_FLOW_ERROR;
     608              :       }
     609          211 :       break;
     610              :     }
     611            0 :     default:
     612            0 :       ret = GST_FLOW_ERROR;
     613              :   }
     614              : 
     615          211 :   gst_memory_unmap (outMem, &outInfo);
     616          211 :   gst_buffer_append_memory (tensor_buf, outMem);
     617          211 :   gst_buffer_copy_into (tensor_buf, tensors_buf, GST_BUFFER_COPY_TIMESTAMPS, 0,
     618              :       -1);
     619              : 
     620          211 : error_ret:
     621          750 :   for (i = 0; i < num_mem; i++) {
     622          539 :     gst_memory_unmap (mem[i], &mInfo[i]);
     623          539 :     gst_memory_unref (mem[i]);
     624              :   }
     625          211 :   return ret;
     626              : }
     627              : 
     628              : /**
     629              :  * @brief Set src pad caps if src pad is not negotiated.
     630              :  */
     631              : static gboolean
     632          211 : gst_tensor_merge_set_src_caps (GstTensorMerge * tensor_merge)
     633              : {
     634          211 :   if (!tensor_merge->negotiated) {
     635              :     GstCaps *newcaps;
     636              :     GstTensorsConfig config;
     637              : 
     638           25 :     if (!gst_tensor_merge_get_merged_config (tensor_merge,
     639           25 :             &tensor_merge->tensors_config, &config)) {
     640            0 :       goto nego_error;
     641              :     }
     642              : 
     643              :     /** Internal Logic Error? */
     644           25 :     g_assert (gst_tensors_config_validate (&config));
     645           25 :     newcaps = gst_tensor_pad_caps_from_config (tensor_merge->srcpad, &config);
     646              : 
     647           25 :     if (gst_pad_set_caps (tensor_merge->srcpad, newcaps)) {
     648           25 :       tensor_merge->negotiated = TRUE;
     649              :     }
     650              : 
     651           25 :     gst_caps_unref (newcaps);
     652              :   }
     653              : 
     654          186 : nego_error:
     655          211 :   if (!tensor_merge->negotiated) {
     656            0 :     GST_WARNING_OBJECT (tensor_merge, "failed to set caps");
     657            0 :     GST_ELEMENT_ERROR (tensor_merge, CORE, NEGOTIATION, (NULL), (NULL));
     658              :   }
     659          211 :   return tensor_merge->negotiated;
     660              : }
     661              : 
     662              : /**
     663              :  * @brief Send segment event if necessary.
     664              :  */
     665              : static void
     666          211 : gst_tensor_merge_send_segment_event (GstTensorMerge * tensor_merge,
     667              :     GstClockTime pts, GstClockTime dts)
     668              : {
     669          211 :   if (tensor_merge->need_segment) {
     670              :     GstSegment segment;
     671           25 :     GstClockTime time = 0;
     672              : 
     673           25 :     if (GST_CLOCK_TIME_IS_VALID (dts)) {
     674            0 :       time = dts;
     675           25 :     } else if (GST_CLOCK_TIME_IS_VALID (pts)) {
     676           25 :       time = pts;
     677              :     }
     678              : 
     679           25 :     gst_segment_init (&segment, GST_FORMAT_TIME);
     680           25 :     segment.start = time;
     681           25 :     gst_pad_push_event (tensor_merge->srcpad, gst_event_new_segment (&segment));
     682           25 :     tensor_merge->need_segment = FALSE;
     683              :   }
     684          211 : }
     685              : 
     686              : /**
     687              :  * @brief Gst Collect Pads Function which is called once collect pads done.
     688              :  * @param pads GstCollectPads
     689              :  * @param tensor_merge Merger
     690              :  * @return GstFlowReturn
     691              :  */
     692              : static GstFlowReturn
     693          386 : gst_tensor_merge_collected (GstCollectPads * pads,
     694              :     GstTensorMerge * tensor_merge)
     695              : {
     696          386 :   GstFlowReturn ret = GST_FLOW_OK;
     697              :   GstBuffer *tensors_buf, *tensor_buf;
     698          386 :   gboolean isEOS = FALSE;
     699              : 
     700          386 :   GST_DEBUG_OBJECT (tensor_merge, " all pads are collected ");
     701              : 
     702          386 :   if (tensor_merge->need_stream_start) {
     703              :     /**
     704              :      * Cannot use gst-pad util to get stream ID (multiple sink pads).
     705              :      * Create stream ID using first sink pad.
     706              :      */
     707           25 :     GstCollectData *data = (GstCollectData *) pads->data->data;
     708           25 :     g_autofree gchar *sink_stream_id = gst_pad_get_stream_id (data->pad);
     709           25 :     g_autofree gchar *element_name = gst_element_get_name (tensor_merge);
     710           25 :     g_autofree gchar *pad_name = gst_pad_get_name (tensor_merge->srcpad);
     711           50 :     g_autofree gchar *stream_id = g_strdup_printf ("%s-%s-nnsmerge-%s-%08x",
     712           25 :         GST_STR_NULL (sink_stream_id), element_name, pad_name, g_random_int ());
     713              : 
     714           25 :     gst_pad_push_event (tensor_merge->srcpad, gst_event_new_stream_start (stream_id));
     715           25 :     tensor_merge->need_stream_start = FALSE;
     716              :   }
     717              : 
     718          386 :   if ((tensors_buf = gst_buffer_new ()) == NULL) {
     719            0 :     ml_logf ("gst_buffer_new() returns NULL. Out of memory?\n");
     720          386 :     return GST_FLOW_ERROR;
     721              :   }
     722              : 
     723          386 :   if (!gst_tensor_merge_collect_buffer (tensor_merge, tensors_buf, &isEOS)) {
     724          175 :     if (isEOS) {
     725           37 :       gst_pad_push_event (tensor_merge->srcpad, gst_event_new_eos ());
     726           37 :       ret = GST_FLOW_EOS;
     727              :     }
     728              : 
     729          175 :     goto beach;
     730              :   }
     731              : 
     732          211 :   if (!gst_tensor_merge_set_src_caps (tensor_merge)) {
     733            0 :     ret = GST_FLOW_NOT_NEGOTIATED;
     734            0 :     goto beach;
     735              :   }
     736              : 
     737          211 :   gst_tensor_merge_send_segment_event (tensor_merge,
     738              :       GST_BUFFER_PTS (tensors_buf), GST_BUFFER_DTS (tensors_buf));
     739              : 
     740          211 :   if ((tensor_buf = gst_buffer_new ()) == NULL) {
     741            0 :     ml_logf ("gst_buffer_new() returns NULL. Out of memory?\n");
     742            0 :     ret = GST_FLOW_ERROR;
     743            0 :     goto beach;
     744              :   }
     745              : 
     746          211 :   gst_tensor_merge_generate_mem (tensor_merge, tensors_buf, tensor_buf);
     747              : 
     748          211 :   ret = gst_pad_push (tensor_merge->srcpad, tensor_buf);
     749          211 :   tensor_merge->need_set_time = TRUE;
     750              : 
     751          211 :   if (ret != GST_FLOW_OK) {
     752            0 :     GST_WARNING_OBJECT (tensor_merge, "pushed outbuf, result = %s",
     753              :         gst_flow_get_name (ret));
     754              :   }
     755          211 : beach:
     756          386 :   gst_buffer_unref (tensors_buf);
     757          386 :   return ret;
     758              : }
     759              : 
     760              : /**
     761              :  * @brief Ready --> Pasuse State Change
     762              :  */
     763              : static void
     764           25 : gst_tensor_merge_ready_to_paused (GstTensorMerge * tensor_merge)
     765              : {
     766           25 :   tensor_merge->need_stream_start = TRUE;
     767           25 :   tensor_merge->need_segment = TRUE;
     768           25 :   tensor_merge->negotiated = FALSE;
     769           25 :   gst_collect_pads_start (tensor_merge->collect);
     770           25 : }
     771              : 
     772              : /**
     773              :  * @brief change state (gst element vmethod)
     774              :  */
     775              : static GstStateChangeReturn
     776          150 : gst_tensor_merge_change_state (GstElement * element, GstStateChange transition)
     777              : {
     778              :   GstTensorMerge *tensor_merge;
     779              :   GstStateChangeReturn ret;
     780          150 :   tensor_merge = GST_TENSOR_MERGE (element);
     781          150 :   switch (transition) {
     782           25 :     case GST_STATE_CHANGE_READY_TO_PAUSED:
     783           25 :       gst_tensor_merge_ready_to_paused (tensor_merge);
     784           25 :       break;
     785           25 :     case GST_STATE_CHANGE_PAUSED_TO_READY:
     786           25 :       gst_collect_pads_stop (tensor_merge->collect);
     787           25 :       break;
     788          100 :     default:
     789          100 :       break;
     790              :   }
     791              : 
     792          150 :   ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
     793          150 :   if (ret == GST_STATE_CHANGE_FAILURE)
     794            0 :     return ret;
     795          150 :   switch (transition) {
     796           25 :     case GST_STATE_CHANGE_PAUSED_TO_READY:
     797           25 :       break;
     798          125 :     default:
     799          125 :       break;
     800              :   }
     801              : 
     802          150 :   return ret;
     803              : }
     804              : 
     805              : /**
     806              :  * @brief Setup internal data (data_* in GstTensor_Merge)
     807              :  * @param[in/out] filter "this" pointer. mode & option MUST BE set already.
     808              :  * @retval TRUE if ok or not configured, yet.
     809              :  * @retval FALSE if given input is configured invalid.
     810              :  */
     811              : static gboolean
     812           54 : gst_tensor_merge_set_option_data (GstTensorMerge * tensor_merge)
     813              : {
     814           54 :   if (tensor_merge->mode == GTT_END || tensor_merge->option == NULL)
     815           27 :     return TRUE;
     816           27 :   switch (tensor_merge->mode) {
     817           27 :     case GTT_LINEAR:
     818              :     {
     819              :       gint idx;
     820              : 
     821              :       idx =
     822           27 :           find_key_strv (gst_tensor_merge_linear_string, tensor_merge->option);
     823           27 :       if (idx < 0)
     824            0 :         return FALSE;
     825              : 
     826           27 :       tensor_merge->data_linear.direction = (tensor_merge_linear_mode) idx;
     827           27 :       tensor_merge->loaded = TRUE;
     828              :     }
     829           27 :       break;
     830            0 :     default:
     831            0 :       GST_ERROR_OBJECT (tensor_merge, "Cannot identify mode\n");
     832            0 :       return FALSE;
     833              :   }
     834           27 :   return TRUE;
     835              : }
     836              : 
     837              : /**
     838              :  * @brief Get property (gst element vmethod)
     839              :  */
     840              : static void
     841           95 : gst_tensor_merge_set_property (GObject * object, guint prop_id,
     842              :     const GValue * value, GParamSpec * pspec)
     843              : {
     844           95 :   GstTensorMerge *tensor_merge = GST_TENSOR_MERGE (object);
     845           95 :   switch (prop_id) {
     846           12 :     case PROP_SILENT:
     847           12 :       tensor_merge->silent = g_value_get_boolean (value);
     848           12 :       break;
     849           27 :     case PROP_MODE:
     850           27 :       tensor_merge->mode =
     851           27 :           gst_tensor_merge_get_mode (g_value_get_string (value));
     852           27 :       if (tensor_merge->mode == GTT_END) {
     853            0 :         ml_logw ("Given mode property is not recognized: %s\n",
     854              :             g_value_get_string (value));
     855            0 :         break;
     856              :       }
     857           27 :       if (!gst_tensor_merge_set_option_data (tensor_merge)) {
     858            0 :         tensor_merge->loaded = FALSE;
     859            0 :         ml_logw ("Given mode property is not consistent with its options.\n");
     860              :       }
     861           27 :       break;
     862           27 :     case PROP_OPTION:
     863           27 :       tensor_merge->option = g_value_dup_string (value);
     864           27 :       if (!gst_tensor_merge_set_option_data (tensor_merge)) {
     865            0 :         tensor_merge->loaded = FALSE;
     866            0 :         ml_logw ("Given option property is not consistent with its mode.\n");
     867              :       }
     868           27 :       break;
     869           17 :     case PROP_SYNC_MODE:
     870           17 :       tensor_merge->sync.mode =
     871           17 :           gst_tensor_time_sync_get_mode (g_value_get_string (value));
     872           17 :       if (tensor_merge->sync.mode == SYNC_END) {
     873            0 :         tensor_merge->sync.mode = SYNC_NOSYNC;
     874              :       }
     875           17 :       silent_debug (tensor_merge, "Mode = %d(%s)\n", tensor_merge->sync.mode,
     876              :           gst_tensor_time_sync_get_mode_string (tensor_merge->sync.mode));
     877           17 :       gst_tensor_time_sync_set_option_data (&tensor_merge->sync);
     878           17 :       break;
     879           12 :     case PROP_SYNC_OPTION:
     880           12 :       tensor_merge->sync.option = g_value_dup_string (value);
     881           12 :       silent_debug (tensor_merge, "Option = %s\n", tensor_merge->sync.option);
     882           12 :       gst_tensor_time_sync_set_option_data (&tensor_merge->sync);
     883           12 :       break;
     884            0 :     default:
     885            0 :       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     886            0 :       break;
     887              :   }
     888           95 : }
     889              : 
     890              : /**
     891              :  * @brief Get property (gst element vmethod)
     892              :  */
     893              : static void
     894            6 : gst_tensor_merge_get_property (GObject * object, guint prop_id,
     895              :     GValue * value, GParamSpec * pspec)
     896              : {
     897            6 :   GstTensorMerge *tensor_merge = GST_TENSOR_MERGE (object);
     898            6 :   switch (prop_id) {
     899            2 :     case PROP_SILENT:
     900            2 :       g_value_set_boolean (value, tensor_merge->silent);
     901            2 :       break;
     902            1 :     case PROP_MODE:
     903            1 :       g_value_set_string (value,
     904            1 :           gst_tensor_merge_mode_string[tensor_merge->mode]);
     905            1 :       break;
     906            1 :     case PROP_OPTION:
     907            1 :       g_value_set_string (value, tensor_merge->option);
     908            1 :       break;
     909            1 :     case PROP_SYNC_MODE:
     910            1 :       g_value_set_string (value,
     911              :           gst_tensor_time_sync_get_mode_string (tensor_merge->sync.mode));
     912            1 :       break;
     913            1 :     case PROP_SYNC_OPTION:
     914            1 :       g_value_set_string (value, tensor_merge->sync.option);
     915            1 :       break;
     916            0 :     default:
     917            0 :       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
     918            0 :       break;
     919              :   }
     920            6 : }
        

Generated by: LCOV version 2.0-1