Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer / NNStreamer tensor_converter subplugin, "Flexbuffer"
4 : * Copyright (C) 2021 Gichan Jang <gichan2.jang@samsung.com>
5 : */
6 : /**
7 : * @file tensor_converter_flexbuf.cc
8 : * @date 12 Mar 2021
9 : * @brief NNStreamer tensor-converter subplugin, "flexbuffer",
10 : * which converts flexbuffers byte stream to tensors.
11 : * @see https://github.com/nnstreamer/nnstreamer
12 : * @author Gichan Jang <gichan2.jang@samsung.com>
13 : * @bug No known bugs except for NYI items
14 : *
15 : */
16 : /**
17 : * SECTION:tensor_converter::flexbuf
18 : * @see https://google.github.io/flatbuffers/flexbuffers.html
19 : *
20 : * tensor_converter::flexbuf converts flexbuffers to tensors stream..
21 : * The output is always in the format of other/tensor or other/tensors.
22 : *
23 : * Binary format of the flexbuffers for tensors (default in nnstreamer).
24 : * Each data is represented in `KEY : TYPE | <VALUE>` form.
25 : *
26 : * Map {
27 : * "num_tensors" : UInt32 | <The number of tensors>
28 : * "rate_n" : Int32 | <Framerate numerator>
29 : * "rate_d" : Int32 | <Framerate denominator>
30 : * "tensor_#": Vector | { String | <tensor name>,
31 : * Int32 | <data type>,
32 : * Vector | <tensor dimension>,
33 : * Blob | <tensor data>
34 : * }
35 : * }
36 : *
37 : * If you want to convert your own binary format of the flexbuffers to tensors,
38 : * You can use custom mode of the tensor converter.
39 : * This is an example of a callback type custom mode.
40 : * @code
41 : * // Define custom callback function
42 : * GstBuffer * tensor_converter_custom_cb (GstBuffer *in_buf,
43 : * void *data, GstTensorsConfig *config) {
44 : * // Write a code to convert flexbuffers to tensors.
45 : * }
46 : *
47 : * ...
48 : * // Register custom callback function
49 : * nnstreamer_converter_custom_register ("tconv", tensor_converter_custom_cb, NULL);
50 : * ...
51 : * // Use the custom tensor converter in a pipeline.
52 : * // E.g., Pipeline of " ... (flexbuffers) ! tensor_converter mode=custom-code:tconv ! (tensors)... "
53 : * ...
54 : * // After everything is done.
55 : * nnstreamer_converter_custom_unregister ("tconv");
56 : * @endcode
57 : */
58 :
59 : #include <flatbuffers/flexbuffers.h>
60 : #include <glib.h>
61 : #include <nnstreamer_log.h>
62 : #include <nnstreamer_plugin_api.h>
63 : #include <nnstreamer_plugin_api_converter.h>
64 : #include <nnstreamer_util.h>
65 : #include "../extra/nnstreamer_flatbuf.h"
66 : #include "tensor_converter_util.h"
67 :
68 : #ifdef __cplusplus
69 : extern "C" {
70 : #endif /* __cplusplus */
71 : void init_flxc (void) __attribute__ ((constructor));
72 : void fini_flxc (void) __attribute__ ((destructor));
73 : #ifdef __cplusplus
74 : }
75 : #endif /* __cplusplus */
76 :
77 : /** @brief tensor converter plugin's NNStreamerExternalConverter callback */
78 : static GstCaps *
79 458 : flxc_query_caps (const GstTensorsConfig *config)
80 : {
81 : UNUSED (config);
82 458 : return gst_caps_from_string (GST_FLEXBUF_CAP_DEFAULT);
83 : }
84 :
85 : /** @brief tensor converter plugin's NNStreamerExternalConverter callback
86 : */
87 : static GstBuffer *
88 50 : flxc_convert (GstBuffer *in_buf, GstTensorsConfig *config, void *priv_data)
89 : {
90 50 : GstBuffer *out_buf = NULL;
91 : GstMemory *in_mem, *out_mem;
92 : GstMapInfo in_info;
93 : gsize mem_size;
94 : GstTensorInfo *_info;
95 :
96 : UNUSED (priv_data);
97 :
98 50 : if (!in_buf || !config) {
99 2 : ml_loge ("NULL parameter is passed to tensor_converter::flexbuf");
100 2 : return NULL;
101 : }
102 :
103 48 : in_mem = gst_buffer_get_all_memory (in_buf);
104 48 : if (!gst_memory_map (in_mem, &in_info, GST_MAP_READ)) {
105 0 : ml_loge ("Cannot map input memory / tensor_converter::flexbuf.\n");
106 0 : gst_memory_unref (in_mem);
107 0 : return NULL;
108 : }
109 :
110 48 : flexbuffers::Map tensors = flexbuffers::GetRoot (in_info.data, in_info.size).AsMap ();
111 48 : config->info.num_tensors = tensors["num_tensors"].AsUInt32 ();
112 :
113 48 : if (config->info.num_tensors > NNS_TENSOR_SIZE_LIMIT) {
114 0 : nns_loge ("The number of tensors is limited to %d", NNS_TENSOR_SIZE_LIMIT);
115 0 : goto done;
116 : }
117 48 : config->rate_n = tensors["rate_n"].AsInt32 ();
118 48 : config->rate_d = tensors["rate_d"].AsInt32 ();
119 48 : config->info.format = (tensor_format) tensors["format"].AsInt32 ();
120 48 : out_buf = gst_buffer_new ();
121 :
122 142 : for (guint i = 0; i < config->info.num_tensors; i++) {
123 94 : gchar *tensor_key = g_strdup_printf ("tensor_%d", i);
124 : gsize offset;
125 94 : flexbuffers::Vector tensor = tensors[tensor_key].AsVector ();
126 94 : flexbuffers::String _name = tensor[0].AsString ();
127 94 : const gchar *name = _name.c_str ();
128 :
129 94 : _info = gst_tensors_info_get_nth_info (&config->info, i);
130 :
131 94 : g_free (_info->name);
132 95 : _info->name = (name && strlen (name) > 0) ? g_strdup (name) : NULL;
133 94 : _info->type = (tensor_type) tensor[1].AsInt32 ();
134 :
135 94 : flexbuffers::TypedVector dim = tensor[2].AsTypedVector ();
136 1598 : for (guint j = 0; j < NNS_TENSOR_RANK_LIMIT; j++) {
137 1504 : _info->dimension[j] = dim[j].AsInt32 ();
138 : }
139 94 : flexbuffers::Blob tensor_data = tensor[3].AsBlob ();
140 94 : mem_size = gst_tensor_info_get_size (_info);
141 94 : if (gst_tensors_config_is_flexible (config)) {
142 : GstTensorMetaInfo meta;
143 9 : gst_tensor_meta_info_parse_header (&meta, (gpointer) tensor_data.data ());
144 9 : mem_size += gst_tensor_meta_info_get_header_size (&meta);
145 : }
146 :
147 94 : offset = tensor_data.data () - in_info.data;
148 :
149 94 : out_mem = gst_memory_share (in_mem, offset, mem_size);
150 :
151 94 : gst_tensor_buffer_append_memory (out_buf, out_mem, _info);
152 94 : g_free (tensor_key);
153 : }
154 :
155 : /** copy timestamps */
156 48 : gst_buffer_copy_into (
157 : out_buf, in_buf, (GstBufferCopyFlags) GST_BUFFER_COPY_METADATA, 0, -1);
158 48 : done:
159 48 : gst_memory_unmap (in_mem, &in_info);
160 48 : gst_memory_unref (in_mem);
161 :
162 48 : return out_buf;
163 : }
164 :
165 : static const gchar converter_subplugin_flexbuf[] = "flexbuf";
166 :
167 : /** @brief flexbuffer tensor converter sub-plugin NNStreamerExternalConverter instance */
168 : static NNStreamerExternalConverter flexBuf = { .name = converter_subplugin_flexbuf,
169 : .convert = flxc_convert,
170 : .get_out_config = tcu_get_out_config,
171 : .query_caps = flxc_query_caps,
172 : .open = NULL,
173 : .close = NULL };
174 :
175 : #ifdef __cplusplus
176 : extern "C" {
177 : #endif /* __cplusplus */
178 : /** @brief Initialize this object for tensor converter sub-plugin */
179 : void
180 420 : init_flxc (void)
181 : {
182 420 : registerExternalConverter (&flexBuf);
183 420 : }
184 :
185 : /** @brief Destruct this object for tensor converter sub-plugin */
186 : void
187 420 : fini_flxc (void)
188 : {
189 420 : unregisterExternalConverter (flexBuf.name);
190 420 : }
191 : #ifdef __cplusplus
192 : }
193 : #endif /* __cplusplus */
|
