Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer / NNStreamer tensor_decoder subplugin, "Flexbuffer"
4 : * Copyright (C) 2021 Gichan Jang <gichan2.jang@samsung.com>
5 : */
6 : /**
7 : * @file tensordec-flexbuf.cc
8 : * @date 12 Mar 2021
9 : * @brief NNStreamer tensor-decoder subplugin, "flexbuffer",
10 : * which converts tensor or tensors to flexbuffer byte stream.
11 : *
12 : * @see https://github.com/nnstreamer/nnstreamer
13 : * @author Gichan Jang <gichan2.jang@samsung.com>
14 : * @bug No known bugs except for NYI items
15 : *
16 : */
17 : /**
18 : * SECTION:tensor_decoder::flexbuf
19 : * @see https://google.github.io/flatbuffers/flexbuffers.html
20 : *
21 : * tensor_decoder::flexbuf converts tensors stream to flexbuffers.
22 : *
23 : * Binary format of the flexbuffers for tensors (default in nnstreamer).
24 : * Each data is represented in `KEY : TYPE | <VALUE>` form.
25 : *
26 : * Map {
27 : * "num_tensors" : UInt32 | <The number of tensors>
28 : * "rate_n" : Int32 | <Framerate numerator>
29 : * "rate_d" : Int32 | <Framerate denominator>
30 : * "tensor_#": Vector | { String | <tensor name>,
31 : * Int32 | <data type>,
32 : * Vector | <tensor dimension>,
33 : * Blob | <tensor data>
34 : * }
35 : * }
36 : *
37 : * If you want to convert tensors to your own binary format of the flexbuffers,
38 : * You can use custom mode of the tensor decoder.
39 : * This is an example of a callback type custom mode.
40 : * @code
41 : * // Define custom callback function
42 : * int tensor_decoder_custom_cb (const GstTensorMemory *input,
43 : * const GstTensorsConfig *config, void *data, GstBuffer *out_buf) {
44 : * // Write a code to convert tensors to flexbuffers.
45 : * }
46 : *
47 : * ...
48 : * // Register custom callback function
49 : * nnstreamer_decoder_custom_register ("tdec", tensor_converter_custom_cb, NULL);
50 : * ...
51 : * // Use the custom tensor converter in a pipeline.
52 : * // E.g., Pipeline of " ... (tensors) ! tensor_decoder mode=custom-code option1=tdec ! (flexbuffers)... "
53 : * ...
54 : * // After everything is done.
55 : * nnstreamer_decoder_custom_unregister ("tdec");
56 : * @endcode
57 : */
58 :
59 : #include <flatbuffers/flexbuffers.h>
60 : #include <glib.h>
61 : #include <nnstreamer_log.h>
62 : #include <nnstreamer_plugin_api.h>
63 : #include <nnstreamer_plugin_api_decoder.h>
64 : #include <nnstreamer_util.h>
65 : #include "../extra/nnstreamer_flatbuf.h"
66 : #include "tensordecutil.h"
67 :
68 : #ifdef __cplusplus
69 : extern "C" {
70 : #endif /* __cplusplus */
71 : void init_flxd (void) __attribute__ ((constructor));
72 : void fini_flxd (void) __attribute__ ((destructor));
73 : #ifdef __cplusplus
74 : }
75 : #endif /* __cplusplus */
76 :
77 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
78 : static int
79 28 : flxd_init (void **pdata)
80 : {
81 28 : *pdata = NULL;
82 28 : return TRUE;
83 : }
84 :
85 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
86 : static void
87 28 : flxd_exit (void **pdata)
88 : {
89 : UNUSED (pdata);
90 28 : return;
91 : }
92 :
93 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
94 : static int
95 0 : flxd_setOption (void **pdata, int opNum, const char *param)
96 : {
97 : UNUSED (pdata);
98 : UNUSED (opNum);
99 : UNUSED (param);
100 0 : return TRUE;
101 : }
102 :
103 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
104 : static GstCaps *
105 195 : flxd_getOutCaps (void **pdata, const GstTensorsConfig *config)
106 : {
107 : GstCaps *caps;
108 : UNUSED (pdata);
109 :
110 195 : caps = gst_caps_from_string (GST_FLEXBUF_CAP_DEFAULT);
111 195 : setFramerateFromConfig (caps, config);
112 195 : return caps;
113 : }
114 :
115 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
116 : static GstFlowReturn
117 56 : flxd_decode (void **pdata, const GstTensorsConfig *config,
118 : const GstTensorMemory *input, GstBuffer *outbuf)
119 : {
120 : GstMapInfo out_info;
121 : GstMemory *out_mem;
122 : guint i, num_tensors;
123 : gboolean need_alloc;
124 : size_t flex_size;
125 56 : flexbuffers::Builder fbb;
126 : gboolean is_flexible;
127 : GstTensorMetaInfo meta;
128 : GstTensorInfo *_info;
129 :
130 : UNUSED (pdata);
131 :
132 56 : if (!config || !input || !outbuf) {
133 3 : ml_loge ("NULL parameter is passed to tensor_decoder::flexbuf");
134 3 : return GST_FLOW_ERROR;
135 : }
136 :
137 53 : is_flexible = gst_tensors_config_is_flexible (config);
138 :
139 53 : num_tensors = config->info.num_tensors;
140 53 : fbb.Map ([&] () {
141 53 : fbb.UInt ("num_tensors", num_tensors);
142 53 : fbb.Int ("rate_n", config->rate_n);
143 53 : fbb.Int ("rate_d", config->rate_d);
144 53 : fbb.Int ("format", config->info.format);
145 152 : for (i = 0; i < num_tensors; i++) {
146 99 : gchar *tensor_key = g_strdup_printf ("tensor_%d", i);
147 99 : gchar *tensor_name = NULL;
148 :
149 99 : _info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, i);
150 :
151 99 : if (is_flexible) {
152 9 : gst_tensor_meta_info_parse_header (&meta, input[i].data);
153 9 : gst_tensor_meta_info_convert (&meta, _info);
154 : }
155 :
156 99 : if (_info->name == NULL) {
157 98 : tensor_name = g_strdup ("");
158 : } else {
159 2 : tensor_name = g_strdup (_info->name);
160 : }
161 99 : tensor_type type = _info->type;
162 :
163 99 : fbb.Vector (tensor_key, [&] () {
164 99 : fbb += tensor_name;
165 99 : fbb += type;
166 99 : fbb.Vector (_info->dimension, NNS_TENSOR_RANK_LIMIT);
167 99 : fbb.Blob (input[i].data, input[i].size);
168 99 : });
169 99 : g_free (tensor_key);
170 99 : g_free (tensor_name);
171 : }
172 53 : });
173 53 : fbb.Finish ();
174 53 : flex_size = fbb.GetSize ();
175 :
176 53 : need_alloc = (gst_buffer_get_size (outbuf) == 0);
177 :
178 53 : if (need_alloc) {
179 53 : out_mem = gst_allocator_alloc (NULL, flex_size, NULL);
180 : } else {
181 0 : if (gst_buffer_get_size (outbuf) < flex_size) {
182 0 : gst_buffer_set_size (outbuf, flex_size);
183 : }
184 0 : out_mem = gst_buffer_get_all_memory (outbuf);
185 : }
186 :
187 53 : if (!gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) {
188 0 : gst_memory_unref (out_mem);
189 0 : nns_loge ("Cannot map gst memory (tensor decoder flexbuf)\n");
190 0 : return GST_FLOW_ERROR;
191 : }
192 :
193 53 : memcpy (out_info.data, fbb.GetBuffer ().data (), flex_size);
194 :
195 53 : gst_memory_unmap (out_mem, &out_info);
196 :
197 53 : if (need_alloc)
198 53 : gst_buffer_append_memory (outbuf, out_mem);
199 : else
200 0 : gst_buffer_replace_all_memory (outbuf, out_mem);
201 :
202 53 : return GST_FLOW_OK;
203 56 : }
204 :
205 : static gchar decoder_subplugin_flexbuf[] = "flexbuf";
206 :
207 : /** @brief flexbuffer tensordec-plugin GstTensorDecoderDef instance */
208 : static GstTensorDecoderDef flexBuf = { .modename = decoder_subplugin_flexbuf,
209 : .init = flxd_init,
210 : .exit = flxd_exit,
211 : .setOption = flxd_setOption,
212 : .getOutCaps = flxd_getOutCaps,
213 : .decode = flxd_decode,
214 : .getTransformSize = NULL };
215 :
216 : #ifdef __cplusplus
217 : extern "C" {
218 : #endif /* __cplusplus */
219 :
220 : /** @brief Initialize this object for tensordec-plugin */
221 : void
222 52 : init_flxd (void)
223 : {
224 52 : nnstreamer_decoder_probe (&flexBuf);
225 52 : }
226 :
227 : /** @brief Destruct this object for tensordec-plugin */
228 : void
229 52 : fini_flxd (void)
230 : {
231 52 : nnstreamer_decoder_exit (flexBuf.modename);
232 52 : }
233 : #ifdef __cplusplus
234 : }
235 : #endif /* __cplusplus */
|
