Line data Source code
1 : /**
2 : * GStreamer / NNStreamer tensor_decoder subplugin, "direct video"
3 : * Copyright (C) 2018 Jijoong Moon <jijoong.moon@samsung.com>
4 : * Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>
5 : *
6 : * This library is free software; you can redistribute it and/or
7 : * modify it under the terms of the GNU Library General Public
8 : * License as published by the Free Software Foundation;
9 : * version 2.1 of the License.
10 : *
11 : * This library is distributed in the hope that it will be useful,
12 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 : * Library General Public License for more details.
15 : *
16 : */
17 : /**
18 : * @file tensordec-directvideo.c
19 : * @date 04 Nov 2018
20 : * @brief NNStreamer tensor-decoder subplugin, "direct video",
21 : * which converts tensors to video directly.
22 : *
23 : * @see https://github.com/nnstreamer/nnstreamer
24 : * @author Jijoong Moon <jijoong.moon@samsung.com>
25 : * @bug If the element size is 2 or larger, padding won't work.
26 : * GRAY16 types has size of 2 and if you have padding, it won't work.
27 : * To correct this, dv_decode() should be fixed.
28 : */
29 :
30 : #include <string.h>
31 : #include <glib.h>
32 : #include <gst/video/video-format.h>
33 : #include <nnstreamer_plugin_api_decoder.h>
34 : #include <nnstreamer_plugin_api.h>
35 : #include <nnstreamer_log.h>
36 : #include <nnstreamer_util.h>
37 : #include "tensordecutil.h"
38 :
39 : void init_dv (void) __attribute__ ((constructor));
40 : void fini_dv (void) __attribute__ ((destructor));
41 :
42 : #define DECODER_DV_FORMATS "{ GRAY8, RGB, BGR, RGBx, BGRx, xRGB, xBGR, RGBA, BGRA, ARGB, ABGR, GRAY16_BE, GRAY16_LE }"
43 :
44 : #define DECODER_DV_VIDEO_CAPS_STR \
45 : GST_VIDEO_CAPS_MAKE (DECODER_DV_FORMATS) \
46 : ", views = (int) 1, interlace-mode = (string) progressive"
47 :
48 : /**
49 : * @brief The supported video formats
50 : */
51 : typedef enum
52 : {
53 : DIRECT_VIDEO_FORMAT_UNKNOWN = 0,
54 :
55 : /* Single Channel, Default: GRAY8 */
56 : DIRECT_VIDEO_FORMAT_GRAY8 = 1,
57 :
58 : /* 3 Channels, Default: RGB */
59 : DIRECT_VIDEO_FORMAT_RGB = 2,
60 : DIRECT_VIDEO_FORMAT_BGR = 3,
61 :
62 : /* 4 Channels, Default: BGRx */
63 : DIRECT_VIDEO_FORMAT_RGBx = 4,
64 : DIRECT_VIDEO_FORMAT_BGRx = 5,
65 : DIRECT_VIDEO_FORMAT_xRGB = 6,
66 : DIRECT_VIDEO_FORMAT_xBGR = 7,
67 : DIRECT_VIDEO_FORMAT_RGBA = 8,
68 : DIRECT_VIDEO_FORMAT_BGRA = 9,
69 : DIRECT_VIDEO_FORMAT_ARGB = 10,
70 : DIRECT_VIDEO_FORMAT_ABGR = 11,
71 : DIRECT_VIDEO_FORMAT_GRAY16_BE = 12,
72 : DIRECT_VIDEO_FORMAT_GRAY16_LE = 13,
73 : } direct_video_formats;
74 :
75 : /**
76 : * @brief Data structure for direct video options.
77 : */
78 : typedef struct
79 : {
80 : /* From option1 */
81 : direct_video_formats format;
82 : } direct_video_ops;
83 :
84 : /**
85 : * @brief List of the formats of direct video
86 : */
87 : static const char *dv_formats[] = {
88 : [DIRECT_VIDEO_FORMAT_UNKNOWN] = "UNKNOWN",
89 : [DIRECT_VIDEO_FORMAT_GRAY8] = "GRAY8",
90 : [DIRECT_VIDEO_FORMAT_RGB] = "RGB",
91 : [DIRECT_VIDEO_FORMAT_BGR] = "BGR",
92 : [DIRECT_VIDEO_FORMAT_RGBx] = "RGBx",
93 : [DIRECT_VIDEO_FORMAT_BGRx] = "BGRx",
94 : [DIRECT_VIDEO_FORMAT_xRGB] = "xRGB",
95 : [DIRECT_VIDEO_FORMAT_xBGR] = "xBGR",
96 : [DIRECT_VIDEO_FORMAT_RGBA] = "RGBA",
97 : [DIRECT_VIDEO_FORMAT_BGRA] = "BGRA",
98 : [DIRECT_VIDEO_FORMAT_ARGB] = "ARGB",
99 : [DIRECT_VIDEO_FORMAT_ABGR] = "ABGR",
100 : [DIRECT_VIDEO_FORMAT_GRAY16_BE] = "GRAY16_BE",
101 : [DIRECT_VIDEO_FORMAT_GRAY16_LE] = "GRAY16_LE",
102 : NULL,
103 : };
104 :
105 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
106 : static int
107 32 : dv_init (void **pdata)
108 : {
109 : direct_video_ops *ddata;
110 32 : ddata = *pdata = g_try_new0 (direct_video_ops, 1);
111 :
112 32 : if (ddata == NULL) {
113 0 : GST_ERROR ("Failed to allocate memory for decoder subplugin.");
114 0 : return FALSE;
115 : }
116 :
117 32 : ddata->format = DIRECT_VIDEO_FORMAT_UNKNOWN;
118 :
119 32 : return TRUE;
120 : }
121 :
122 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
123 : static void
124 31 : dv_exit (void **pdata)
125 : {
126 31 : if (pdata)
127 31 : g_free (*pdata);
128 31 : return;
129 : }
130 :
131 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
132 : static int
133 42 : dv_setOption (void **pdata, int opNum, const char *param)
134 : {
135 : direct_video_ops *ddata;
136 :
137 42 : if (!pdata || !*pdata) {
138 0 : GST_ERROR ("There is no plugin data.");
139 0 : return FALSE;
140 : }
141 :
142 42 : if (NULL == param || *param == '\0') {
143 0 : GST_ERROR ("Please set the valid value at option.");
144 0 : return FALSE;
145 : }
146 :
147 42 : ddata = *pdata;
148 :
149 : /* When the dimension[0] is 4, the video format will be decided by option1. */
150 42 : switch (opNum) {
151 18 : case 0:
152 : {
153 18 : int f = find_key_strv (dv_formats, param);
154 :
155 18 : ddata->format = (f < 0) ? DIRECT_VIDEO_FORMAT_UNKNOWN : f;
156 18 : if (ddata->format == DIRECT_VIDEO_FORMAT_UNKNOWN) {
157 3 : return FALSE;
158 : }
159 15 : break;
160 : }
161 24 : default:
162 24 : break;
163 : }
164 :
165 39 : return TRUE;
166 : }
167 :
168 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
169 : static GstCaps *
170 205 : dv_getOutCaps (void **pdata, const GstTensorsConfig * config)
171 : {
172 205 : direct_video_ops *ddata = *pdata;
173 : /* Old gst_tensordec_video_caps_from_config () had this */
174 : GstVideoFormat format;
175 : gint width, height, channel;
176 : GstCaps *caps;
177 205 : guint element_size_from_cap = 1; /** Assume 1 byte per element */
178 205 : tensor_type input_tensor_type = config->info.info[0].type;
179 :
180 205 : g_return_val_if_fail (config != NULL, NULL);
181 205 : GST_INFO ("Num Tensors = %d", config->info.num_tensors);
182 205 : g_return_val_if_fail (config->info.num_tensors >= 1, NULL);
183 :
184 : /* Direct video uses the first tensor only even if it's multi-tensor */
185 205 : channel = config->info.info[0].dimension[0];
186 205 : if (channel == 1) {
187 32 : switch (ddata->format) {
188 4 : case DIRECT_VIDEO_FORMAT_GRAY8:
189 4 : format = GST_VIDEO_FORMAT_GRAY8;
190 4 : break;
191 8 : case DIRECT_VIDEO_FORMAT_GRAY16_BE:
192 8 : format = GST_VIDEO_FORMAT_GRAY16_BE;
193 8 : element_size_from_cap = 2;
194 8 : break;
195 8 : case DIRECT_VIDEO_FORMAT_GRAY16_LE:
196 8 : format = GST_VIDEO_FORMAT_GRAY16_LE;
197 8 : element_size_from_cap = 2;
198 8 : break;
199 12 : case DIRECT_VIDEO_FORMAT_UNKNOWN:
200 : default:
201 12 : GST_WARNING ("Default format has been applied: GRAY8");
202 12 : format = GST_VIDEO_FORMAT_GRAY8;
203 12 : break;
204 : }
205 173 : } else if (channel == 3) {
206 47 : switch (ddata->format) {
207 4 : case DIRECT_VIDEO_FORMAT_RGB:
208 4 : format = GST_VIDEO_FORMAT_RGB;
209 4 : break;
210 4 : case DIRECT_VIDEO_FORMAT_BGR:
211 4 : format = GST_VIDEO_FORMAT_BGR;
212 4 : break;
213 39 : case DIRECT_VIDEO_FORMAT_UNKNOWN:
214 : default:
215 39 : GST_WARNING ("Default format has been applied: RGB");
216 39 : format = GST_VIDEO_FORMAT_RGB;
217 : }
218 126 : } else if (channel == 4) {
219 56 : switch (ddata->format) {
220 4 : case DIRECT_VIDEO_FORMAT_RGBx:
221 4 : format = GST_VIDEO_FORMAT_RGBx;
222 4 : break;
223 4 : case DIRECT_VIDEO_FORMAT_BGRx:
224 4 : format = GST_VIDEO_FORMAT_BGRx;
225 4 : break;
226 4 : case DIRECT_VIDEO_FORMAT_xRGB:
227 4 : format = GST_VIDEO_FORMAT_xRGB;
228 4 : break;
229 4 : case DIRECT_VIDEO_FORMAT_xBGR:
230 4 : format = GST_VIDEO_FORMAT_xBGR;
231 4 : break;
232 4 : case DIRECT_VIDEO_FORMAT_RGBA:
233 4 : format = GST_VIDEO_FORMAT_RGBA;
234 4 : break;
235 4 : case DIRECT_VIDEO_FORMAT_BGRA:
236 4 : format = GST_VIDEO_FORMAT_BGRA;
237 4 : break;
238 4 : case DIRECT_VIDEO_FORMAT_ARGB:
239 4 : format = GST_VIDEO_FORMAT_ARGB;
240 4 : break;
241 4 : case DIRECT_VIDEO_FORMAT_ABGR:
242 4 : format = GST_VIDEO_FORMAT_ABGR;
243 4 : break;
244 24 : case DIRECT_VIDEO_FORMAT_UNKNOWN:
245 : default:
246 24 : GST_WARNING ("Default format has been applied: BGRx");
247 24 : format = GST_VIDEO_FORMAT_BGRx;
248 24 : break;
249 : }
250 : } else {
251 70 : GST_ERROR ("%d channel is not supported", channel);
252 70 : return NULL;
253 : }
254 :
255 135 : if (gst_tensor_get_element_size (input_tensor_type) != element_size_from_cap) {
256 0 : GST_ERROR ("The element size of input tensor (%" G_GSIZE_FORMAT
257 : " byte / %s) for tensor_decoder::direct_video must be same as the element size of output (%u byte / %s). Note that except for GrayScale-16 format, it should be 1 byte / element, normally; i.e., RGB, RGBA, and BGRx. It is recommended to convert to UINT8 tensor stream or UINT16 tensor stream (GreyScale-16) before tensor_decoder::direct_video.",
258 : gst_tensor_get_element_size (input_tensor_type),
259 : gst_tensor_get_type_string (input_tensor_type),
260 : element_size_from_cap,
261 : ((element_size_from_cap == 2) ? "uint16" : "uint8"));
262 0 : return NULL;
263 : }
264 :
265 135 : if (input_tensor_type != _NNS_UINT8 && input_tensor_type != _NNS_UINT16) {
266 0 : GST_WARNING
267 : ("The input tensor type for tensor_decoder::direct_video is recommended to be either UINT8 or UINT16. The current type, %s, does not incur buffer size mismatch, but the actual behavior might be inconsistent. Try UINT8/UINT16, which is meant to be the video/x-raw element type.",
268 : gst_tensor_get_type_string (input_tensor_type));
269 : }
270 :
271 135 : width = config->info.info[0].dimension[1];
272 135 : height = config->info.info[0].dimension[2];
273 :
274 135 : caps = gst_caps_from_string (DECODER_DV_VIDEO_CAPS_STR);
275 :
276 135 : if (format != GST_VIDEO_FORMAT_UNKNOWN) {
277 135 : const char *format_string = gst_video_format_to_string (format);
278 135 : gst_caps_set_simple (caps, "format", G_TYPE_STRING, format_string, NULL);
279 : }
280 :
281 135 : if (width > 0) {
282 131 : gst_caps_set_simple (caps, "width", G_TYPE_INT, width, NULL);
283 : }
284 :
285 135 : if (height > 0) {
286 131 : gst_caps_set_simple (caps, "height", G_TYPE_INT, height, NULL);
287 : }
288 :
289 135 : setFramerateFromConfig (caps, config);
290 :
291 135 : return gst_caps_simplify (caps);
292 : }
293 :
294 : /** @brief get video output buffer size */
295 : static size_t
296 103 : _get_video_xraw_bufsize (const tensor_dim dim, gsize data_size)
297 : {
298 : /* dim[0] is bpp and there is zeropadding only when dim[0]%4 > 0 */
299 103 : return (size_t) ((dim[0] * dim[1] - 1) / 4 + 1) * 4 * dim[2] * data_size;
300 : }
301 :
302 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
303 : static size_t
304 51 : dv_getTransformSize (void **pdata, const GstTensorsConfig * config,
305 : GstCaps * caps, size_t size, GstCaps * othercaps, GstPadDirection direction)
306 : {
307 : /* Direct video uses the first tensor only even if it's multi-tensor */
308 51 : const uint32_t *dim = &(config->info.info[0].dimension[0]);
309 51 : gsize data_size = gst_tensor_get_element_size (config->info.info[0].type);
310 51 : gsize transform_size = 0;
311 : UNUSED (pdata);
312 : UNUSED (caps);
313 : UNUSED (size);
314 : UNUSED (othercaps);
315 :
316 51 : if (direction == GST_PAD_SINK)
317 51 : transform_size = _get_video_xraw_bufsize (dim, data_size);
318 :
319 51 : return transform_size;
320 : }
321 :
322 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
323 : static GstFlowReturn
324 52 : dv_decode (void **pdata, const GstTensorsConfig * config,
325 : const GstTensorMemory * input, GstBuffer * outbuf)
326 : {
327 : GstMapInfo out_info;
328 : GstMemory *out_mem;
329 : /* Direct video uses the first tensor only even if it's multi-tensor */
330 52 : const uint32_t *dim = &(config->info.info[0].dimension[0]);
331 52 : gsize data_size = gst_tensor_get_element_size (config->info.info[0].type);
332 :
333 52 : size_t size = _get_video_xraw_bufsize (dim, data_size);
334 : UNUSED (pdata);
335 :
336 52 : g_assert (outbuf);
337 52 : if (gst_buffer_get_size (outbuf) > 0 && gst_buffer_get_size (outbuf) != size) {
338 0 : gst_buffer_set_size (outbuf, size);
339 : }
340 :
341 52 : if (gst_buffer_get_size (outbuf) == 0) {
342 0 : out_mem = gst_allocator_alloc (NULL, size, NULL);
343 : } else {
344 : /* Don't reallocate. Reuse what's already given */
345 52 : out_mem = gst_buffer_get_all_memory (outbuf);
346 : }
347 52 : if (!gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) {
348 0 : gst_memory_unref (out_mem);
349 0 : ml_loge ("Cannot map output memory / tensordec-directvideo.\n");
350 52 : return GST_FLOW_ERROR;
351 : }
352 :
353 52 : if (0 == ((dim[0] * dim[1]) % 4)) {
354 : /* No Padding Required */
355 50 : memcpy (out_info.data, input->data, input->size);
356 : } else {
357 : /* Do Padding */
358 : unsigned int h;
359 : uint8_t *ptr, *inp;
360 :
361 2 : ptr = (uint8_t *) out_info.data;
362 2 : inp = (uint8_t *) input->data;
363 693 : for (h = 0; h < dim[2]; h++) {
364 691 : memcpy (ptr, inp, (size_t) dim[0] * dim[1]);
365 691 : inp += (dim[0] * dim[1]);
366 691 : ptr += ((dim[0] * dim[1] - 1) / 4 + 1) * 4;
367 : }
368 : }
369 52 : gst_memory_unmap (out_mem, &out_info);
370 :
371 52 : if (gst_buffer_get_size (outbuf) == 0)
372 0 : gst_buffer_append_memory (outbuf, out_mem);
373 : else
374 52 : gst_buffer_replace_all_memory (outbuf, out_mem);
375 :
376 : /** @todo Caller of dv_decode in tensordec.c should call gst_memory_unmap to inbuf */
377 :
378 52 : return GST_FLOW_OK;
379 : }
380 :
381 : static gchar decoder_subplugin_direct_video[] = "direct_video";
382 :
383 : /** @brief Direct-Video tensordec-plugin GstTensorDecoderDef instance */
384 : static GstTensorDecoderDef directVideo = {
385 : .modename = decoder_subplugin_direct_video,
386 : .init = dv_init,
387 : .exit = dv_exit,
388 : .setOption = dv_setOption,
389 : .getOutCaps = dv_getOutCaps,
390 : .getTransformSize = dv_getTransformSize,
391 : .decode = dv_decode
392 : };
393 :
394 : /** @brief Initialize this object for tensordec-plugin */
395 : void
396 58 : init_dv (void)
397 : {
398 58 : nnstreamer_decoder_probe (&directVideo);
399 58 : nnstreamer_decoder_set_custom_property_desc (decoder_subplugin_direct_video,
400 : "option1",
401 : "The output video format. If this is unspecified, it is 'GRAY8' (dim[0]/channel == 1), 'RGB' (dim[0]/channel == 3), or 'BGRx' (dim[0]/channel == 4). Available options are: "
402 : DECODER_DV_FORMATS, NULL);
403 58 : }
404 :
405 : /** @brief Destruct this object for tensordec-plugin */
406 : void
407 58 : fini_dv (void)
408 : {
409 58 : nnstreamer_decoder_exit (directVideo.modename);
410 58 : }
|
