Line data Source code
1 : /**
2 : * GStreamer / NNStreamer tensor_decoder subplugin, "image segment"
3 : * Copyright (C) 2019 Jihoon Lee <ulla4571@gmail.com>
4 : * Copyright (C) 2019 niklasjang <niklasjang@gmail.com>
5 : * Copyright (C) 2020 Dongju Chae <dongju.chae@samsung.com>
6 : *
7 : * This library is free software; you can redistribute it and/or
8 : * modify it under the terms of the GNU Library General Public
9 : * License as published by the Free Software Foundation;
10 : * version 2.1 of the License.
11 : *
12 : * This library is distributed in the hope that it will be useful,
13 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : * Library General Public License for more details.
16 : *
17 : */
18 : /**
19 : * @file tensordec-imagesegment.c
20 : * @date 19 Oct 2019
21 : * @brief NNStreamer tensor-decoder subplugin, "image segment",
22 : * which detects objects and paints their regions.
23 : *
24 : * @see https://github.com/nnstreamer/nnstreamer
25 : * @author Jihoon Lee <ulla4571@gmail.com>
26 : * niklasjang <niklasjang@gmail.com>
27 : * Dongju Chae <dongju.chae@samsung.com>
28 : * @bug No known bugs except for NYI items
29 : *
30 : * option1: Decoder mode of image segmentation
31 : * Available : tflite-deeplab
32 : * Available : snpe-deeplab
33 : * Available : snpe-depth
34 : *
35 : * option2: Maximum number of class labels (except background), default is 20 (Pascal)
36 : *
37 : * expected models
38 : * - tflite-deeplab : deeplabv3_257_mv_gpu.tflite (designed for embedded devices)
39 : * - snpe-deeplab : deeplabv3_mnv2_pascal_train_aug.dlc (converted from a TF model)
40 : * - snpe-depth : any snpe models (.dlc) producing grayscale images
41 : *
42 : * expected input dims
43 : * - tflite-deeplab : #labels x width x height (float32, label probability)
44 : * (e.g., 21 x 257 x 257)
45 : * - snpe-deeplab : width x height x 1 (float32, label index)
46 : * (e.g., 513 x 513 x 1)
47 : * - snpe-depth : 1 x width x height (float32, grayscale)
48 : * (e.g., 1 x 320 x 240)
49 : *
50 : * pipeline:
51 : * filesrc
52 : * |
53 : * decodebin
54 : * |
55 : * videoconvert
56 : * |
57 : * videoscale
58 : * |
59 : * imagefreeze -- tee ----------------------------------------------- videomixer -- videoconvert -- autovideosink
60 : * | |
61 : * tensor_converter -- tensor_transform -- tensor_filter -- tensor_decoder
62 : *
63 : * - Used model is deeplabv3_257_mv_gpu.tflite.
64 : * - Resize image into 257:257 at the first videoscale.
65 : * - Transform RGB value into float32 in range [0,1] at tensor_transform.
66 : *
67 : * gst-launch-1.0 -v \
68 : * filesrc location=cat.png ! decodebin ! videoconvert ! videoscale ! imagefreeze !\
69 : * video/x-raw,format=RGB,width=257,height=257,framerate=10/1 ! tee name=t \
70 : * t. ! queue ! mix. \
71 : * t. ! queue ! tensor_converter !\
72 : * tensor_transform mode=arithmetic option=typecast:float32,add:0.0,div:255.0 !\
73 : * tensor_filter framework=tensorflow-lite model=deeplabv3_257_mv_gpu.tflite !\
74 : * tensor_decoder mode=image_segment option1=tflite-deeplab ! mix. \
75 : * videomixer name=mix sink_0::alpha=0.7 sink_1::alpha=0.6 ! videoconvert ! videoscale ! autovideosink \
76 : */
77 :
78 : #include <string.h>
79 : #include <glib.h>
80 : #include <gst/video/video-format.h>
81 : #include <nnstreamer_plugin_api_decoder.h>
82 : #include <nnstreamer_plugin_api.h>
83 : #include <nnstreamer_log.h>
84 : #include <nnstreamer_util.h>
85 : #include "tensordecutil.h"
86 :
87 : #if defined(__aarch64__)
88 : #include <arm_neon.h>
89 :
90 : #define NEON64_ENABLED
91 : #define GRAYSCALE_HEX (0x00010101)
92 : #define ALPHA_HEX (0xFF000000)
93 : #endif
94 :
95 : #define DEFAULT_LABELS (20)
96 : #define RGBA_CHANNEL (4)
97 : #define MAX_RGB (255)
98 :
99 : void init_is (void) __attribute__ ((constructor));
100 : void fini_is (void) __attribute__ ((destructor));
101 :
102 : static const float DETECTION_THRESHOLD = 0.5f;
103 :
104 : /**
105 : * @brief There can be different schemes for image segmentation
106 : */
107 : typedef enum
108 : {
109 : MODE_TFLITE_DEEPLAB = 0,
110 : MODE_SNPE_DEEPLAB = 1,
111 : MODE_SNPE_DEPTH = 2,
112 : MODE_UNKNOWN,
113 : } image_segment_modes;
114 :
115 : /**
116 : * @brief List of image-segmentation decoding schemes in string
117 : */
118 : static const char *is_modes[] = {
119 : [MODE_TFLITE_DEEPLAB] = "tflite-deeplab",
120 : [MODE_SNPE_DEEPLAB] = "snpe-deeplab",
121 : [MODE_SNPE_DEPTH] = "snpe-depth",
122 : NULL,
123 : };
124 :
125 : /**
126 : * @brief Data structure for image segmentation info
127 : */
128 : typedef struct
129 : {
130 : image_segment_modes mode; /**< The image segmentation decoding mode */
131 : float *segment_map; /**< The image segmentated map */
132 :
133 : guint max_labels; /**< Maximum number of labels */
134 : guint *color_map; /**< The RGBA color map (up to max labels) */
135 :
136 : guint width; /**< Input video width */
137 : guint height; /**< Input video height */
138 :
139 : GRand *rand; /**< random value generator */
140 : guint rgb_modifier; /**< rgb modifier according to # labels */
141 : } image_segments;
142 :
143 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
144 : static int
145 5 : is_init (void **pdata)
146 : {
147 : image_segments *idata;
148 :
149 5 : idata = *pdata = g_new0 (image_segments, 1);
150 5 : if (idata == NULL) {
151 0 : GST_ERROR ("Failed to allocate memory for decoder subplugin.");
152 0 : return FALSE;
153 : }
154 :
155 5 : idata->rand = g_rand_new ();
156 5 : idata->mode = MODE_UNKNOWN;
157 5 : idata->width = 0;
158 5 : idata->height = 0;
159 5 : idata->max_labels = DEFAULT_LABELS;
160 5 : idata->segment_map = NULL;
161 5 : idata->color_map = NULL;
162 5 : idata->rgb_modifier = 0;
163 :
164 5 : return TRUE;
165 : }
166 :
167 : /** @brief Free the allocated resources */
168 : static void
169 5 : _free_resources (image_segments * idata)
170 : {
171 5 : g_free (idata->segment_map);
172 5 : g_free (idata->color_map);
173 5 : g_rand_free (idata->rand);
174 :
175 5 : idata->segment_map = NULL;
176 5 : idata->color_map = NULL;
177 5 : idata->rand = NULL;
178 5 : }
179 :
180 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
181 : static void
182 5 : is_exit (void **pdata)
183 : {
184 5 : image_segments *idata = *pdata;
185 :
186 5 : _free_resources (idata);
187 :
188 5 : g_free (*pdata);
189 5 : *pdata = NULL;
190 5 : }
191 :
192 : /** @brief fill rgba color map */
193 : static void
194 3 : _fill_color_map (image_segments * idata)
195 : {
196 : guint i;
197 :
198 3 : idata->color_map[0] = 0; /* background */
199 :
200 : #if defined (NEON64_ENABLED)
201 : idata->rgb_modifier = 0xFFFFFF / (idata->max_labels + 1);
202 : for (i = 1; i <= idata->max_labels; i++) {
203 : /* colors should be the same with neon calculations */
204 : idata->color_map[i] = idata->rgb_modifier * i;
205 : ((guint8 *) & idata->color_map[i])[3] = '\xff'; /* alpha */
206 : }
207 : #else
208 223 : for (i = 1; i <= idata->max_labels; i++) {
209 : /* any color value would be acceptable */
210 220 : idata->color_map[i] = g_rand_int_range (idata->rand, 0x101010, 0xFFFFFF);
211 220 : ((guint8 *) & idata->color_map[i])[3] = '\xff'; /* alpha */
212 : }
213 : #endif
214 3 : }
215 :
216 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
217 : static int
218 6 : is_setOption (void **pdata, int op_num, const char *param)
219 : {
220 6 : image_segments *idata = *pdata;
221 :
222 6 : if (op_num == 0) {
223 : /* The first option indicates mode of image segmentation decoder */
224 5 : image_segment_modes previous = idata->mode;
225 5 : idata->mode = find_key_strv (is_modes, param);
226 :
227 5 : if (NULL == param || *param == '\0') {
228 0 : GST_ERROR ("Please set the valid mode at option1");
229 0 : return FALSE;
230 : }
231 :
232 5 : if (idata->mode != previous && idata->mode != MODE_UNKNOWN) {
233 5 : return TRUE;
234 : }
235 0 : return TRUE;
236 1 : } else if (op_num == 1) {
237 1 : guint64 max_labels_64 = g_ascii_strtoll (param, NULL, 10);
238 1 : if (max_labels_64 != 0 && max_labels_64 <= UINT_MAX)
239 1 : idata->max_labels = (guint) max_labels_64;
240 : }
241 :
242 1 : GST_WARNING ("mode-option-\"%d\" is not defined.", op_num);
243 1 : return TRUE;
244 : }
245 :
246 : /** @brief Initialize image_segments per mode */
247 : static gboolean
248 10 : _init_modes (image_segments * idata)
249 : {
250 10 : if (idata->mode == MODE_TFLITE_DEEPLAB) {
251 : /* init image segments if seg map is null */
252 5 : if (idata->segment_map == NULL)
253 2 : idata->segment_map = g_new0 (float, idata->height * idata->width);
254 :
255 5 : if (idata->color_map == NULL) {
256 2 : idata->color_map = g_new (guint, idata->max_labels + 1);
257 2 : _fill_color_map (idata);
258 : }
259 :
260 5 : return TRUE;
261 5 : } else if (idata->mode == MODE_SNPE_DEEPLAB) {
262 4 : if (idata->color_map == NULL) {
263 1 : idata->color_map = g_new (guint, idata->max_labels + 1);
264 1 : _fill_color_map (idata);
265 : }
266 4 : return TRUE;
267 1 : } else if (idata->mode == MODE_SNPE_DEPTH) {
268 1 : return TRUE;
269 : }
270 :
271 0 : GST_ERROR ("Failed to initialize, unknown mode %d.", idata->mode);
272 0 : return FALSE;
273 : }
274 :
275 : /**
276 : * @brief tensordec-plugin's GstTensorDecoderDef callback
277 : *
278 : * [DeeplabV3 model]
279 : * Just one tensor with [21(#labels):width:height:1], float32
280 : * Probability that each pixel is assumed to be labeled object.
281 : */
282 : static GstCaps *
283 34 : is_getOutCaps (void **pdata, const GstTensorsConfig * config)
284 : {
285 34 : image_segments *idata = *pdata;
286 : GstCaps *caps;
287 : char *str;
288 :
289 34 : g_return_val_if_fail (config != NULL, NULL);
290 34 : GST_INFO ("Num Tensors = %d", config->info.num_tensors);
291 34 : g_return_val_if_fail (config->info.num_tensors >= 1, NULL);
292 :
293 34 : if (idata->mode == MODE_SNPE_DEEPLAB) {
294 8 : idata->width = config->info.info[0].dimension[0];
295 8 : idata->height = config->info.info[0].dimension[1];
296 : } else {
297 26 : idata->width = config->info.info[0].dimension[1];
298 26 : idata->height = config->info.info[0].dimension[2];
299 : }
300 :
301 34 : str = g_strdup_printf ("video/x-raw, format = RGBA, "
302 : "width = %u, height = %u", idata->width, idata->height);
303 34 : caps = gst_caps_from_string (str);
304 34 : setFramerateFromConfig (caps, config);
305 34 : g_free (str);
306 :
307 34 : return gst_caps_simplify (caps);
308 : }
309 :
310 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
311 : static size_t
312 10 : is_getTransformSize (void **pdata, const GstTensorsConfig * config,
313 : GstCaps * caps, size_t size, GstCaps * othercaps, GstPadDirection direction)
314 : {
315 : UNUSED (pdata);
316 : UNUSED (config);
317 : UNUSED (caps);
318 : UNUSED (size);
319 : UNUSED (othercaps);
320 : UNUSED (direction);
321 :
322 10 : return 0;
323 : /** @todo Use appropriate values */
324 : }
325 :
326 : /** @brief Set color according to each pixel's label (RGBA) */
327 : static void
328 9 : set_color_according_to_label (image_segments * idata, GstMapInfo * out_info)
329 : {
330 9 : float *input = idata->segment_map;
331 9 : uint32_t *output = (uint32_t *) out_info->data;
332 9 : guint num_pixels = idata->height * idata->width;
333 9 : guint label_idx, idx = 0;
334 :
335 : #if defined (NEON64_ENABLED)
336 : float32x4_t v_src_float;
337 :
338 : uint32x4_t v_src_uint;
339 : uint32x4_t v_magic;
340 : uint32x4_t v_mask;
341 : uint32x4_t v_alpha;
342 : uint32x4_t v_zero;
343 :
344 : guint num_lanes = 4;
345 :
346 : v_magic = vdupq_n_u32 (idata->rgb_modifier);
347 : v_alpha = vdupq_n_u32 (ALPHA_HEX);
348 : v_zero = vdupq_n_u32 (0);
349 :
350 : for (idx = 0; idx < num_pixels; idx += num_lanes) {
351 : /* load float32 vector */
352 : v_src_float = vld1q_f32 (input);
353 : input += num_lanes;
354 :
355 : /* convert float32 vector to uint32 vector */
356 : v_src_uint = vcvtq_u32_f32 (v_src_float);
357 :
358 : /* multiply by magic number to fill RGB values */
359 : v_src_uint = vmulq_u32 (v_src_uint, v_magic);
360 :
361 : /* check whether the label is zero (i.e., background) */
362 : v_mask = vceqq_u32 (v_src_uint, v_zero);
363 : v_mask = vbslq_u32 (v_mask, v_zero, v_alpha);
364 :
365 : /* set the alpha value unless it's background */
366 : v_src_uint = vorrq_u32 (v_src_uint, v_mask);
367 :
368 : /* store uint32 vector */
369 : vst1q_u32 (output, v_src_uint);
370 : output += num_lanes;
371 : }
372 :
373 : if (num_pixels == idx)
374 : return;
375 :
376 : /* handle remaining data */
377 : input = (float *) idata->segment_map;
378 : output = (uint32_t *) out_info->data;
379 : idx -= num_lanes;
380 : #endif
381 351842 : for (; idx < num_pixels; idx++) {
382 351833 : label_idx = (guint) input[idx];
383 :
384 : /* If out-of-range, don't draw it */
385 351833 : if (G_UNLIKELY (label_idx > idata->max_labels))
386 0 : continue;
387 :
388 351833 : output[idx] = idata->color_map[label_idx];
389 : }
390 9 : }
391 :
392 : /** @brief Find the maximum grayscale value */
393 : static float
394 0 : find_max_grayscale (image_segments * idata)
395 : {
396 0 : float *input = idata->segment_map;
397 0 : float gray_max = 0.0;
398 0 : guint num_pixels = idata->height * idata->width;
399 0 : guint idx = 0;
400 :
401 : #if defined (NEON64_ENABLED)
402 : float32x4_t v_src, v_max;
403 : guint num_lanes = 4;
404 :
405 : v_max = vdupq_n_f32 (0);
406 :
407 : /* find the maximum value per lane */
408 : for (idx = 0; idx < num_pixels; idx += num_lanes) {
409 : v_src = vld1q_f32 (input);
410 : input += num_lanes;
411 :
412 : v_max = vmaxq_f32 (v_src, v_max);
413 : }
414 :
415 : /* find the maximum value among all lanes */
416 : gray_max = MAX (gray_max, vgetq_lane_f32 (v_max, 0));
417 : gray_max = MAX (gray_max, vgetq_lane_f32 (v_max, 1));
418 : gray_max = MAX (gray_max, vgetq_lane_f32 (v_max, 2));
419 : gray_max = MAX (gray_max, vgetq_lane_f32 (v_max, 3));
420 :
421 : if (num_pixels == idx)
422 : return gray_max;
423 :
424 : /* handle remaining data */
425 : input = idata->segment_map;
426 : idx -= num_lanes;
427 : #endif
428 0 : for (; idx < num_pixels; idx++)
429 0 : gray_max = MAX (gray_max, input[idx]);
430 :
431 0 : return gray_max;
432 : }
433 :
434 : /** @brief Set color with grayscale value */
435 : static void
436 0 : set_color_grayscale (image_segments * idata, GstMapInfo * out_info)
437 : {
438 0 : float *input = idata->segment_map;
439 0 : uint32_t *output = (uint32_t *) out_info->data;
440 : float max_grayscale;
441 0 : guint num_pixels = idata->height * idata->width;
442 : guint grayscale;
443 0 : guint idx = 0;
444 :
445 : /* find the maximum grayscale value */
446 0 : max_grayscale = find_max_grayscale (idata);
447 0 : if (G_UNLIKELY (max_grayscale == 0.0))
448 0 : return;
449 :
450 : #if defined (NEON64_ENABLED)
451 : {
452 : float32x4_t v_src_float;
453 : float32x4_t v_max_gray;
454 : float32x4_t v_max_rgb;
455 :
456 : uint32x4_t v_src_uint;
457 : uint32x4_t v_magic;
458 : uint32x4_t v_alpha;
459 :
460 : guint num_lanes = 4;
461 :
462 : v_max_gray = vdupq_n_f32 (max_grayscale);
463 : v_max_rgb = vdupq_n_f32 (MAX_RGB);
464 : v_magic = vdupq_n_u32 (GRAYSCALE_HEX);
465 : v_alpha = vdupq_n_u32 (ALPHA_HEX);
466 :
467 : for (idx = 0; idx < num_pixels; idx += num_lanes) {
468 : /* load float32 vector */
469 : v_src_float = vld1q_f32 (input);
470 : input += num_lanes;
471 :
472 : /* normalized_gray = (gray / max_gray) x max_rgb */
473 : v_src_float = vdivq_f32 (v_src_float, v_max_gray);
474 : v_src_float = vmulq_f32 (v_src_float, v_max_rgb);
475 :
476 : /* convert float32 vector to uint32 vector */
477 : v_src_uint = vcvtq_u32_f32 (v_src_float);
478 :
479 : /* multiply by magic number to fill the same RGB values */
480 : v_src_uint = vmulq_u32 (v_src_uint, v_magic);
481 : v_src_uint = vaddq_u32 (v_src_uint, v_alpha);
482 :
483 : /* store uint32 vector */
484 : vst1q_u32 (output, v_src_uint);
485 : output += num_lanes;
486 : }
487 :
488 : if (num_pixels == idx)
489 : return;
490 :
491 : /* handle remaining data */
492 : input = idata->segment_map;
493 : output = (uint32_t *) out_info->data;
494 : idx -= num_lanes;
495 : }
496 : #endif
497 0 : for (; idx < num_pixels; idx++) {
498 : /* normalize grayscale values to RGB_MAX */
499 0 : grayscale = (guint) ((input[idx] / max_grayscale) * MAX_RGB);
500 :
501 : /* Should be less than 256 */
502 0 : if (G_UNLIKELY (grayscale > MAX_RGB))
503 0 : continue;
504 :
505 0 : grayscale = grayscale | (grayscale << 8) | (grayscale << 16) | 0xFF000000;
506 0 : output[idx] = grayscale;
507 : }
508 : }
509 :
510 : /** @brief Set label index according to each pixel's label probabilities */
511 : static void
512 5 : set_label_index (image_segments * idata, void *data)
513 : {
514 5 : float *prob_map = (float *) data;
515 : guint idx, i, j;
516 : int max_idx;
517 : float max_prob;
518 5 : guint total_labels = idata->max_labels + 1;
519 :
520 5 : memset (idata->segment_map, '\x00',
521 5 : (size_t) idata->width * idata->height * sizeof (float));
522 :
523 1290 : for (i = 0; i < idata->height; i++) {
524 331530 : for (j = 0; j < idata->width; j++) {
525 330245 : max_idx = 0;
526 330245 : max_prob = prob_map[i * idata->width * total_labels + j * total_labels];
527 6935145 : for (idx = 1; idx < total_labels; idx++) {
528 6604900 : float prob = prob_map[i * idata->width * total_labels
529 6604900 : + j * total_labels + idx];
530 6604900 : if (prob > max_prob) {
531 0 : max_prob = prob;
532 0 : max_idx = idx;
533 : }
534 : }
535 330245 : if (max_prob > DETECTION_THRESHOLD) {
536 330245 : idata->segment_map[i * idata->width + j] = (float) max_idx;
537 : } /* otherwise, regarded as background */
538 : }
539 : }
540 5 : }
541 :
542 : /** @brief set color to output buffer depending on each mode */
543 : static void
544 9 : set_color (image_segments * idata, void *data, GstMapInfo * out_info)
545 : {
546 : /* tflite-deeplab needs to perform extra post-processing to set labels */
547 9 : if (idata->mode == MODE_TFLITE_DEEPLAB) {
548 5 : set_label_index (idata, data);
549 5 : set_color_according_to_label (idata, out_info);
550 5 : return;
551 : }
552 :
553 : /* snpe-deeplab already has labeled data as input */
554 4 : idata->segment_map = data;
555 :
556 4 : if (idata->mode == MODE_SNPE_DEEPLAB)
557 4 : set_color_according_to_label (idata, out_info);
558 0 : else if (idata->mode == MODE_SNPE_DEPTH)
559 0 : set_color_grayscale (idata, out_info);
560 :
561 4 : idata->segment_map = NULL;
562 : }
563 :
564 : /** @brief sanity check for each mode */
565 : static gboolean
566 10 : check_sanity (image_segments * idata, const GstTensorsConfig * config)
567 : {
568 10 : if (idata->mode == MODE_TFLITE_DEEPLAB) {
569 10 : return (config->info.info[0].type == _NNS_FLOAT32) &&
570 5 : (config->info.info[0].dimension[0] == idata->max_labels + 1);
571 5 : } else if (idata->mode == MODE_SNPE_DEEPLAB) {
572 4 : return (config->info.info[0].type == _NNS_FLOAT32);
573 1 : } else if (idata->mode == MODE_SNPE_DEPTH) {
574 2 : return (config->info.info[0].type == _NNS_FLOAT32) &&
575 1 : (config->info.info[0].dimension[0] == 1);
576 : }
577 :
578 0 : return FALSE;
579 : }
580 :
581 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
582 : static GstFlowReturn
583 10 : is_decode (void **pdata, const GstTensorsConfig * config,
584 : const GstTensorMemory * input, GstBuffer * outbuf)
585 : {
586 10 : image_segments *idata = *pdata;
587 10 : const size_t size = (size_t) idata->width * idata->height * RGBA_CHANNEL;
588 : gboolean need_output_alloc;
589 : GstMapInfo out_info;
590 : GstMemory *out_mem;
591 :
592 10 : if (!_init_modes (idata) || outbuf == NULL)
593 10 : return GST_FLOW_ERROR;
594 :
595 10 : need_output_alloc = (gst_buffer_get_size (outbuf) == 0);
596 10 : if (need_output_alloc) {
597 10 : out_mem = gst_allocator_alloc (NULL, size, NULL);
598 : } else {
599 0 : if (gst_buffer_get_size (outbuf) < size) {
600 0 : gst_buffer_set_size (outbuf, size);
601 : }
602 0 : out_mem = gst_buffer_get_all_memory (outbuf);
603 : }
604 10 : if (!gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) {
605 0 : ml_loge ("Cannot map output memory / tensordec-imagesegment.\n");
606 0 : goto error_free;
607 : }
608 :
609 10 : memset (out_info.data, '\x00', size);
610 :
611 10 : if (!check_sanity (idata, config)) {
612 1 : ml_loge ("Invalid input data format detected.\n");
613 1 : goto error_unmap;
614 : }
615 :
616 9 : set_color (idata, input->data, &out_info);
617 :
618 9 : gst_memory_unmap (out_mem, &out_info);
619 :
620 9 : if (need_output_alloc)
621 9 : gst_buffer_append_memory (outbuf, out_mem);
622 : else
623 0 : gst_buffer_replace_all_memory (outbuf, out_mem);
624 :
625 9 : return GST_FLOW_OK;
626 :
627 1 : error_unmap:
628 1 : gst_memory_unmap (out_mem, &out_info);
629 1 : error_free:
630 1 : gst_memory_unref (out_mem);
631 :
632 1 : return GST_FLOW_ERROR;
633 : }
634 :
635 : static gchar decoder_subplugin_image_segment[] = "image_segment";
636 :
637 : /** @brief Image Segmentation tensordec-plugin GstTensorDecoderDef instance */
638 : static GstTensorDecoderDef imageSegment = {
639 : .modename = decoder_subplugin_image_segment,
640 : .init = is_init,
641 : .exit = is_exit,
642 : .setOption = is_setOption,
643 : .getOutCaps = is_getOutCaps,
644 : .getTransformSize = is_getTransformSize,
645 : .decode = is_decode
646 : };
647 :
648 : /** @brief Initialize this object for tensordec-plugin */
649 : void
650 36 : init_is (void)
651 : {
652 36 : nnstreamer_decoder_probe (&imageSegment);
653 36 : nnstreamer_decoder_set_custom_property_desc ( decoder_subplugin_image_segment,
654 : "option1",
655 : "Mode of image segmentation. { tflite-deeplab (input: #labels x width x height (float32, label probability). e.g., deeplabv3_257_mv_gpu.tflite), snpe-deeplab (input: width x height x 1 (float32, label index) e.g., deeplabv3_mnv2_pascal_train_aug.dlc), snpe-depth (input: 1 x width x height (float32, grayscale) e.g., .dlc snpe models producing grayscale images) }",
656 : "option2", "Maximum number of labels. 20 is applied if not specified.",
657 : NULL);
658 36 : }
659 :
660 : /** @brief Destruct this object for tensordec-plugin */
661 : void
662 36 : fini_is (void)
663 : {
664 36 : nnstreamer_decoder_exit (imageSegment.modename);
665 36 : }
|
