Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer / NNStreamer tensor_decoder subplugin, "tensor_region"
4 : * Copyright (C) 2023 Harsh Jain <hjain24in@gmail.com>
5 : */
6 : /**
7 : * @file tensordec-tensor_region.c
8 : * @date 15th June, 2023
9 : * @brief NNStreamer tensor-decoder subplugin, "tensor region",
10 : * which converts tensors to cropping info for tensor _crop element
11 : * This code is NYI/WIP and not compilable.
12 : *
13 : * @see https://github.com/nnstreamer/nnstreamer
14 : * @author Harsh Jain <hjain24in@gmail.com>
15 : * @bug No known bugs except for NYI items
16 : *
17 : * option1: number of cropping regions required (default is 1)
18 : * option2: Location of label file
19 : * This is independent from option1
20 : * option3:
21 : * for mobilenet-ssd mode:
22 : * The option3 definition scheme is, in order, the following:
23 : * - box priors location file (mandatory)
24 : * - Detection threshold (optional, default set to 0.5)
25 : * - Y box scale (optional, default set to 10.0)
26 : * - X box scale (optional, default set to 10.0)
27 : * - h box scale (optional, default set to 5.0)
28 : * - w box scale (optional, default set to 5.0)
29 : * - IOU box valid threshold (optional, default set to 0.5)
30 : * The default parameters value could be set in the following ways:
31 : * option3=box-priors.txt:0.5:10.0:10.0:5.0:5.0:0.5
32 : * option3=box-priors.txt
33 : * option3=box-priors.txt::::::
34 : *
35 : * It's possible to set only few values, using the default values for
36 : * those not specified through the command line.
37 : * You could specify respectively the detection and IOU thresholds to 0.65
38 : * and 0.6 with the option3 parameter as follow:
39 : * option3=box-priors.txt:0.65:::::0.6
40 : * option4: Video input Dimension (WIDTH:HEIGHT) (default 300:300)
41 : * This is independent from option1
42 : *
43 : * @todo Remove duplicate codes*
44 : * @todo give support for other models*
45 : */
46 :
47 : #include <glib.h>
48 : #include <gst/gst.h>
49 : #include <math.h> /** expf */
50 : #include <nnstreamer_log.h>
51 : #include <nnstreamer_plugin_api.h>
52 : #include <nnstreamer_plugin_api_decoder.h>
53 : #include <nnstreamer_util.h>
54 : #include <stdint.h>
55 : #include <stdio.h>
56 : #include <stdlib.h>
57 : #include <string.h>
58 : #include "tensordecutil.h"
59 :
60 : #define _tensor_region_size_default_ 1
61 : void init_tr (void) __attribute__ ((constructor));
62 : void fini_tr (void) __attribute__ ((destructor));
63 :
64 : #define BOX_SIZE (4)
65 : #define MOBILENET_SSD_DETECTION_MAX (2034) /**add ssd_mobilenet v3 */
66 : #define MOBILENET_SSD_MAX_TENSORS (2U)
67 : #define INPUT_VIDEO_WIDTH_DEFAULT (300)
68 : #define INPUT_VIDEO_HEIGHT_DEFAULT (300)
69 : /**
70 : * @brief There can be different schemes for input tensor.
71 : */
72 : typedef enum
73 : {
74 : MOBILENET_SSD_BOUNDING_BOX = 0,
75 : BOUNDING_BOX_UNKNOWN,
76 : } tensor_region_modes;
77 :
78 : /** @brief Internal data structure for identifying cropping region */
79 : typedef struct {
80 : guint x;
81 : guint y;
82 : guint w;
83 : guint h;
84 : } crop_region;
85 :
86 : /**
87 : * @brief Data structure for SSD tensor_region info for mobilenet ssd model.
88 : */
89 : typedef struct {
90 : /**From option3, box prior data */
91 : char *box_prior_path; /**< Box Prior file path */
92 : gfloat box_priors[BOX_SIZE][MOBILENET_SSD_DETECTION_MAX + 1]; /** loaded box prior */
93 : #define MOBILENET_SSD_PARAMS_THRESHOLD_IDX 0
94 : #define MOBILENET_SSD_PARAMS_Y_SCALE_IDX 1
95 : #define MOBILENET_SSD_PARAMS_X_SCALE_IDX 2
96 : #define MOBILENET_SSD_PARAMS_H_SCALE_IDX 3
97 : #define MOBILENET_SSD_PARAMS_W_SCALE_IDX 4
98 : #define MOBILENET_SSD_PARAMS_IOU_THRESHOLD_IDX 5
99 : #define MOBILENET_SSD_PARAMS_MAX 6
100 :
101 : gfloat params[MOBILENET_SSD_PARAMS_MAX]; /** Post Processing parameters */
102 : gfloat sigmoid_threshold; /** Inverse value of valid detection threshold in sigmoid domain */
103 : } properties_MOBILENET_SSD;
104 :
105 : /** @brief Internal data structure for tensor region */
106 : typedef struct {
107 : tensor_region_modes mode; /**< When a mode is being changed, _cleanup_mode_properties () should be called before changing it */
108 : union {
109 : properties_MOBILENET_SSD mobilenet_ssd; /**< Properties for mobilenet_ssd */
110 : };
111 : imglabel_t labeldata;
112 : char *label_path;
113 :
114 : /**From option4 */
115 : guint i_width; /**< Input Video Width */
116 : guint i_height; /**< Input Video Height */
117 :
118 : /**From option1 */
119 : guint num; /**number of cropped regions required */
120 :
121 : guint max_detections;
122 : GArray *regions;
123 : gboolean flag_use_label;
124 :
125 : } tensor_region;
126 :
127 : /** @brief Internal function for mode change preparation */
128 : static void
129 2 : _cleanup_mode_properties (tensor_region *tr)
130 : {
131 2 : switch (tr->mode) {
132 2 : case MOBILENET_SSD_BOUNDING_BOX: {
133 2 : properties_MOBILENET_SSD *mobilenet_ssd = &tr->mobilenet_ssd;
134 2 : g_free (mobilenet_ssd->box_prior_path);
135 2 : break;
136 : }
137 0 : default:
138 0 : break;
139 : }
140 2 : }
141 :
142 : /** @brief Mathematic inverse of sigmoid function, aka logit */
143 : static float
144 4 : logit (float x)
145 : {
146 4 : if (x <= 0.0f)
147 0 : return -INFINITY;
148 :
149 4 : if (x >= 1.0f)
150 0 : return INFINITY;
151 :
152 4 : return log (x / (1.0 - x));
153 : }
154 :
155 : /**
156 : * @brief Load box-prior data from a file
157 : * @param[in/out] trData The internal data.
158 : * @return TRUE if loaded and configured. FALSE if failed to do so.
159 : */
160 : static int
161 2 : _mobilenet_ssd_loadBoxPrior (tensor_region *trData)
162 : {
163 2 : properties_MOBILENET_SSD *mobilenet_ssd = &trData->mobilenet_ssd;
164 2 : gboolean failed = FALSE;
165 2 : GError *err = NULL;
166 : gchar **priors;
167 2 : gchar *line = NULL;
168 2 : gchar *contents = NULL;
169 : guint row;
170 2 : gint prev_reg = -1;
171 :
172 : /**Read file contents */
173 2 : if (!g_file_get_contents (mobilenet_ssd->box_prior_path, &contents, NULL, &err)) {
174 0 : GST_ERROR ("Decoder/Tensor-Region/SSD's box prior file %s cannot be read: %s",
175 : mobilenet_ssd->box_prior_path, err->message);
176 0 : g_clear_error (&err);
177 2 : return FALSE;
178 : }
179 :
180 2 : priors = g_strsplit (contents, "\n", -1);
181 : /**If given prior file is inappropriate, report back to tensor-decoder */
182 2 : if (g_strv_length (priors) < BOX_SIZE) {
183 0 : ml_loge ("The given prior file, %s, should have at least %d lines.\n",
184 : mobilenet_ssd->box_prior_path, BOX_SIZE);
185 0 : failed = TRUE;
186 0 : goto error;
187 : }
188 :
189 10 : for (row = 0; row < BOX_SIZE; row++) {
190 8 : gint column = 0, registered = 0;
191 :
192 8 : line = priors[row];
193 8 : if (line) {
194 8 : gchar **list = g_strsplit_set (line, " \t,", -1);
195 : gchar *word;
196 :
197 15416 : while ((word = list[column]) != NULL) {
198 15408 : column++;
199 :
200 15408 : if (word && *word) {
201 15336 : if (registered > MOBILENET_SSD_DETECTION_MAX) {
202 0 : GST_WARNING ("Decoder/Tensor-region/SSD's box prior data file has too many priors. %d >= %d",
203 : registered, MOBILENET_SSD_DETECTION_MAX);
204 0 : break;
205 : }
206 : mobilenet_ssd->box_priors[row][registered]
207 15336 : = (gfloat) g_ascii_strtod (word, NULL);
208 15336 : registered++;
209 : }
210 : }
211 :
212 8 : g_strfreev (list);
213 : }
214 :
215 8 : if (prev_reg != -1 && prev_reg != registered) {
216 0 : GST_ERROR ("Decoder/Tensor-Region/SSD's box prior data file is not consistent.");
217 0 : failed = TRUE;
218 0 : break;
219 : }
220 8 : prev_reg = registered;
221 : }
222 :
223 2 : error:
224 2 : g_strfreev (priors);
225 2 : g_free (contents);
226 2 : return !failed;
227 : }
228 :
229 :
230 : /** @brief Initialize tensor_region per mode */
231 : static int
232 2 : _init_modes (tensor_region * trData){
233 2 : if(trData->mode == MOBILENET_SSD_BOUNDING_BOX){
234 2 : properties_MOBILENET_SSD *data = &trData->mobilenet_ssd;
235 : #define DETECTION_THRESHOLD (.5f)
236 : #define THRESHOLD_IOU (.5f)
237 : #define Y_SCALE (10.0f)
238 : #define X_SCALE (10.0f)
239 : #define H_SCALE (5.0f)
240 : #define W_SCALE (5.0f)
241 :
242 2 : data->params[MOBILENET_SSD_PARAMS_THRESHOLD_IDX] = DETECTION_THRESHOLD;
243 2 : data->params[MOBILENET_SSD_PARAMS_Y_SCALE_IDX] = Y_SCALE;
244 2 : data->params[MOBILENET_SSD_PARAMS_X_SCALE_IDX] = X_SCALE;
245 2 : data->params[MOBILENET_SSD_PARAMS_H_SCALE_IDX] = H_SCALE;
246 2 : data->params[MOBILENET_SSD_PARAMS_W_SCALE_IDX] = W_SCALE;
247 2 : data->params[MOBILENET_SSD_PARAMS_IOU_THRESHOLD_IDX] = THRESHOLD_IOU;
248 2 : data->sigmoid_threshold = logit (DETECTION_THRESHOLD);
249 2 : return TRUE;
250 : }
251 0 : return TRUE;
252 : }
253 :
254 :
255 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
256 : static int
257 2 : tr_init (void **pdata)
258 : {
259 : tensor_region *trData;
260 2 : trData = *pdata = g_new0 (tensor_region, 1);
261 2 : if (*pdata == NULL) {
262 0 : GST_ERROR ("Failed to allocate memory for decoder subplugin.");
263 0 : return FALSE;
264 : }
265 2 : trData->mode = MOBILENET_SSD_BOUNDING_BOX;
266 2 : trData->num = 1;
267 2 : trData->max_detections = 0;
268 2 : trData->regions = NULL;
269 2 : trData->flag_use_label = FALSE;
270 2 : trData->i_width = INPUT_VIDEO_WIDTH_DEFAULT;
271 2 : trData->i_height = INPUT_VIDEO_HEIGHT_DEFAULT;
272 2 : return _init_modes(trData);
273 : }
274 :
275 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
276 : static void
277 2 : tr_exit (void **pdata)
278 : {
279 2 : tensor_region *trData = *pdata;
280 2 : g_array_free (trData->regions, TRUE);
281 2 : _free_labels (&trData->labeldata);
282 2 : _cleanup_mode_properties (trData);
283 :
284 2 : if (trData->label_path)
285 2 : g_free (trData->label_path);
286 2 : g_free (*pdata);
287 2 : *pdata = NULL;
288 2 : }
289 :
290 : /** @brief configure per-mode option (option1) */
291 : static int
292 2 : _setOption_mode (tensor_region *trData, const char *param)
293 : {
294 2 : if (trData->mode == MOBILENET_SSD_BOUNDING_BOX) {
295 2 : properties_MOBILENET_SSD *mobilenet_ssd = &trData->mobilenet_ssd;
296 : gchar **options;
297 : int noptions, idx;
298 2 : int ret = 1;
299 :
300 2 : options = g_strsplit (param, ":", -1);
301 2 : noptions = g_strv_length (options);
302 :
303 2 : if (mobilenet_ssd->box_prior_path)
304 0 : g_free (mobilenet_ssd->box_prior_path);
305 :
306 2 : mobilenet_ssd->box_prior_path = g_strdup (options[0]);
307 :
308 2 : if (NULL != mobilenet_ssd->box_prior_path) {
309 2 : ret = _mobilenet_ssd_loadBoxPrior (trData);
310 2 : if (ret == 0) {
311 0 : g_strfreev (options);
312 0 : return ret;
313 : }
314 : }
315 :
316 2 : for (idx = 1; idx < noptions; idx++) {
317 0 : if (strlen (options[idx]) == 0)
318 0 : continue;
319 0 : mobilenet_ssd->params[idx - 1] = strtod (options[idx], NULL);
320 : }
321 :
322 : mobilenet_ssd->sigmoid_threshold
323 2 : = logit (mobilenet_ssd->params[MOBILENET_SSD_PARAMS_THRESHOLD_IDX]);
324 2 : g_strfreev (options);
325 2 : return ret;
326 : }
327 0 : return TRUE;
328 : }
329 :
330 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
331 : static int
332 6 : tr_setOption (void **pdata, int opNum, const char *param)
333 : {
334 6 : tensor_region *trData = *pdata;
335 6 : if (opNum == 0) {
336 : /**option1 number of crop regions required */
337 2 : trData->num = atoi (param);
338 2 : return TRUE;
339 4 : } else if (opNum == 1) {
340 : /**option2 label path for mobilenet_ssd model */
341 2 : if (NULL != trData->label_path)
342 0 : g_free (trData->label_path);
343 2 : trData->label_path = g_strdup (param);
344 :
345 2 : if (NULL != trData->label_path)
346 2 : loadImageLabels (trData->label_path, &trData->labeldata);
347 :
348 2 : if (trData->labeldata.total_labels > 0)
349 2 : return TRUE;
350 : else
351 0 : return FALSE;
352 2 : } else if (opNum == 2){
353 : /**option3 setting box prior path for mobilenet ssd model */
354 2 : return _setOption_mode (trData, param);
355 : }
356 0 : else if (opNum == 3) {
357 : /**option4 = input model size (width:height) */
358 : tensor_dim dim;
359 0 : int rank = gst_tensor_parse_dimension (param, dim);
360 :
361 0 : trData->i_width = 0;
362 0 : trData->i_height = 0;
363 0 : if (param == NULL || *param == '\0')
364 0 : return TRUE;
365 :
366 0 : if (rank < 2) {
367 0 : GST_ERROR ("mode-option-4 of tensor region is input video dimension (WIDTH:HEIGHT). The given parameter, \"%s\", is not acceptable.",
368 : param);
369 0 : return TRUE; /**Ignore this param */
370 : }
371 0 : if (rank > 2) {
372 0 : GST_WARNING ("mode-option-4 of tensor region is input video dimension (WIDTH:HEIGHT). The third and later elements of the given parameter, \"%s\", are ignored.",
373 : param);
374 : }
375 0 : trData->i_width = dim[0];
376 0 : trData->i_height = dim[1];
377 0 : return TRUE;
378 : }
379 :
380 0 : GST_INFO ("Property mode-option-%d is ignored", opNum + 1);
381 0 : return TRUE;
382 : }
383 :
384 : typedef struct {
385 : int valid;
386 : int class_id;
387 : gfloat score;
388 : int x;
389 : int y;
390 : int height;
391 : int width;
392 : } detected_object;
393 :
394 : /**
395 : * @brief transfer crop region info with the given results to the output buffer
396 : * @param[out] out_info The output buffer
397 : * @param[in] data The Tensor_region internal data.
398 : * @param[in] results The final results to be transferred.
399 : */
400 : static void
401 3 : gst_tensor_top_detectedObjects_cropInfo (GstMapInfo *out_info, const tensor_region *data, GArray *results)
402 : {
403 :
404 : guint i;
405 3 : gsize size = sizeof (crop_region); /**Assuming crop_region is a structure with four integer fields */
406 3 : guint *out_data = (guint *) out_info->data;
407 : crop_region region;
408 3 : guint maxx = MIN (results->len, data->num);
409 6 : for (i = 0; i < maxx; i++) {
410 3 : detected_object *temp = &g_array_index (results, detected_object, i);
411 3 : region.x = temp->x;
412 3 : region.y = temp->y;
413 3 : region.w = temp->width;
414 3 : region.h = temp->height;
415 3 : memcpy (out_data, ®ion, size);
416 3 : out_data += size / sizeof (guint);
417 : }
418 3 : }
419 :
420 : #define _expit(x) (1.f / (1.f + expf (-((float) x))))
421 :
422 :
423 : /**
424 : * @brief C++-Template-like box location calculation for box-priors
425 : * @bug This is not macro-argument safe. Use parenthesis!
426 : * @param[in] bb The configuration, "tensor region"
427 : * @param[in] index The index (3rd dimension of BOX_SIZE:1:MOBILENET_SSD_DETECTION_MAX:1)
428 : * @param[in] total_labels The count of total labels. We can get this from input tensor info. (1st dimension of LABEL_SIZE:MOBILENET_SSD_DETECTION_MAX:1:1)
429 : * @param[in] boxprior The box prior data from the box file of SSD.
430 : * @param[in] boxinputptr Cursor pointer of input + byte-per-index * index (box)
431 : * @param[in] detinputptr Cursor pointer of input + byte-per-index * index (detection)
432 : * @param[in] result The object returned. (pointer to object)
433 : */
434 : #define _get_object_i_mobilenet_ssd( \
435 : bb, index, total_labels, boxprior, boxinputptr, detinputptr, result) \
436 : do { \
437 : unsigned int c; \
438 : properties_MOBILENET_SSD *data = &bb->mobilenet_ssd; \
439 : float sigmoid_threshold = data->sigmoid_threshold; \
440 : float y_scale = data->params[MOBILENET_SSD_PARAMS_Y_SCALE_IDX]; \
441 : float x_scale = data->params[MOBILENET_SSD_PARAMS_X_SCALE_IDX]; \
442 : float h_scale = data->params[MOBILENET_SSD_PARAMS_H_SCALE_IDX]; \
443 : float w_scale = data->params[MOBILENET_SSD_PARAMS_W_SCALE_IDX]; \
444 : result->valid = FALSE; \
445 : for (c = 1; c < total_labels; c++) { \
446 : if (detinputptr[c] >= sigmoid_threshold) { \
447 : gfloat score = _expit (detinputptr[c]); \
448 : float ycenter \
449 : = boxinputptr[0] / y_scale * boxprior[2][index] + boxprior[0][index]; \
450 : float xcenter \
451 : = boxinputptr[1] / x_scale * boxprior[3][index] + boxprior[1][index]; \
452 : float h = (float) expf (boxinputptr[2] / h_scale) * boxprior[2][index]; \
453 : float w = (float) expf (boxinputptr[3] / w_scale) * boxprior[3][index]; \
454 : float ymin = ycenter - h / 2.f; \
455 : float xmin = xcenter - w / 2.f; \
456 : int x = xmin * bb->i_width; \
457 : int y = ymin * bb->i_height; \
458 : int width = w * bb->i_width; \
459 : int height = h * bb->i_height; \
460 : result->class_id = c; \
461 : result->x = MAX (0, x); \
462 : result->y = MAX (0, y); \
463 : result->width = width; \
464 : result->height = height; \
465 : result->score = score; \
466 : result->valid = TRUE; \
467 : break; \
468 : } \
469 : } \
470 : } while (0);
471 :
472 : /**
473 : * @brief C++-Template-like box location calculation for box-priors for Mobilenet SSD Model
474 : * @param[in] bb The configuration, "tensor region"
475 : * @param[in] type The tensor type of inputptr
476 : * @param[in] typename nnstreamer enum corresponding to the type
477 : * @param[in] boxprior The box prior data from the box file of MOBILENET_SSD.
478 : * @param[in] boxinput Input Tensor Data (Boxes)
479 : * @param[in] detinput Input Tensor Data (Detection). Null if not available. (numtensor ==1)
480 : * @param[in] config Tensor configs of the input tensors
481 : * @param[out] results The object returned. (GArray with detectedObject)
482 : */
483 : #define _get_objects_mobilenet_ssd( \
484 : bb, _type, typename, boxprior, boxinput, detinput, config, results) \
485 : case typename: \
486 : { \
487 : int d; \
488 : _type *boxinput_ = (_type *) boxinput; \
489 : size_t boxbpi = config->info.info[0].dimension[0]; \
490 : _type *detinput_ = (_type *) detinput; \
491 : size_t detbpi = config->info.info[1].dimension[0]; \
492 : int num = (MOBILENET_SSD_DETECTION_MAX > bb->max_detections) ? \
493 : bb->max_detections : \
494 : MOBILENET_SSD_DETECTION_MAX; \
495 : detected_object object = { \
496 : .valid = FALSE, .class_id = 0, .x = 0, .y = 0, .width = 0, .height = 0, .score = .0 \
497 : }; \
498 : for (d = 0; d < num; d++) { \
499 : _get_object_i_mobilenet_ssd (bb, d, detbpi, boxprior, \
500 : (boxinput_ + (d * boxbpi)), (detinput_ + (d * detbpi)), (&object)); \
501 : if (object.valid == TRUE) { \
502 : g_array_append_val (results, object); \
503 : } \
504 : } \
505 : } \
506 : break
507 :
508 : /** @brief Macro to simplify calling _get_objects_mobilenet_ssd */
509 : #define _get_objects_mobilenet_ssd_(type, typename) \
510 : _get_objects_mobilenet_ssd (trData, type, typename, (trData->mobilenet_ssd.box_priors), \
511 : (boxes->data), (detections->data), config, results)
512 :
513 : /**
514 : * @brief Compare Function for g_array_sort with detectedObject.
515 : */
516 : static gint
517 57 : compare_detection (gconstpointer _a, gconstpointer _b)
518 : {
519 57 : const detected_object *a = _a;
520 57 : const detected_object *b = _b;
521 :
522 : /**Larger comes first */
523 57 : return (a->score > b->score) ? -1 : ((a->score == b->score) ? 0 : 1);
524 : }
525 :
526 : /**
527 : * @brief Calculate the intersected surface
528 : */
529 : static gfloat
530 24 : iou (detected_object *a, detected_object *b)
531 : {
532 24 : int x1 = MAX (a->x, b->x);
533 24 : int y1 = MAX (a->y, b->y);
534 24 : int x2 = MIN (a->x + a->width, b->x + b->width);
535 24 : int y2 = MIN (a->y + a->height, b->y + b->height);
536 24 : int w = MAX (0, (x2 - x1 + 1));
537 24 : int h = MAX (0, (y2 - y1 + 1));
538 24 : float inter = w * h;
539 24 : float areaA = a->width * a->height;
540 24 : float areaB = b->width * b->height;
541 24 : float o = inter / (areaA + areaB - inter);
542 24 : return (o >= 0) ? o : 0;
543 : }
544 :
545 : /**
546 : * @brief Apply NMS to the given results (objects[])
547 : * @param[in/out] results The results to be filtered with nms
548 : */
549 : static void
550 3 : nms (GArray *results, gfloat threshold)
551 : {
552 : guint boxes_size;
553 : guint i, j;
554 :
555 3 : boxes_size = results->len;
556 3 : if (boxes_size == 0U)
557 0 : return;
558 :
559 3 : g_array_sort (results, compare_detection);
560 :
561 30 : for (i = 0; i < boxes_size; i++) {
562 27 : detected_object *a = &g_array_index (results, detected_object, i);
563 27 : if (a->valid == TRUE) {
564 27 : for (j = i + 1; j < boxes_size; j++) {
565 24 : detected_object *b = &g_array_index (results, detected_object, j);
566 24 : if (b->valid == TRUE) {
567 24 : if (iou (a, b) > threshold) {
568 24 : b->valid = FALSE;
569 : }
570 : }
571 : }
572 : }
573 : }
574 :
575 3 : i = 0;
576 : do {
577 27 : detected_object *a = &g_array_index (results, detected_object, i);
578 27 : if (a->valid == FALSE)
579 24 : g_array_remove_index (results, i);
580 : else
581 3 : i++;
582 27 : } while (i < results->len);
583 : }
584 :
585 : /**
586 : * @brief Private function to initialize the meta info
587 : */
588 : static void
589 3 : init_meta(GstTensorMetaInfo * meta, const tensor_region * trData){
590 3 : gst_tensor_meta_info_init (meta);
591 3 : meta->type = _NNS_UINT32;
592 3 : meta->dimension[0] = BOX_SIZE;
593 3 : meta->dimension[1] = trData->num;
594 3 : meta->media_type = _NNS_TENSOR;
595 3 : }
596 :
597 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
598 : static GstFlowReturn
599 3 : tr_decode (void **pdata, const GstTensorsConfig *config,
600 : const GstTensorMemory *input, GstBuffer *outbuf)
601 : {
602 3 : tensor_region *trData = *pdata;
603 : GstTensorMetaInfo meta;
604 : GstMapInfo out_info;
605 : GstMemory *out_mem, *tmp_mem;
606 3 : GArray *results = NULL;
607 3 : const guint num_tensors = config->info.num_tensors;
608 3 : gboolean need_output_alloc = gst_buffer_get_size (outbuf) == 0;
609 3 : const size_t size = (size_t) 4 * trData->num * sizeof(uint32_t); /**4 field per block */
610 :
611 3 : g_assert (outbuf);
612 : /** Ensure we have outbuf properly allocated */
613 3 : if (need_output_alloc) {
614 3 : out_mem = gst_allocator_alloc (NULL, size, NULL);
615 : } else {
616 0 : if (gst_buffer_get_size (outbuf) < size) {
617 0 : gst_buffer_set_size (outbuf, size);
618 : }
619 0 : out_mem = gst_buffer_get_all_memory (outbuf);
620 : }
621 3 : if (!gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) {
622 0 : ml_loge ("Cannot map output memory / tensordec-tensor_region.\n");
623 0 : goto error_free;
624 : }
625 :
626 : /** reset the buffer with 0 */
627 3 : memset (out_info.data, 0, size);
628 3 : if(trData->mode == MOBILENET_SSD_BOUNDING_BOX){
629 3 : const GstTensorMemory *boxes, *detections = NULL;
630 3 : properties_MOBILENET_SSD *data = &trData->mobilenet_ssd;
631 :
632 3 : g_assert (num_tensors >= MOBILENET_SSD_MAX_TENSORS);
633 3 : results = g_array_sized_new (FALSE, TRUE, sizeof (detected_object), 100);
634 :
635 3 : boxes = &input[0];
636 3 : if (num_tensors >= MOBILENET_SSD_MAX_TENSORS) /**lgtm[cpp/constant-comparison] */
637 3 : detections = &input[1];
638 :
639 3 : switch (config->info.info[0].type) {
640 0 : _get_objects_mobilenet_ssd_ (uint8_t, _NNS_UINT8);
641 0 : _get_objects_mobilenet_ssd_ (int8_t, _NNS_INT8);
642 0 : _get_objects_mobilenet_ssd_ (uint16_t, _NNS_UINT16);
643 0 : _get_objects_mobilenet_ssd_ (int16_t, _NNS_INT16);
644 0 : _get_objects_mobilenet_ssd_ (uint32_t, _NNS_UINT32);
645 0 : _get_objects_mobilenet_ssd_ (int32_t, _NNS_INT32);
646 0 : _get_objects_mobilenet_ssd_ (uint64_t, _NNS_UINT64);
647 0 : _get_objects_mobilenet_ssd_ (int64_t, _NNS_INT64);
648 522372 : _get_objects_mobilenet_ssd_ (float, _NNS_FLOAT32);
649 0 : _get_objects_mobilenet_ssd_ (double, _NNS_FLOAT64);
650 0 : default:
651 0 : g_assert (0);
652 : }
653 :
654 3 : nms (results, data->params[MOBILENET_SSD_PARAMS_IOU_THRESHOLD_IDX]);
655 : } else {
656 0 : GST_ERROR ("Failed to get output buffer, unknown mode %d.", trData->mode);
657 0 : goto error_unmap;
658 : }
659 3 : gst_tensor_top_detectedObjects_cropInfo (&out_info, trData, results);
660 :
661 3 : g_array_free (results, TRUE);
662 :
663 3 : gst_memory_unmap (out_mem, &out_info);
664 :
665 : /** converting to Flexible tensor since
666 : * info pad of tensor_crop has capability for flexible tensor stream
667 : */
668 3 : init_meta (&meta, trData);
669 3 : tmp_mem = out_mem;
670 3 : out_mem = gst_tensor_meta_info_append_header (&meta, tmp_mem);
671 3 : gst_memory_unref (tmp_mem);
672 :
673 3 : if (need_output_alloc) {
674 3 : gst_buffer_append_memory (outbuf, out_mem);
675 : } else {
676 0 : gst_buffer_replace_all_memory (outbuf, out_mem);
677 : }
678 :
679 3 : return GST_FLOW_OK;
680 0 : error_unmap:
681 0 : gst_memory_unmap (out_mem, &out_info);
682 0 : error_free:
683 0 : gst_memory_unref (out_mem);
684 :
685 0 : return GST_FLOW_ERROR;
686 : }
687 :
688 : /**
689 : * @brief set the max_detection
690 : */
691 : static int
692 10 : _set_max_detection (tensor_region *data, guint max_detection, unsigned int limit)
693 : {
694 : /**Check consistency with max_detection */
695 10 : if (data->max_detections == 0)
696 2 : data->max_detections = max_detection;
697 : else
698 8 : g_return_val_if_fail (max_detection == data->max_detections, FALSE);
699 :
700 10 : if (data->max_detections > limit) {
701 0 : GST_ERROR ("Incoming tensor has too large detection-max : %u", max_detection);
702 0 : return FALSE;
703 : }
704 10 : return TRUE;
705 : }
706 :
707 :
708 : /**
709 : * @brief tensordec-plugin's GstTensorDecoderDef callback
710 : *
711 : * [Mobilenet SSD Model]
712 : * The first tensor is boxes. BOX_SIZE : 1 : #MaxDetection, ANY-TYPE
713 : * The second tensor is labels. #MaxLabel : #MaxDetection, ANY-TYPE
714 : * Both tensors are MANDATORY!
715 : * If there are third or more tensors, such tensors will be ignored.
716 : */
717 :
718 : static GstCaps *
719 13 : tr_getOutCaps (void **pdata, const GstTensorsConfig *config)
720 : {
721 13 : tensor_region *data = *pdata;
722 : GstCaps *caps;
723 : char *str;
724 : guint max_detection, max_label;
725 : const uint32_t *dim1, *dim2;
726 : int i;
727 :
728 : /**Check if the first tensor is compatible */
729 13 : dim1 = config->info.info[0].dimension;
730 13 : g_return_val_if_fail (dim1[0] == BOX_SIZE, NULL);
731 10 : g_return_val_if_fail (dim1[1] == 1, NULL);
732 10 : max_detection = dim1[2];
733 10 : g_return_val_if_fail (max_detection > 0, NULL);
734 : /** @todo unused dimension value should be 0 */
735 140 : for (i = 3; i < NNS_TENSOR_RANK_LIMIT; i++)
736 130 : g_return_val_if_fail (dim1[i] == 0 || dim1[i] == 1, NULL);
737 :
738 : /**Check if the second tensor is compatible */
739 10 : dim2 = config->info.info[1].dimension;
740 10 : max_label = dim2[0];
741 10 : g_return_val_if_fail (max_label <= data->labeldata.total_labels, NULL);
742 10 : if (max_label < data->labeldata.total_labels)
743 5 : GST_WARNING ("The given tensor (2nd) has max_label (first dimension: %u) smaller than the number of labels in labels file (%s: %u).",
744 : max_label, data->label_path, data->labeldata.total_labels);
745 10 : g_return_val_if_fail (max_detection == dim2[1], NULL);
746 150 : for (i = 2; i < NNS_TENSOR_RANK_LIMIT; i++)
747 140 : g_return_val_if_fail (dim2[i] == 0 || dim2[i] == 1, NULL);
748 :
749 : /**Check consistency with max_detection */
750 10 : if (!_set_max_detection (data, max_detection, MOBILENET_SSD_DETECTION_MAX)) {
751 0 : return NULL;
752 : }
753 10 : str = g_strdup_printf("other/tensors,format=flexible");
754 10 : caps = gst_caps_from_string (str);
755 10 : setFramerateFromConfig (caps, config);
756 10 : g_free (str);
757 : (void) *pdata;
758 10 : return caps;
759 : }
760 :
761 :
762 : static gchar decoder_subplugin_tensor_region[] = "tensor_region";
763 :
764 : /** @brief Tensor Region tensordec-plugin GstTensorDecoderDef instance */
765 : static GstTensorDecoderDef tensorRegion = { .modename = decoder_subplugin_tensor_region,
766 : .init = tr_init,
767 : .exit = tr_exit,
768 : .setOption = tr_setOption,
769 : .getOutCaps = tr_getOutCaps,
770 : /** .getTransformSize = tr_getTransformsize, */
771 : .decode = tr_decode };
772 :
773 : /** @brief Initialize this object for tensordec-plugin */
774 : void
775 33 : init_tr (void)
776 : {
777 33 : nnstreamer_decoder_probe (&tensorRegion);
778 33 : }
779 :
780 : /** @brief Destruct this object for tensordec-plugin */
781 : void
782 33 : fini_tr (void)
783 : {
784 33 : nnstreamer_decoder_exit (tensorRegion.modename);
785 33 : }
|
