Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer / NNStreamer tensor-decoder bounding box properties
4 : * Copyright (C) 2024 Yelin Jeong <yelini.jeong@samsung.com>
5 : */
6 : /**
7 : * @file mobilenetssd.cc
8 : * @date 13 May 2024
9 : * @brief NNStreamer tensor-decoder bounding box properties
10 : *
11 : * @see https://github.com/nnstreamer/nnstreamer
12 : * @author Yelin Jeong <yelini.jeong@samsung.com>
13 : * @bug No known bugs except for NYI items
14 : *
15 : */
16 :
17 : #include "../tensordec-boundingbox.h"
18 :
19 : #define MAX_TENSORS (2U)
20 :
21 : #define THRESHOLD_IDX (0)
22 : #define Y_SCALE_IDX (1)
23 : #define X_SCALE_IDX (2)
24 : #define H_SCALE_IDX (3)
25 : #define W_SCALE_IDX (4)
26 : #define IOU_THRESHOLD_IDX (5)
27 :
28 : #define DETECTION_THRESHOLD_DEFAULT (0.5f)
29 : #define THRESHOLD_IOU_DEFAULT (0.5f)
30 : #define Y_SCALE_DEFAULT (10.0f)
31 : #define X_SCALE_DEFAULT (10.0f)
32 : #define H_SCALE_DEFAULT (5.0f)
33 : #define W_SCALE_DEFAULT (5.0f)
34 :
35 : #define BOX_SIZE (4)
36 : #define DETECTION_MAX (2034) /* add ssd_mobilenet v3 support */
37 : #define PARAMS_MAX (6)
38 :
39 : #define _expit(x) (1.f / (1.f + expf (-((float) x))))
40 :
41 : /**
42 : * @brief Class for MobilenetSSD box properties
43 : */
44 : class MobilenetSSD : public BoxProperties
45 : {
46 : public:
47 : MobilenetSSD ();
48 : ~MobilenetSSD ();
49 : int mobilenet_ssd_loadBoxPrior ();
50 :
51 : int setOptionInternal (const char *param);
52 : int checkCompatible (const GstTensorsConfig *config);
53 : GArray *decode (const GstTensorsConfig *config, const GstTensorMemory *input);
54 :
55 : private:
56 : char *box_prior_path; /**< Box Prior file path */
57 : gfloat box_priors[BOX_SIZE][DETECTION_MAX + 1]; /** loaded box prior */
58 : gfloat params[PARAMS_MAX]; /** Post Processing parameters */
59 : gfloat sigmoid_threshold; /** Inverse value of valid detection threshold in sigmoid domain */
60 : };
61 :
62 : /**
63 : * @brief C++-Template-like box location calculation for box-priors
64 : * @bug This is not macro-argument safe. Use parenthesis!
65 : * @param[in] bb The configuration, "bounding_boxes"
66 : * @param[in] index The index (3rd dimension of BOX_SIZE:1:DETECTION_MAX:1)
67 : * @param[in] total_labels The count of total labels. We can get this from input tensor info. (1st dimension of LABEL_SIZE:DETECTION_MAX:1:1)
68 : * @param[in] boxprior The box prior data from the box file of SSD.
69 : * @param[in] boxinputptr Cursor pointer of input + byte-per-index * index (box)
70 : * @param[in] detinputptr Cursor pointer of input + byte-per-index * index (detection)
71 : * @param[in] result The object returned. (pointer to object)
72 : */
73 : #define _get_object_i_mobilenet_ssd(index, total_labels, boxprior, \
74 : boxinputptr, detinputptr, result, i_width, i_height) \
75 : do { \
76 : unsigned int c; \
77 : gfloat highscore = -FLT_MAX; \
78 : float y_scale = params[Y_SCALE_IDX]; \
79 : float x_scale = params[X_SCALE_IDX]; \
80 : float h_scale = params[H_SCALE_IDX]; \
81 : float w_scale = params[W_SCALE_IDX]; \
82 : result->valid = FALSE; \
83 : for (c = 1; c < total_labels; c++) { \
84 : if (detinputptr[c] >= sigmoid_threshold) { \
85 : gfloat score = _expit (detinputptr[c]); \
86 : float ycenter \
87 : = boxinputptr[0] / y_scale * boxprior[2][index] + boxprior[0][index]; \
88 : float xcenter \
89 : = boxinputptr[1] / x_scale * boxprior[3][index] + boxprior[1][index]; \
90 : float h = (float) expf (boxinputptr[2] / h_scale) * boxprior[2][index]; \
91 : float w = (float) expf (boxinputptr[3] / w_scale) * boxprior[3][index]; \
92 : float ymin = ycenter - h / 2.f; \
93 : float xmin = xcenter - w / 2.f; \
94 : int x = xmin * i_width; \
95 : int y = ymin * i_height; \
96 : int width = w * i_width; \
97 : int height = h * i_height; \
98 : if (highscore < score) { \
99 : result->class_id = c; \
100 : result->x = MAX (0, x); \
101 : result->y = MAX (0, y); \
102 : result->width = width; \
103 : result->height = height; \
104 : result->prob = score; \
105 : result->valid = TRUE; \
106 : } \
107 : } \
108 : } \
109 : } while (0);
110 :
111 : /**
112 : * @brief C++-Template-like box location calculation for box-priors for Mobilenet SSD Model
113 : * @param[in] type The tensor type of inputptr
114 : * @param[in] typename nnstreamer enum corresponding to the type
115 : * @param[in] boxprior The box prior data from the box file of MOBILENET_SSD.
116 : * @param[in] boxinput Input Tensor Data (Boxes)
117 : * @param[in] detinput Input Tensor Data (Detection). Null if not available. (numtensor ==1)
118 : * @param[in] config Tensor configs of the input tensors
119 : * @param[out] results The object returned. (GArray with detectedObject)
120 : */
121 : #define _get_objects_mobilenet_ssd(_type, typename, boxprior, boxinput, \
122 : detinput, config, results, i_width, i_height, max_detection) \
123 : case typename: \
124 : { \
125 : int d; \
126 : _type *boxinput_ = (_type *) boxinput; \
127 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, 0); \
128 : size_t boxbpi = info->dimension[0]; \
129 : _type *detinput_ = (_type *) detinput; \
130 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, 1); \
131 : size_t detbpi = info->dimension[0]; \
132 : int num = (DETECTION_MAX > max_detection) ? max_detection : DETECTION_MAX; \
133 : detectedObject object = { \
134 : .valid = FALSE, \
135 : .class_id = 0, \
136 : .x = 0, \
137 : .y = 0, \
138 : .width = 0, \
139 : .height = 0, \
140 : .angle = 0, \
141 : .prob = .0, \
142 : .tracking_id = 0, \
143 : }; \
144 : for (d = 0; d < num; d++) { \
145 : _get_object_i_mobilenet_ssd (d, detbpi, boxprior, (boxinput_ + (d * boxbpi)), \
146 : (detinput_ + (d * detbpi)), (&object), i_width, i_height); \
147 : if (object.valid == TRUE) { \
148 : g_array_append_val (results, object); \
149 : } \
150 : } \
151 : } \
152 : break
153 :
154 : /** @brief Macro to simplify calling _get_objects_mobilenet_ssd */
155 : #define _get_objects_mobilenet_ssd_(type, typename) \
156 : _get_objects_mobilenet_ssd (type, typename, box_priors, (boxes->data), \
157 : (detections->data), config, results, i_width, i_height, max_detection)
158 :
159 : /** @brief Mathematic inverse of sigmoid function, aka logit */
160 : static float
161 19 : logit (float x)
162 : {
163 19 : if (x <= 0.0f)
164 0 : return -INFINITY;
165 :
166 19 : if (x >= 1.0f)
167 0 : return INFINITY;
168 :
169 19 : return log (x / (1.0 - x));
170 : }
171 :
172 : static BoxProperties *mobilenet = nullptr;
173 :
174 : #ifdef __cplusplus
175 : extern "C" {
176 : #endif /* __cplusplus */
177 : void init_properties_mobilenetssd (void) __attribute__ ((constructor));
178 : void fini_properties_mobilenetssd (void) __attribute__ ((destructor));
179 : #ifdef __cplusplus
180 : }
181 : #endif /* __cplusplus */
182 :
183 : /** @brief Constructor of MobilenetSSD */
184 15 : MobilenetSSD::MobilenetSSD ()
185 : {
186 15 : params[THRESHOLD_IDX] = DETECTION_THRESHOLD_DEFAULT;
187 15 : params[Y_SCALE_IDX] = Y_SCALE_DEFAULT;
188 15 : params[X_SCALE_IDX] = X_SCALE_DEFAULT;
189 15 : params[H_SCALE_IDX] = H_SCALE_DEFAULT;
190 15 : params[W_SCALE_IDX] = W_SCALE_DEFAULT;
191 15 : params[IOU_THRESHOLD_IDX] = THRESHOLD_IOU_DEFAULT;
192 15 : sigmoid_threshold = logit (DETECTION_THRESHOLD_DEFAULT);
193 :
194 15 : max_detection = 0;
195 15 : total_labels = 0;
196 15 : box_prior_path = nullptr;
197 15 : name = g_strdup_printf ("mobilenet-ssd");
198 15 : }
199 :
200 : /** @brief Destructor of MobilenetSSD */
201 30 : MobilenetSSD::~MobilenetSSD ()
202 : {
203 15 : g_free (name);
204 30 : }
205 :
206 : /**
207 : * @brief Load box-prior data from a file
208 : * @param[in/out] bdata The internal data.
209 : * @return TRUE if loaded and configured. FALSE if failed to do so.
210 : */
211 : int
212 4 : MobilenetSSD::mobilenet_ssd_loadBoxPrior ()
213 : {
214 4 : gboolean failed = FALSE;
215 4 : GError *err = NULL;
216 : gchar **priors;
217 4 : gchar *line = NULL;
218 4 : gchar *contents = NULL;
219 : guint row;
220 4 : gint prev_reg = -1;
221 :
222 : /* Read file contents */
223 4 : if (!g_file_get_contents (box_prior_path, &contents, NULL, &err)) {
224 0 : GST_ERROR ("Decoder/Bound-Box/SSD's box prior file %s cannot be read: %s",
225 : box_prior_path, err->message);
226 0 : g_clear_error (&err);
227 0 : return FALSE;
228 : }
229 :
230 4 : priors = g_strsplit (contents, "\n", -1);
231 : /* If given prior file is inappropriate, report back to tensor-decoder */
232 4 : if (g_strv_length (priors) < BOX_SIZE) {
233 0 : ml_loge ("The given prior file, %s, should have at least %d lines.\n",
234 : box_prior_path, BOX_SIZE);
235 0 : failed = TRUE;
236 0 : goto error;
237 : }
238 :
239 20 : for (row = 0; row < BOX_SIZE; row++) {
240 16 : gint column = 0, registered = 0;
241 :
242 16 : line = priors[row];
243 16 : if (line) {
244 16 : gchar **list = g_strsplit_set (line, " \t,", -1);
245 : gchar *word;
246 :
247 30832 : while ((word = list[column]) != NULL) {
248 30816 : column++;
249 :
250 30816 : if (word && *word) {
251 30672 : if (registered > DETECTION_MAX) {
252 0 : GST_WARNING ("Decoder/Bound-Box/SSD's box prior data file has too many priors. %d >= %d",
253 : registered, DETECTION_MAX);
254 0 : break;
255 : }
256 30672 : box_priors[row][registered] = (gfloat) g_ascii_strtod (word, NULL);
257 30672 : registered++;
258 : }
259 : }
260 :
261 16 : g_strfreev (list);
262 : }
263 :
264 16 : if (prev_reg != -1 && prev_reg != registered) {
265 0 : GST_ERROR ("Decoder/Bound-Box/SSD's box prior data file is not consistent.");
266 0 : failed = TRUE;
267 0 : break;
268 : }
269 16 : prev_reg = registered;
270 : }
271 :
272 4 : error:
273 4 : g_strfreev (priors);
274 4 : g_free (contents);
275 4 : return !failed;
276 : }
277 :
278 : /** @brief Set internal option of MobilenetSSD
279 : * @param[in] param The option string.
280 : */
281 : int
282 4 : MobilenetSSD::setOptionInternal (const char *param)
283 : {
284 : gchar **options;
285 : int noptions, idx;
286 4 : int ret = 1;
287 :
288 4 : options = g_strsplit (param, ":", -1);
289 4 : noptions = g_strv_length (options);
290 :
291 4 : if (noptions > (PARAMS_MAX + 1))
292 0 : noptions = PARAMS_MAX + 1;
293 :
294 4 : if (box_prior_path) {
295 0 : g_free (box_prior_path);
296 0 : box_prior_path = nullptr;
297 : }
298 :
299 4 : box_prior_path = g_strdup (options[0]);
300 :
301 4 : if (NULL != box_prior_path) {
302 4 : ret = mobilenet_ssd_loadBoxPrior ();
303 4 : if (ret == 0)
304 0 : goto exit_mobilenet_ssd;
305 : }
306 :
307 4 : for (idx = 1; idx < noptions; idx++) {
308 0 : if (strlen (options[idx]) == 0)
309 0 : continue;
310 0 : params[idx - 1] = strtod (options[idx], NULL);
311 : }
312 :
313 4 : sigmoid_threshold = logit (params[THRESHOLD_IDX]);
314 :
315 4 : return TRUE;
316 :
317 0 : exit_mobilenet_ssd:
318 0 : g_strfreev (options);
319 0 : return ret;
320 : }
321 :
322 : /** @brief Check compatibility of given tensors config
323 : * @param[in] config The tensors config to check compatibility
324 : */
325 : int
326 36 : MobilenetSSD::checkCompatible (const GstTensorsConfig *config)
327 : {
328 : const uint32_t *dim1, *dim2;
329 : int i;
330 : guint max_label;
331 36 : GstTensorInfo *info = nullptr;
332 :
333 36 : if (!check_tensors (config, MAX_TENSORS))
334 20 : return FALSE;
335 :
336 : /* Check if the first tensor is compatible */
337 16 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, 0);
338 16 : dim1 = info->dimension;
339 :
340 16 : g_return_val_if_fail (dim1[0] == BOX_SIZE, FALSE);
341 16 : g_return_val_if_fail (dim1[1] == 1, FALSE);
342 16 : g_return_val_if_fail (dim1[2] > 0, FALSE);
343 :
344 : /** @todo unused dimension value should be 0 */
345 224 : for (i = 3; i < NNS_TENSOR_RANK_LIMIT; i++)
346 208 : g_return_val_if_fail (dim1[i] == 0 || dim1[i] == 1, FALSE);
347 :
348 : /* Check if the second tensor is compatible */
349 16 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, 1);
350 16 : dim2 = info->dimension;
351 :
352 16 : max_label = dim2[0];
353 16 : g_return_val_if_fail (max_label <= total_labels, FALSE);
354 16 : if (max_label < total_labels)
355 0 : GST_WARNING ("The given tensor (2nd) has max_label (first dimension: %u) smaller than the number of labels in labels file (%u).",
356 : max_label, total_labels);
357 16 : g_return_val_if_fail (dim1[2] == dim2[1], FALSE);
358 240 : for (i = 2; i < NNS_TENSOR_RANK_LIMIT; i++)
359 224 : g_return_val_if_fail (dim2[i] == 0 || dim2[i] == 1, FALSE);
360 :
361 : /* Check consistency with max_detection */
362 16 : if (max_detection != 0 && max_detection != dim1[2]) {
363 0 : GST_ERROR ("Failed to check consistency with max_detection");
364 0 : return FALSE;
365 : } else {
366 16 : max_detection = dim1[2];
367 : }
368 :
369 16 : if (max_detection > DETECTION_MAX) {
370 0 : GST_ERROR ("Incoming tensor has too large detection-max : %u", max_detection);
371 0 : return FALSE;
372 : }
373 :
374 16 : return TRUE;
375 : }
376 :
377 : /**
378 : * @brief Decode input memory to out buffer
379 : * @param[in] config The structure of input tensor info.
380 : * @param[in] input The array of input tensor data. The maximum array size of input data is NNS_TENSOR_SIZE_LIMIT.
381 : */
382 : GArray *
383 8 : MobilenetSSD::decode (const GstTensorsConfig *config, const GstTensorMemory *input)
384 : {
385 8 : const GstTensorMemory *boxes, *detections = NULL;
386 : GArray *results;
387 8 : const guint num_tensors = config->info.num_tensors;
388 8 : GstTensorInfo *info = nullptr;
389 :
390 : /**
391 : * @todo 100 is a heuristic number of objects in a picture frame
392 : * We may have better "heuristics" than this.
393 : * For the sake of performance, don't make it too small.
394 : */
395 :
396 : /* Already checked with getOutCaps. Thus, this is an internal bug */
397 8 : g_assert (num_tensors >= MAX_TENSORS);
398 8 : results = g_array_sized_new (FALSE, TRUE, sizeof (detectedObject), 100);
399 :
400 8 : boxes = &input[0];
401 8 : detections = &input[1];
402 8 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, 0);
403 :
404 8 : switch (info->type) {
405 0 : _get_objects_mobilenet_ssd_ (uint8_t, _NNS_UINT8);
406 0 : _get_objects_mobilenet_ssd_ (int8_t, _NNS_INT8);
407 0 : _get_objects_mobilenet_ssd_ (uint16_t, _NNS_UINT16);
408 0 : _get_objects_mobilenet_ssd_ (int16_t, _NNS_INT16);
409 0 : _get_objects_mobilenet_ssd_ (uint32_t, _NNS_UINT32);
410 0 : _get_objects_mobilenet_ssd_ (int32_t, _NNS_INT32);
411 0 : _get_objects_mobilenet_ssd_ (uint64_t, _NNS_UINT64);
412 0 : _get_objects_mobilenet_ssd_ (int64_t, _NNS_INT64);
413 1395584 : _get_objects_mobilenet_ssd_ (float, _NNS_FLOAT32);
414 0 : _get_objects_mobilenet_ssd_ (double, _NNS_FLOAT64);
415 0 : default:
416 0 : g_assert (0);
417 : }
418 8 : nms (results, params[IOU_THRESHOLD_IDX], MOBILENET_SSD_BOUNDING_BOX);
419 8 : return results;
420 : }
421 :
422 : /** @brief Initialize this object for tensor decoder bounding box */
423 : void
424 15 : init_properties_mobilenetssd ()
425 : {
426 15 : mobilenet = new MobilenetSSD ();
427 15 : BoundingBox::addProperties (mobilenet);
428 15 : }
429 :
430 : /** @brief Destruct this object for tensor decoder bounding box */
431 : void
432 15 : fini_properties_mobilenetssd ()
433 : {
434 15 : delete mobilenet;
435 15 : }
|
