Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer / NNStreamer tensor-decoder bounding box properties
4 : * Copyright (C) 2024 Yelin Jeong <yelini.jeong@samsung.com>
5 : */
6 : /**
7 : * @file mobilenetssdpp.cc
8 : * @date 13 May 2024
9 : * @brief NNStreamer tensor-decoder bounding box properties
10 : *
11 : * @see https://github.com/nnstreamer/nnstreamer
12 : * @author Yelin Jeong <yelini.jeong@samsung.com>
13 : * @bug No known bugs except for NYI items
14 : *
15 : */
16 :
17 : #include <stdio.h>
18 : #include "../tensordec-boundingbox.h"
19 :
20 : #define BOX_SIZE (4)
21 : #define DETECTION_MAX (100)
22 : #define LOCATIONS_IDX (0)
23 : #define CLASSES_IDX (1)
24 : #define SCORES_IDX (2)
25 : #define NUM_IDX (3)
26 : #define MAX_TENSORS (4U)
27 :
28 : #define LOCATIONS_DEFAULT (3)
29 : #define CLASSES_DEFAULT (1)
30 : #define SCORES_DEFAULT (2)
31 : #define NUM_DEFAULT (0)
32 : #define THRESHOLD_DEFAULT (G_MINFLOAT)
33 :
34 : /**
35 : * @brief Class for MobilenetSSDPP box properties
36 : */
37 : class MobilenetSSDPP : public BoxProperties
38 : {
39 : public:
40 : MobilenetSSDPP ();
41 : ~MobilenetSSDPP ();
42 : int get_mobilenet_ssd_pp_tensor_idx (int idx);
43 :
44 : int setOptionInternal (const char *param);
45 : int checkCompatible (const GstTensorsConfig *config);
46 : GArray *decode (const GstTensorsConfig *config, const GstTensorMemory *input);
47 :
48 : private:
49 : gint tensor_mapping[MAX_TENSORS]; /* Output tensor index mapping */
50 : gfloat threshold; /* Detection threshold */
51 : };
52 :
53 : /**
54 : * @brief C++-Template-like box location calculation for Tensorflow SSD model
55 : * @param[in] type The tensor type of inputptr
56 : * @param[in] typename nnstreamer enum corresponding to the type
57 : * @param[in] numinput Input Tensor Data (The number of detections)
58 : * @param[in] classinput Input Tensor Data (Detected classes)
59 : * @param[in] scoreinput Input Tensor Data (Detection scores)
60 : * @param[in] boxesinput Input Tensor Data (Boxes)
61 : * @param[in] config Tensor configs of the input tensors
62 : * @param[out] results The object returned. (GArray with detectedObject)
63 : */
64 : #define _get_objects_mobilenet_ssd_pp(_type, typename, numinput, classinput, \
65 : scoreinput, boxesinput, config, results, i_width, i_height) \
66 : case typename: \
67 : { \
68 : int d, num; \
69 : size_t boxbpi; \
70 : _type *num_detection_ = (_type *) numinput; \
71 : _type *classes_ = (_type *) classinput; \
72 : _type *scores_ = (_type *) scoreinput; \
73 : _type *boxes_ = (_type *) boxesinput; \
74 : int locations_idx \
75 : = get_mobilenet_ssd_pp_tensor_idx (MOBILENET_SSD_PP_BBOX_IDX_LOCATIONS); \
76 : num = (int) num_detection_[0]; \
77 : results = g_array_sized_new (FALSE, TRUE, sizeof (detectedObject), num); \
78 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, locations_idx); \
79 : boxbpi = info->dimension[0]; \
80 : for (d = 0; d < num; d++) { \
81 : _type x1, x2, y1, y2; \
82 : detectedObject object; \
83 : gfloat score = (gfloat) scores_[d]; \
84 : if (score < threshold) \
85 : continue; \
86 : object.valid = TRUE; \
87 : object.class_id = (int) classes_[d]; \
88 : x1 = MIN (MAX (boxes_[d * boxbpi + 1], 0), 1); \
89 : y1 = MIN (MAX (boxes_[d * boxbpi], 0), 1); \
90 : x2 = MIN (MAX (boxes_[d * boxbpi + 3], 0), 1); \
91 : y2 = MIN (MAX (boxes_[d * boxbpi + 2], 0), 1); \
92 : object.x = (int) (x1 * i_width); \
93 : object.y = (int) (y1 * i_height); \
94 : object.width = (int) ((x2 - x1) * i_width); \
95 : object.height = (int) ((y2 - y1) * i_height); \
96 : object.prob = score; \
97 : g_array_append_val (results, object); \
98 : } \
99 : } \
100 : break
101 :
102 : /** @brief Macro to simplify calling _get_objects_mobilenet_ssd_pp */
103 : #define _get_objects_mobilenet_ssd_pp_(type, typename) \
104 : _get_objects_mobilenet_ssd_pp (type, typename, (mem_num->data), (mem_classes->data), \
105 : (mem_scores->data), (mem_boxes->data), config, results, i_width, i_height)
106 :
107 : static BoxProperties *mobilenetpp = nullptr;
108 :
109 : #ifdef __cplusplus
110 : extern "C" {
111 : #endif /* __cplusplus */
112 : void init_properties_mobilenetssd_pp (void) __attribute__ ((constructor));
113 : void fini_properties_mobilenetssd_pp (void) __attribute__ ((destructor));
114 : #ifdef __cplusplus
115 : }
116 : #endif /* __cplusplus */
117 :
118 : /**
119 : * @brief MOBILENET SSD PostProcess Output tensor feature mapping.
120 : */
121 : typedef enum {
122 : MOBILENET_SSD_PP_BBOX_IDX_LOCATIONS = 0,
123 : MOBILENET_SSD_PP_BBOX_IDX_CLASSES = 1,
124 : MOBILENET_SSD_PP_BBOX_IDX_SCORES = 2,
125 : MOBILENET_SSD_PP_BBOX_IDX_NUM = 3,
126 : MOBILENET_SSD_PP_BBOX_IDX_UNKNOWN
127 : } mobilenet_ssd_pp_bbox_idx_t;
128 :
129 : /** @brief Constructor of MobilenetSSDPP */
130 15 : MobilenetSSDPP::MobilenetSSDPP ()
131 : {
132 15 : max_detection = 0;
133 15 : tensor_mapping[LOCATIONS_IDX] = LOCATIONS_DEFAULT;
134 15 : tensor_mapping[CLASSES_IDX] = CLASSES_DEFAULT;
135 15 : tensor_mapping[SCORES_IDX] = SCORES_DEFAULT;
136 15 : tensor_mapping[NUM_IDX] = NUM_DEFAULT;
137 15 : threshold = THRESHOLD_DEFAULT;
138 15 : name = g_strdup_printf ("mobilenet-ssd-postprocess");
139 15 : }
140 :
141 : /** @brief Destructor of MobilenetSSDPP */
142 30 : MobilenetSSDPP::~MobilenetSSDPP ()
143 : {
144 15 : g_free (name);
145 30 : }
146 :
147 : /** @brief Helper to retrieve tensor index by feature */
148 : int
149 104 : MobilenetSSDPP::get_mobilenet_ssd_pp_tensor_idx (int idx)
150 : {
151 104 : return tensor_mapping[idx];
152 : }
153 :
154 : /** @brief Set internal option of MobilenetSSDPP
155 : * @param[in] param The option string.
156 : */
157 : int
158 0 : MobilenetSSDPP::setOptionInternal (const char *param)
159 : {
160 : int threshold_percent;
161 0 : int ret = sscanf (param, "%i:%i:%i:%i,%i", &tensor_mapping[LOCATIONS_IDX],
162 : &tensor_mapping[CLASSES_IDX], &tensor_mapping[SCORES_IDX],
163 : &tensor_mapping[NUM_IDX], &threshold_percent);
164 :
165 0 : if ((ret == EOF) || (ret < 5)) {
166 0 : GST_ERROR ("Invalid options, must be \"locations idx:classes idx:scores idx:num idx,threshold\"");
167 0 : return FALSE;
168 : }
169 :
170 0 : GST_INFO ("MOBILENET SSD POST PROCESS output tensors mapping: "
171 : "locations idx (%d), classes idx (%d), scores idx (%d), num detections idx (%d)",
172 : tensor_mapping[LOCATIONS_IDX], tensor_mapping[CLASSES_IDX],
173 : tensor_mapping[SCORES_IDX], tensor_mapping[NUM_IDX]);
174 :
175 0 : if ((threshold_percent > 100) || (threshold_percent < 0)) {
176 0 : GST_ERROR ("Invalid MOBILENET SSD POST PROCESS threshold detection (%i), must be in range [0 100]",
177 : threshold_percent);
178 : } else {
179 0 : threshold = threshold_percent / 100.0;
180 : }
181 :
182 0 : GST_INFO ("MOBILENET SSD POST PROCESS object detection threshold: %.2f", threshold);
183 :
184 0 : return TRUE;
185 : }
186 :
187 : /** @brief Check compatibility of given tensors config */
188 : int
189 36 : MobilenetSSDPP::checkCompatible (const GstTensorsConfig *config)
190 : {
191 : const uint32_t *dim1, *dim2, *dim3, *dim4;
192 : int locations_idx, classes_idx, scores_idx, num_idx, i;
193 36 : GstTensorInfo *info = nullptr;
194 :
195 36 : if (!check_tensors (config, MAX_TENSORS))
196 20 : return FALSE;
197 :
198 16 : locations_idx = get_mobilenet_ssd_pp_tensor_idx (LOCATIONS_IDX);
199 16 : classes_idx = get_mobilenet_ssd_pp_tensor_idx (CLASSES_IDX);
200 16 : scores_idx = get_mobilenet_ssd_pp_tensor_idx (SCORES_IDX);
201 16 : num_idx = get_mobilenet_ssd_pp_tensor_idx (NUM_IDX);
202 :
203 : /* Check if the number of detections tensor is compatible */
204 16 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, num_idx);
205 16 : dim1 = info->dimension;
206 16 : g_return_val_if_fail (dim1[0] == 1, FALSE);
207 256 : for (i = 1; i < NNS_TENSOR_RANK_LIMIT; ++i)
208 240 : g_return_val_if_fail (dim1[i] == 0 || dim1[i] == 1, FALSE);
209 :
210 : /* Check if the classes & scores tensors are compatible */
211 16 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, classes_idx);
212 16 : dim2 = info->dimension;
213 16 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, scores_idx);
214 16 : dim3 = info->dimension;
215 16 : g_return_val_if_fail (dim3[0] == dim2[0], FALSE);
216 256 : for (i = 1; i < NNS_TENSOR_RANK_LIMIT; ++i) {
217 240 : g_return_val_if_fail (dim2[i] == 0 || dim2[i] == 1, FALSE);
218 240 : g_return_val_if_fail (dim3[i] == 0 || dim3[i] == 1, FALSE);
219 : }
220 :
221 : /* Check if the bbox locations tensor is compatible */
222 16 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, locations_idx);
223 16 : dim4 = info->dimension;
224 16 : g_return_val_if_fail (BOX_SIZE == dim4[0], FALSE);
225 16 : g_return_val_if_fail (dim2[0] == dim4[1], FALSE);
226 240 : for (i = 2; i < NNS_TENSOR_RANK_LIMIT; ++i)
227 224 : g_return_val_if_fail (dim4[i] == 0 || dim4[i] == 1, FALSE);
228 :
229 : /* Check consistency with max_detection */
230 16 : if (max_detection != 0 && max_detection != dim2[0]) {
231 0 : GST_ERROR ("Failed to check consistency with max_detection");
232 0 : return FALSE;
233 : } else {
234 16 : max_detection = dim2[0];
235 : }
236 :
237 16 : if (max_detection > DETECTION_MAX) {
238 0 : GST_ERROR ("Incoming tensor has too large detection-max : %u", max_detection);
239 0 : return FALSE;
240 : }
241 16 : return TRUE;
242 : }
243 :
244 : /**
245 : * @brief Decode input memory to out buffer
246 : * @param[in] config The structure of input tensor info.
247 : * @param[in] input The array of input tensor data. The maximum array size of input data is NNS_TENSOR_SIZE_LIMIT.
248 : */
249 : GArray *
250 8 : MobilenetSSDPP::decode (const GstTensorsConfig *config, const GstTensorMemory *input)
251 : {
252 :
253 : const GstTensorMemory *mem_num, *mem_classes, *mem_scores, *mem_boxes;
254 : int locations_idx, classes_idx, scores_idx, num_idx;
255 8 : GArray *results = NULL;
256 8 : const guint num_tensors = config->info.num_tensors;
257 8 : GstTensorInfo *info = nullptr;
258 :
259 : /* Already checked with getOutCaps. Thus, this is an internal bug */
260 8 : g_assert (num_tensors >= MAX_TENSORS);
261 :
262 8 : locations_idx = get_mobilenet_ssd_pp_tensor_idx (LOCATIONS_IDX);
263 8 : classes_idx = get_mobilenet_ssd_pp_tensor_idx (CLASSES_IDX);
264 8 : scores_idx = get_mobilenet_ssd_pp_tensor_idx (SCORES_IDX);
265 8 : num_idx = get_mobilenet_ssd_pp_tensor_idx (NUM_IDX);
266 :
267 8 : mem_num = &input[num_idx];
268 8 : mem_classes = &input[classes_idx];
269 8 : mem_scores = &input[scores_idx];
270 8 : mem_boxes = &input[locations_idx];
271 8 : info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, num_idx);
272 8 : switch (info->type) {
273 0 : _get_objects_mobilenet_ssd_pp_ (uint8_t, _NNS_UINT8);
274 0 : _get_objects_mobilenet_ssd_pp_ (int8_t, _NNS_INT8);
275 0 : _get_objects_mobilenet_ssd_pp_ (uint16_t, _NNS_UINT16);
276 0 : _get_objects_mobilenet_ssd_pp_ (int16_t, _NNS_INT16);
277 0 : _get_objects_mobilenet_ssd_pp_ (uint32_t, _NNS_UINT32);
278 0 : _get_objects_mobilenet_ssd_pp_ (int32_t, _NNS_INT32);
279 0 : _get_objects_mobilenet_ssd_pp_ (uint64_t, _NNS_UINT64);
280 0 : _get_objects_mobilenet_ssd_pp_ (int64_t, _NNS_INT64);
281 36 : _get_objects_mobilenet_ssd_pp_ (float, _NNS_FLOAT32);
282 0 : _get_objects_mobilenet_ssd_pp_ (double, _NNS_FLOAT64);
283 0 : default:
284 0 : g_assert (0);
285 : }
286 8 : return results;
287 : }
288 :
289 : /** @brief Initialize this object for tensor decoder bounding box */
290 : void
291 15 : init_properties_mobilenetssd_pp ()
292 : {
293 15 : mobilenetpp = new MobilenetSSDPP ();
294 15 : BoundingBox::addProperties (mobilenetpp);
295 15 : }
296 :
297 : /** @brief Destruct this object for tensor decoder bounding box */
298 : void
299 15 : fini_properties_mobilenetssd_pp ()
300 : {
301 15 : delete mobilenetpp;
302 15 : }
|
