Line data Source code
1 : /**
2 : * GStreamer / NNStreamer tensor_decoder subplugin, "image labeling"
3 : * Copyright (C) 2018 Jinhyuck Park <jinhyuck83.park@samsung.com>
4 : * Copyright (C) 2018 MyungJoo Ham <myungjoo.ham@samsung.com>
5 : *
6 : * This library is free software; you can redistribute it and/or
7 : * modify it under the terms of the GNU Library General Public
8 : * License as published by the Free Software Foundation;
9 : * version 2.1 of the License.
10 : *
11 : * This library is distributed in the hope that it will be useful,
12 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 : * Library General Public License for more details.
15 : *
16 : */
17 : /**
18 : * @file tensordec-imagelabel.c
19 : * @date 14 Nov 2018
20 : * @brief NNStreamer tensor-decoder subplugin, "image labeling",
21 : * which converts image label tensors to text stream.
22 : *
23 : * @see https://github.com/nnstreamer/nnstreamer
24 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
25 : * @bug No known bugs except for NYI items
26 : *
27 : */
28 :
29 : #include <stdio.h>
30 : #include <stdlib.h>
31 : #include <string.h>
32 : #include <glib.h>
33 : #include <gst/gstinfo.h>
34 : #include <nnstreamer_plugin_api_decoder.h>
35 : #include <nnstreamer_plugin_api.h>
36 : #include <nnstreamer_log.h>
37 : #include <nnstreamer_util.h>
38 : #include "tensordecutil.h"
39 :
40 : void init_il (void) __attribute__ ((constructor));
41 : void fini_il (void) __attribute__ ((destructor));
42 :
43 : #define DECODER_IL_TEXT_CAPS_STR \
44 : "text/x-raw, format = (string) utf8"
45 :
46 : /** @brief Internal data structure for image labeling */
47 : typedef struct
48 : {
49 : imglabel_t labels;
50 : char *label_path;
51 : } ImageLabelData;
52 :
53 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
54 : static int
55 16 : il_init (void **pdata)
56 : {
57 : /** @todo check if we need to ensure plugin_data is not yet allocated */
58 16 : *pdata = g_new0 (ImageLabelData, 1);
59 16 : if (*pdata == NULL) {
60 0 : GST_ERROR ("Failed to allocate memory for decoder subplugin.");
61 0 : return FALSE;
62 : }
63 :
64 16 : return TRUE;
65 : }
66 :
67 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
68 : static void
69 16 : il_exit (void **pdata)
70 : {
71 16 : ImageLabelData *data = *pdata;
72 :
73 16 : _free_labels (&data->labels);
74 :
75 16 : if (data->label_path)
76 16 : g_free (data->label_path);
77 :
78 16 : g_free (*pdata);
79 16 : *pdata = NULL;
80 16 : }
81 :
82 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
83 : static int
84 16 : il_setOption (void **pdata, int opNum, const char *param)
85 : {
86 16 : ImageLabelData *data = *pdata;
87 :
88 : /* opNum 1 = label file path */
89 16 : if (opNum == 0) {
90 16 : if (NULL != data->label_path)
91 0 : g_free (data->label_path);
92 16 : data->label_path = g_strdup (param);
93 :
94 16 : if (NULL != data->label_path)
95 16 : loadImageLabels (data->label_path, &data->labels);
96 :
97 16 : if (data->labels.total_labels > 0)
98 16 : return TRUE;
99 : else
100 0 : return FALSE;
101 : }
102 :
103 0 : GST_INFO ("Property mode-option-%d is ignored", opNum + 1);
104 0 : return TRUE;
105 : }
106 :
107 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
108 : static GstCaps *
109 108 : il_getOutCaps (void **pdata, const GstTensorsConfig * config)
110 : {
111 : const uint32_t *dim;
112 : GstCaps *caps;
113 : int i;
114 : UNUSED (pdata);
115 :
116 108 : g_return_val_if_fail (config != NULL, NULL);
117 108 : g_return_val_if_fail (config->info.num_tensors >= 1, NULL);
118 :
119 : /* Even if it's multi-tensor, we use the first tensor only in image labeling */
120 90 : dim = config->info.info[0].dimension;
121 : /* This allows N:1 only! */
122 90 : g_return_val_if_fail (dim[0] > 0 && dim[1] == 1, NULL);
123 1170 : for (i = 2; i < NNS_TENSOR_RANK_LIMIT; i++)
124 1092 : g_return_val_if_fail (dim[i] == 0, NULL);
125 :
126 78 : caps = gst_caps_from_string (DECODER_IL_TEXT_CAPS_STR);
127 78 : setFramerateFromConfig (caps, config);
128 78 : return caps;
129 : }
130 :
131 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
132 : static size_t
133 11 : il_getTransformSize (void **pdata, const GstTensorsConfig * config,
134 : GstCaps * caps, size_t size, GstCaps * othercaps, GstPadDirection direction)
135 : {
136 : UNUSED (pdata);
137 : UNUSED (config);
138 : UNUSED (caps);
139 : UNUSED (size);
140 : UNUSED (othercaps);
141 : UNUSED (direction);
142 :
143 11 : return 0;
144 : /** @todo Use max_word_length if that's appropriate */
145 : }
146 :
147 : /** @brief Search for max. Macro for tensor_element union */
148 : #define search_max(type, i, max_index, max_val, bpe, data, num_data) \
149 : do {\
150 : unsigned int i;\
151 : type *cursor = (type *) (data);\
152 : max_val = cursor[0];\
153 : max_index = 0;\
154 : for (i = 1; i < (num_data); i++) {\
155 : if (cursor[i] > (max_val)) {\
156 : max_val = cursor[i];\
157 : max_index = i;\
158 : }\
159 : }\
160 : } while (0);
161 :
162 : /** @brief Shorter case statement for search_max */
163 : #define search_max_case(type, typename) \
164 : case typename:\
165 : search_max(type, i, max_index, max_val._##type, bpe, input_data, num_data);\
166 : break;
167 :
168 :
169 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
170 : static GstFlowReturn
171 11 : il_decode (void **pdata, const GstTensorsConfig * config,
172 : const GstTensorMemory * input, GstBuffer * outbuf)
173 : {
174 11 : ImageLabelData *data = *pdata;
175 : GstMapInfo out_info;
176 : GstMemory *out_mem;
177 :
178 11 : gsize bpe = gst_tensor_get_element_size (config->info.info[0].type);
179 : tensor_element max_val;
180 11 : guint max_index = 0;
181 : gsize num_data; /* Size / bpe */
182 : void *input_data;
183 :
184 : gsize size;
185 : char *str;
186 :
187 11 : g_assert (bpe > 0);
188 11 : g_assert (outbuf);
189 :
190 11 : input_data = input->data;
191 11 : num_data = gst_tensor_info_get_size (&config->info.info[0]) / bpe;
192 :
193 11 : switch (config->info.info[0].type) {
194 1001 : search_max_case (int32_t, _NNS_INT32);
195 1001 : search_max_case (uint32_t, _NNS_UINT32);
196 1001 : search_max_case (int16_t, _NNS_INT16);
197 1001 : search_max_case (uint16_t, _NNS_UINT16);
198 1001 : search_max_case (int8_t, _NNS_INT8);
199 1001 : search_max_case (uint8_t, _NNS_UINT8);
200 1001 : search_max_case (double, _NNS_FLOAT64);
201 2001 : search_max_case (float, _NNS_FLOAT32);
202 1001 : search_max_case (int64_t, _NNS_INT64);
203 1001 : search_max_case (uint64_t, _NNS_UINT64);
204 0 : default:
205 11 : return GST_FLOW_NOT_SUPPORTED;
206 : }
207 :
208 11 : g_assert (max_index < data->labels.total_labels);
209 :
210 : /** @todo With option-2, allow to change output format */
211 11 : str = data->labels.labels[max_index];
212 :
213 11 : if (!str || (size = strlen (str)) == 0) {
214 0 : ml_loge ("Invalid labels. Please check the label data.");
215 0 : return GST_FLOW_ERROR;
216 : }
217 :
218 : /* Ensure we have outbuf properly allocated */
219 11 : if (gst_buffer_get_size (outbuf) == 0) {
220 11 : out_mem = gst_allocator_alloc (NULL, size, NULL);
221 : } else {
222 0 : if (gst_buffer_get_size (outbuf) < size) {
223 0 : gst_buffer_set_size (outbuf, size);
224 : }
225 0 : out_mem = gst_buffer_get_all_memory (outbuf);
226 : }
227 :
228 11 : if (!gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) {
229 0 : ml_loge ("Cannot map output memory / tensordec-imagelabel.\n");
230 0 : gst_memory_unref (out_mem);
231 0 : return GST_FLOW_ERROR;
232 : }
233 :
234 11 : memcpy (out_info.data, str, size);
235 :
236 11 : gst_memory_unmap (out_mem, &out_info);
237 :
238 11 : if (gst_buffer_get_size (outbuf) == 0)
239 11 : gst_buffer_append_memory (outbuf, out_mem);
240 : else
241 0 : gst_buffer_replace_all_memory (outbuf, out_mem);
242 :
243 11 : return GST_FLOW_OK;
244 : }
245 :
246 : static gchar decoder_subplugin_image_labeling[] = "image_labeling";
247 :
248 : /** @brief Image Labeling tensordec-plugin GstTensorDecoderDef instance */
249 : static GstTensorDecoderDef imageLabeling = {
250 : .modename = decoder_subplugin_image_labeling,
251 : .init = il_init,
252 : .exit = il_exit,
253 : .setOption = il_setOption,
254 : .getOutCaps = il_getOutCaps,
255 : .getTransformSize = il_getTransformSize,
256 : .decode = il_decode
257 : };
258 :
259 : /** @brief Initialize this object for tensordec-plugin */
260 : void
261 38 : init_il (void)
262 : {
263 38 : nnstreamer_decoder_probe (&imageLabeling);
264 38 : nnstreamer_decoder_set_custom_property_desc (
265 : decoder_subplugin_image_labeling, "option1", "The path to the label file",
266 : NULL);
267 38 : }
268 :
269 : /** @brief Destruct this object for tensordec-plugin */
270 : void
271 38 : fini_il (void)
272 : {
273 38 : nnstreamer_decoder_exit (imageLabeling.modename);
274 38 : }
|
