Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * Copyright (C) 2021 Samsung Electronics Co., Ltd.
4 : *
5 : * @file gsttensor_sparsedec.c
6 : * @date 27 Jul 2021
7 : * @brief GStreamer element to decode sparse tensors into dense tensors
8 : * @see https://github.com/nnstreamer/nnstreamer
9 : * @author Yongjoo Ahn <yongjoo1.ahn@samsung.com>
10 : * @bug No known bugs except for NYI items
11 : */
12 :
13 : /**
14 : * SECTION:element-tensor_sparse_dec
15 : *
16 : * tensor_sparse_dec is a GStreamer element to decode incoming sparse tensor into static (dense) format.
17 : *
18 : * The input is always in the format of other/tensors,format=sparse.
19 : * The output is always in the format of ohter/tensors,format=static.
20 : *
21 : * Please see also tensor_sparse_enc.
22 : *
23 : * <refsect2>
24 : * <title>Example launch line</title>
25 : * |[
26 : * gst-launch-1.0 ... ! other/tensors,format=static ! \
27 : * tensor_sparse_enc ! other/tensors,format=sparse ! \
28 : * tensor_sparse_dec ! tensor_sink
29 : * ]|
30 : * </refsect2>
31 : */
32 :
33 : #ifdef HAVE_CONFIG_H
34 : #include <config.h>
35 : #endif
36 :
37 : #include <string.h>
38 : #include <nnstreamer_util.h>
39 : #include "gsttensor_sparsedec.h"
40 :
41 : /**
42 : * @brief Macro for debug mode.
43 : */
44 : #ifndef DBG
45 : #define DBG (!self->silent)
46 : #endif
47 :
48 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_sparse_dec_debug);
49 : #define GST_CAT_DEFAULT gst_tensor_sparse_dec_debug
50 :
51 : /**
52 : * @brief tensor_sparse_dec properties
53 : */
54 : enum
55 : {
56 : PROP_0,
57 : PROP_SILENT
58 : };
59 :
60 : /**
61 : * @brief Flag to print minimized log.
62 : */
63 : #define DEFAULT_SILENT TRUE
64 :
65 : /**
66 : * @brief Template for sink pad.
67 : */
68 : static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
69 : GST_PAD_SINK,
70 : GST_PAD_ALWAYS,
71 : GST_STATIC_CAPS (GST_TENSORS_SPARSE_CAP_DEFAULT));
72 :
73 : /**
74 : * @brief Template for src pad.
75 : */
76 : static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
77 : GST_PAD_SRC,
78 : GST_PAD_ALWAYS,
79 : GST_STATIC_CAPS (GST_TENSORS_CAP_DEFAULT));
80 :
81 : #define gst_tensor_sparse_dec_parent_class parent_class
82 937 : G_DEFINE_TYPE (GstTensorSparseDec, gst_tensor_sparse_dec, GST_TYPE_ELEMENT);
83 :
84 : static void gst_tensor_sparse_dec_finalize (GObject * object);
85 : static void gst_tensor_sparse_dec_set_property (GObject * object, guint prop_id,
86 : const GValue * value, GParamSpec * pspec);
87 : static void gst_tensor_sparse_dec_get_property (GObject * object, guint prop_id,
88 : GValue * value, GParamSpec * pspec);
89 : static GstFlowReturn
90 : gst_tensor_sparse_dec_chain (GstPad * pad, GstObject * parent, GstBuffer * buf);
91 : static gboolean
92 : gst_tensor_sparse_dec_sink_event (GstPad * pad, GstObject * parent,
93 : GstEvent * event);
94 : static gboolean gst_tensor_sparse_dec_sink_query (GstPad * pad,
95 : GstObject * parent, GstQuery * query);
96 :
97 : /**
98 : * @brief Initialize the tensor_sparse's class.
99 : */
100 : static void
101 6 : gst_tensor_sparse_dec_class_init (GstTensorSparseDecClass * klass)
102 : {
103 : GObjectClass *object_class;
104 : GstElementClass *element_class;
105 :
106 6 : GST_DEBUG_CATEGORY_INIT (gst_tensor_sparse_dec_debug, "tensor_sparse_dec", 0,
107 : "Element to decode sparse tensors");
108 :
109 6 : object_class = (GObjectClass *) klass;
110 6 : element_class = (GstElementClass *) klass;
111 :
112 6 : object_class->set_property = gst_tensor_sparse_dec_set_property;
113 6 : object_class->get_property = gst_tensor_sparse_dec_get_property;
114 6 : object_class->finalize = gst_tensor_sparse_dec_finalize;
115 :
116 : /**
117 : * GstTensorSparseDec::silent:
118 : *
119 : * The flag to enable/disable debugging messages.
120 : */
121 6 : g_object_class_install_property (object_class, PROP_SILENT,
122 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
123 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
124 :
125 6 : gst_element_class_add_pad_template (element_class,
126 : gst_static_pad_template_get (&src_template));
127 6 : gst_element_class_add_pad_template (element_class,
128 : gst_static_pad_template_get (&sink_template));
129 :
130 6 : gst_element_class_set_static_metadata (element_class,
131 : "TensorSparseDec",
132 : "Filter/Tensor",
133 : "Element to decode dense tensors into sparse tensors",
134 : "Samsung Electronics Co., Ltd.");
135 6 : }
136 :
137 : /**
138 : * @brief Initialize tensor_sparse_dec element.
139 : */
140 : static void
141 6 : gst_tensor_sparse_dec_init (GstTensorSparseDec * self)
142 : {
143 : /* setup sink pad */
144 6 : self->sinkpad = gst_pad_new_from_static_template (&sink_template, "sink");
145 6 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);
146 :
147 : /* setup src pad */
148 6 : self->srcpad = gst_pad_new_from_static_template (&src_template, "src");
149 6 : gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
150 :
151 : /* setup chain function */
152 6 : gst_pad_set_chain_function (self->sinkpad,
153 : GST_DEBUG_FUNCPTR (gst_tensor_sparse_dec_chain));
154 :
155 : /* setup event function */
156 6 : gst_pad_set_event_function (self->sinkpad,
157 : GST_DEBUG_FUNCPTR (gst_tensor_sparse_dec_sink_event));
158 :
159 6 : gst_pad_set_query_function (self->sinkpad,
160 : GST_DEBUG_FUNCPTR (gst_tensor_sparse_dec_sink_query));
161 :
162 : /* init properties */
163 6 : self->silent = DEFAULT_SILENT;
164 6 : gst_tensors_config_init (&self->in_config);
165 6 : gst_tensors_config_init (&self->out_config);
166 6 : }
167 :
168 : /**
169 : * @brief Function to finalize instance.
170 : */
171 : static void
172 6 : gst_tensor_sparse_dec_finalize (GObject * object)
173 : {
174 : GstTensorSparseDec *self;
175 6 : self = GST_TENSOR_SPARSE_DEC (object);
176 :
177 6 : gst_tensors_config_free (&self->in_config);
178 6 : gst_tensors_config_free (&self->out_config);
179 :
180 6 : G_OBJECT_CLASS (parent_class)->finalize (object);
181 6 : }
182 :
183 : /**
184 : * @brief Setter for tensor_sparse_dec properties.
185 : */
186 : static void
187 1 : gst_tensor_sparse_dec_set_property (GObject * object, guint prop_id,
188 : const GValue * value, GParamSpec * pspec)
189 : {
190 : GstTensorSparseDec *self;
191 :
192 1 : self = GST_TENSOR_SPARSE_DEC (object);
193 :
194 1 : switch (prop_id) {
195 1 : case PROP_SILENT:
196 1 : self->silent = g_value_get_boolean (value);
197 1 : break;
198 0 : default:
199 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
200 0 : break;
201 : }
202 1 : }
203 :
204 : /**
205 : * @brief Getter for tensor_sparse_dec properties.
206 : */
207 : static void
208 2 : gst_tensor_sparse_dec_get_property (GObject * object, guint prop_id,
209 : GValue * value, GParamSpec * pspec)
210 : {
211 : GstTensorSparseDec *self;
212 :
213 2 : self = GST_TENSOR_SPARSE_DEC (object);
214 :
215 2 : switch (prop_id) {
216 2 : case PROP_SILENT:
217 2 : g_value_set_boolean (value, self->silent);
218 2 : break;
219 0 : default:
220 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
221 0 : break;
222 : }
223 2 : }
224 :
225 : /**
226 : * @brief Get pad caps for caps negotiation.
227 : */
228 : static GstCaps *
229 16 : gst_tensor_sparse_dec_query_caps (GstTensorSparseDec * self, GstPad * pad,
230 : GstCaps * filter)
231 : {
232 : GstCaps *caps;
233 :
234 16 : caps = gst_pad_get_current_caps (pad);
235 16 : if (!caps) {
236 : /** pad don't have current caps. use the template caps */
237 16 : caps = gst_pad_get_pad_template_caps (pad);
238 : }
239 :
240 16 : silent_debug_caps (self, caps, "caps");
241 16 : silent_debug_caps (self, filter, "filter");
242 :
243 16 : if (filter) {
244 : GstCaps *intersection;
245 : intersection =
246 5 : gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
247 :
248 5 : gst_caps_unref (caps);
249 5 : caps = intersection;
250 : }
251 :
252 16 : silent_debug_caps (self, caps, "result");
253 16 : return caps;
254 : }
255 :
256 : /**
257 : * @brief This function handles sink pad query.
258 : */
259 : static gboolean
260 24 : gst_tensor_sparse_dec_sink_query (GstPad * pad, GstObject * parent,
261 : GstQuery * query)
262 : {
263 : GstTensorSparseDec *self;
264 24 : self = GST_TENSOR_SPARSE_DEC (parent);
265 :
266 24 : GST_DEBUG_OBJECT (self, "Received %s query: %" GST_PTR_FORMAT,
267 : GST_QUERY_TYPE_NAME (query), query);
268 :
269 24 : switch (GST_QUERY_TYPE (query)) {
270 16 : case GST_QUERY_CAPS:
271 : {
272 : GstCaps *caps;
273 : GstCaps *filter;
274 :
275 16 : gst_query_parse_caps (query, &filter);
276 16 : caps = gst_tensor_sparse_dec_query_caps (self, pad, filter);
277 16 : silent_debug_caps (self, filter, "filter");
278 16 : silent_debug_caps (self, caps, "caps");
279 16 : gst_query_set_caps_result (query, caps);
280 16 : gst_caps_unref (caps);
281 16 : return TRUE;
282 : }
283 5 : case GST_QUERY_ACCEPT_CAPS:
284 : {
285 : GstCaps *caps;
286 : GstCaps *template_caps;
287 5 : gboolean res = FALSE;
288 :
289 5 : gst_query_parse_accept_caps (query, &caps);
290 5 : silent_debug_caps (self, caps, "caps");
291 :
292 5 : if (gst_caps_is_fixed (caps)) {
293 5 : template_caps = gst_pad_get_pad_template_caps (pad);
294 :
295 5 : res = gst_caps_can_intersect (template_caps, caps);
296 5 : gst_caps_unref (template_caps);
297 : }
298 :
299 5 : gst_query_set_accept_caps_result (query, res);
300 5 : return TRUE;
301 : }
302 3 : default:
303 3 : break;
304 : }
305 :
306 3 : return gst_pad_query_default (pad, parent, query);
307 : }
308 :
309 : /**
310 : * @brief This function handles sink pad event.
311 : */
312 : static gboolean
313 20 : gst_tensor_sparse_dec_sink_event (GstPad * pad, GstObject * parent,
314 : GstEvent * event)
315 : {
316 : GstTensorSparseDec *self;
317 20 : self = GST_TENSOR_SPARSE_DEC (parent);
318 20 : g_return_val_if_fail (event != NULL, FALSE);
319 :
320 20 : switch (GST_EVENT_TYPE (event)) {
321 5 : case GST_EVENT_CAPS:
322 : {
323 : GstCaps *caps, *out_caps;
324 : GstStructure *structure;
325 :
326 5 : gst_event_parse_caps (event, &caps);
327 5 : silent_debug_caps (self, caps, "caps");
328 :
329 : /* set in_config */
330 5 : structure = gst_caps_get_structure (caps, 0);
331 5 : gst_tensors_config_from_structure (&self->in_config, structure);
332 :
333 : /* set out_config as srcpad's peer */
334 5 : gst_tensors_config_from_peer (self->srcpad, &self->out_config, NULL);
335 5 : self->out_config.rate_n = self->in_config.rate_n;
336 5 : self->out_config.rate_d = self->in_config.rate_d;
337 :
338 10 : out_caps = gst_tensor_pad_caps_from_config (self->srcpad,
339 5 : &self->out_config);
340 :
341 5 : silent_debug_caps (self, out_caps, "out_caps");
342 5 : gst_pad_set_caps (self->srcpad, out_caps);
343 5 : gst_caps_unref (out_caps);
344 :
345 5 : gst_event_unref (event);
346 5 : return TRUE;
347 : }
348 15 : default:
349 15 : break;
350 : }
351 :
352 15 : return gst_pad_event_default (pad, parent, event);
353 : }
354 :
355 : /**
356 : * @brief Internal function to transform the input buffer.
357 : */
358 : static GstFlowReturn
359 14 : gst_tensor_sparse_dec_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
360 : {
361 14 : GstFlowReturn ret = GST_FLOW_ERROR;
362 14 : GstTensorSparseDec *self = GST_TENSOR_SPARSE_DEC (parent);
363 : GstTensorMetaInfo meta;
364 : GstMemory *in_mem, *out_mem;
365 : GstBuffer *outbuf;
366 : GstTensorsInfo info;
367 : GstTensorInfo *_info;
368 : guint i;
369 :
370 : UNUSED (pad);
371 :
372 14 : buf = gst_tensor_buffer_from_config (buf, &self->in_config);
373 14 : outbuf = gst_buffer_new ();
374 :
375 14 : gst_tensors_info_init (&info);
376 14 : info.num_tensors = gst_tensor_buffer_get_count (buf);
377 :
378 39 : for (i = 0; i < info.num_tensors; ++i) {
379 25 : in_mem = gst_tensor_buffer_get_nth_memory (buf, i);
380 25 : out_mem = gst_tensor_sparse_to_dense (&meta, in_mem);
381 25 : gst_memory_unref (in_mem);
382 :
383 25 : if (!out_mem) {
384 0 : nns_loge ("failed to convert to dense tensor");
385 0 : goto done;
386 : }
387 :
388 25 : _info = gst_tensors_info_get_nth_info (&info, i);
389 25 : gst_tensor_meta_info_convert (&meta, _info);
390 25 : gst_tensor_buffer_append_memory (outbuf, out_mem, _info);
391 : }
392 :
393 : /* check the decoded tensor with negotiated config when it's valid */
394 14 : if (gst_tensors_config_validate (&self->out_config)) {
395 12 : if (!gst_tensors_info_is_equal (&self->out_config.info, &info)) {
396 : /* if it's not compatible with downstream, do not send the buffer */
397 : /** @todo consider more error handling */
398 0 : gst_buffer_unref (outbuf);
399 0 : ret = GST_FLOW_OK;
400 0 : goto done;
401 : }
402 : }
403 :
404 14 : ret = gst_pad_push (self->srcpad, outbuf);
405 :
406 14 : done:
407 14 : gst_buffer_unref (buf);
408 14 : if (ret != GST_FLOW_OK)
409 0 : gst_buffer_unref (outbuf);
410 :
411 14 : return ret;
412 : }
|
