Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * Copyright (C) 2021 Samsung Electronics Co., Ltd.
4 : *
5 : * @file gsttensor_sparseenc.c
6 : * @date 27 Jul 2021
7 : * @brief GStreamer element to encode dense tensors into sparse tensors
8 : * @see https://github.com/nnstreamer/nnstreamer
9 : * @author Yongjoo Ahn <yongjoo1.ahn@samsung.com>
10 : * @bug No known bugs except for NYI items
11 : */
12 :
13 : /**
14 : * SECTION:element-tensor_sparse_enc
15 : *
16 : * tensor_sparse_enc is a GStreamer element to encode incoming tensor into sparse format.
17 : *
18 : * The input is always in the format of other/tensors,format=static.
19 : * The output is always in the format of ohter/tensors,format=sparse.
20 : *
21 : * Please see also tensor_sparse_dec.
22 : *
23 : * <refsect2>
24 : * <title>Example launch line</title>
25 : * |[
26 : * gst-launch-1.0 ... ! other/tensors,format=static ! \
27 : * tensor_sparse_enc ! tensor_sink
28 : * ]|
29 : * </refsect2>
30 : */
31 :
32 : #ifdef HAVE_CONFIG_H
33 : #include <config.h>
34 : #endif
35 :
36 : #include <string.h>
37 : #include <nnstreamer_util.h>
38 : #include "gsttensor_sparseenc.h"
39 :
40 : /**
41 : * @brief Macro for debug mode.
42 : */
43 : #ifndef DBG
44 : #define DBG (!self->silent)
45 : #endif
46 :
47 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_sparse_enc_debug);
48 : #define GST_CAT_DEFAULT gst_tensor_sparse_enc_debug
49 :
50 : /**
51 : * @brief tensor_sparse_enc properties
52 : */
53 : enum
54 : {
55 : PROP_0,
56 : PROP_SILENT
57 : };
58 :
59 : /**
60 : * @brief Flag to print minimized log.
61 : */
62 : #define DEFAULT_SILENT TRUE
63 :
64 : /**
65 : * @brief Template for sink pad.
66 : */
67 : static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
68 : GST_PAD_SINK,
69 : GST_PAD_ALWAYS,
70 : GST_STATIC_CAPS (GST_TENSORS_CAP_DEFAULT));
71 :
72 : /**
73 : * @brief Template for src pad.
74 : */
75 : static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
76 : GST_PAD_SRC,
77 : GST_PAD_ALWAYS,
78 : GST_STATIC_CAPS (GST_TENSORS_SPARSE_CAP_DEFAULT));
79 :
80 : #define gst_tensor_sparse_enc_parent_class parent_class
81 1023 : G_DEFINE_TYPE (GstTensorSparseEnc, gst_tensor_sparse_enc, GST_TYPE_ELEMENT);
82 :
83 : static void gst_tensor_sparse_enc_finalize (GObject * object);
84 : static void gst_tensor_sparse_enc_set_property (GObject * object, guint prop_id,
85 : const GValue * value, GParamSpec * pspec);
86 : static void gst_tensor_sparse_enc_get_property (GObject * object, guint prop_id,
87 : GValue * value, GParamSpec * pspec);
88 : static GstFlowReturn
89 : gst_tensor_sparse_enc_chain (GstPad * pad, GstObject * parent, GstBuffer * buf);
90 : static GstCaps *gst_tensor_sparse_enc_query_caps (GstTensorSparseEnc * self,
91 : GstPad * pad, GstCaps * filter);
92 : static gboolean gst_tensor_sparse_enc_sink_event (GstPad * pad,
93 : GstObject * parent, GstEvent * event);
94 : static gboolean gst_tensor_sparse_enc_sink_query (GstPad * pad,
95 : GstObject * parent, GstQuery * query);
96 :
97 : /**
98 : * @brief Initialize the tensor_sparse's class.
99 : */
100 : static void
101 9 : gst_tensor_sparse_enc_class_init (GstTensorSparseEncClass * klass)
102 : {
103 : GObjectClass *object_class;
104 : GstElementClass *element_class;
105 :
106 9 : GST_DEBUG_CATEGORY_INIT (gst_tensor_sparse_enc_debug, "tensor_sparse_enc", 0,
107 : "Element to encode sparse tensors");
108 :
109 9 : object_class = (GObjectClass *) klass;
110 9 : element_class = (GstElementClass *) klass;
111 :
112 9 : object_class->set_property = gst_tensor_sparse_enc_set_property;
113 9 : object_class->get_property = gst_tensor_sparse_enc_get_property;
114 9 : object_class->finalize = gst_tensor_sparse_enc_finalize;
115 :
116 : /**
117 : * GstTensorSparseEnc::silent:
118 : *
119 : * The flag to enable/disable debugging messages.
120 : */
121 9 : g_object_class_install_property (object_class, PROP_SILENT,
122 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
123 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
124 :
125 9 : gst_element_class_add_pad_template (element_class,
126 : gst_static_pad_template_get (&src_template));
127 :
128 9 : gst_element_class_add_pad_template (element_class,
129 : gst_static_pad_template_get (&sink_template));
130 :
131 9 : gst_element_class_set_static_metadata (element_class,
132 : "TensorSparseEnc",
133 : "Filter/Tensor",
134 : "Element to encode dense tensors into sparse tensors",
135 : "Samsung Electronics Co., Ltd.");
136 9 : }
137 :
138 : /**
139 : * @brief Initialize tensor_sparse_enc element.
140 : */
141 : static void
142 10 : gst_tensor_sparse_enc_init (GstTensorSparseEnc * self)
143 : {
144 : /* setup sink pad */
145 10 : self->sinkpad = gst_pad_new_from_static_template (&sink_template, "sink");
146 10 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);
147 :
148 : /* setup src pad */
149 10 : self->srcpad = gst_pad_new_from_static_template (&src_template, "src");
150 10 : gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
151 :
152 : /* setup chain function */
153 10 : gst_pad_set_chain_function (self->sinkpad,
154 : GST_DEBUG_FUNCPTR (gst_tensor_sparse_enc_chain));
155 :
156 : /* setup event function */
157 10 : gst_pad_set_event_function (self->sinkpad,
158 : GST_DEBUG_FUNCPTR (gst_tensor_sparse_enc_sink_event));
159 :
160 10 : gst_pad_set_query_function (self->sinkpad,
161 : GST_DEBUG_FUNCPTR (gst_tensor_sparse_enc_sink_query));
162 :
163 : /* init properties */
164 10 : self->silent = DEFAULT_SILENT;
165 10 : gst_tensors_config_init (&self->in_config);
166 10 : }
167 :
168 : /**
169 : * @brief Function to finalize instance.
170 : */
171 : static void
172 10 : gst_tensor_sparse_enc_finalize (GObject * object)
173 : {
174 : GstTensorSparseEnc *self;
175 10 : self = GST_TENSOR_SPARSE_ENC (object);
176 :
177 10 : gst_tensors_config_free (&self->in_config);
178 :
179 10 : G_OBJECT_CLASS (parent_class)->finalize (object);
180 10 : }
181 :
182 : /**
183 : * @brief Setter for tensor_sparse_enc properties.
184 : */
185 : static void
186 1 : gst_tensor_sparse_enc_set_property (GObject * object, guint prop_id,
187 : const GValue * value, GParamSpec * pspec)
188 : {
189 : GstTensorSparseEnc *self;
190 :
191 1 : self = GST_TENSOR_SPARSE_ENC (object);
192 :
193 1 : switch (prop_id) {
194 1 : case PROP_SILENT:
195 1 : self->silent = g_value_get_boolean (value);
196 1 : break;
197 0 : default:
198 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
199 0 : break;
200 : }
201 1 : }
202 :
203 : /**
204 : * @brief Getter for tensor_sparse_enc properties.
205 : */
206 : static void
207 2 : gst_tensor_sparse_enc_get_property (GObject * object, guint prop_id,
208 : GValue * value, GParamSpec * pspec)
209 : {
210 : GstTensorSparseEnc *self;
211 :
212 2 : self = GST_TENSOR_SPARSE_ENC (object);
213 :
214 2 : switch (prop_id) {
215 2 : case PROP_SILENT:
216 2 : g_value_set_boolean (value, self->silent);
217 2 : break;
218 0 : default:
219 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
220 0 : break;
221 : }
222 2 : }
223 :
224 : /**
225 : * @brief This function handles sink pad event.
226 : */
227 : static gboolean
228 33 : gst_tensor_sparse_enc_sink_event (GstPad * pad, GstObject * parent,
229 : GstEvent * event)
230 : {
231 : GstTensorSparseEnc *self;
232 33 : self = GST_TENSOR_SPARSE_ENC (parent);
233 :
234 33 : g_return_val_if_fail (event != NULL, FALSE);
235 :
236 33 : switch (GST_EVENT_TYPE (event)) {
237 8 : case GST_EVENT_CAPS:
238 : {
239 : GstCaps *caps;
240 : gboolean ret;
241 :
242 8 : gst_event_parse_caps (event, &caps);
243 8 : silent_debug_caps (self, caps, "caps");
244 :
245 8 : ret = gst_tensors_config_from_cap (&self->in_config, caps);
246 8 : gst_event_unref (event);
247 8 : return ret;
248 : }
249 25 : default:
250 25 : break;
251 : }
252 :
253 25 : return gst_pad_event_default (pad, parent, event);
254 : }
255 :
256 : /**
257 : * @brief Get pad caps for caps negotiation.
258 : */
259 : static GstCaps *
260 48 : gst_tensor_sparse_enc_query_caps (GstTensorSparseEnc * self, GstPad * pad,
261 : GstCaps * filter)
262 : {
263 : GstCaps *caps;
264 :
265 48 : caps = gst_pad_get_current_caps (pad);
266 48 : if (!caps) {
267 : /** pad don't have current caps. use the template caps */
268 48 : caps = gst_pad_get_pad_template_caps (pad);
269 : }
270 :
271 48 : silent_debug_caps (self, caps, "caps");
272 48 : silent_debug_caps (self, filter, "filter");
273 :
274 48 : if (filter) {
275 : GstCaps *intersection;
276 : intersection =
277 7 : gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
278 :
279 7 : gst_caps_unref (caps);
280 7 : caps = intersection;
281 : }
282 :
283 48 : silent_debug_caps (self, caps, "result");
284 48 : return caps;
285 : }
286 :
287 : /**
288 : * @brief This function handles sink pad query.
289 : */
290 : static gboolean
291 69 : gst_tensor_sparse_enc_sink_query (GstPad * pad, GstObject * parent,
292 : GstQuery * query)
293 : {
294 : GstTensorSparseEnc *self;
295 :
296 69 : self = GST_TENSOR_SPARSE_ENC (parent);
297 :
298 69 : GST_DEBUG_OBJECT (self, "Received %s query: %" GST_PTR_FORMAT,
299 : GST_QUERY_TYPE_NAME (query), query);
300 :
301 69 : switch (GST_QUERY_TYPE (query)) {
302 48 : case GST_QUERY_CAPS:
303 : {
304 : GstCaps *caps;
305 : GstCaps *filter;
306 :
307 48 : gst_query_parse_caps (query, &filter);
308 48 : caps = gst_tensor_sparse_enc_query_caps (self, pad, filter);
309 48 : silent_debug_caps (self, filter, "filter");
310 48 : silent_debug_caps (self, caps, "caps");
311 48 : gst_query_set_caps_result (query, caps);
312 48 : gst_caps_unref (caps);
313 48 : return TRUE;
314 : }
315 16 : case GST_QUERY_ACCEPT_CAPS:
316 : {
317 : GstCaps *caps;
318 : GstCaps *template_caps;
319 16 : gboolean res = FALSE;
320 :
321 16 : gst_query_parse_accept_caps (query, &caps);
322 16 : silent_debug_caps (self, caps, "caps");
323 :
324 16 : if (gst_caps_is_fixed (caps)) {
325 16 : template_caps = gst_pad_get_pad_template_caps (pad);
326 :
327 16 : res = gst_caps_can_intersect (template_caps, caps);
328 16 : gst_caps_unref (template_caps);
329 : }
330 :
331 16 : gst_query_set_accept_caps_result (query, res);
332 16 : return TRUE;
333 : }
334 5 : default:
335 5 : break;
336 : }
337 :
338 5 : return gst_pad_query_default (pad, parent, query);
339 : }
340 :
341 :
342 : /**
343 : * @brief Internal function to transform the input buffer.
344 : */
345 : static GstFlowReturn
346 35 : gst_tensor_sparse_enc_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
347 : {
348 35 : GstFlowReturn ret = GST_FLOW_OK;
349 35 : GstTensorSparseEnc *self = GST_TENSOR_SPARSE_ENC (parent);
350 : GstMemory *in_mem, *out_mem;
351 : GstBuffer *outbuf;
352 : GstTensorsInfo *info;
353 : GstTensorInfo *_info;
354 : guint i;
355 :
356 : UNUSED (pad);
357 :
358 35 : info = &self->in_config.info;
359 35 : buf = gst_tensor_buffer_from_config (buf, &self->in_config);
360 35 : outbuf = gst_buffer_new ();
361 :
362 101 : for (i = 0; i < info->num_tensors; ++i) {
363 : GstTensorMetaInfo meta;
364 :
365 66 : _info = gst_tensors_info_get_nth_info (info, i);
366 66 : gst_tensor_info_convert_to_meta (_info, &meta);
367 :
368 66 : meta.format = _NNS_TENSOR_FORMAT_SPARSE;
369 66 : meta.media_type = _NNS_TENSOR;
370 :
371 : /* do real encoding here */
372 66 : in_mem = gst_tensor_buffer_get_nth_memory (buf, i);
373 66 : out_mem = gst_tensor_sparse_from_dense (&meta, in_mem);
374 66 : gst_memory_unref (in_mem);
375 :
376 66 : if (!out_mem) {
377 0 : nns_loge ("failed to convert to sparse tensor");
378 0 : ret = GST_FLOW_ERROR;
379 0 : goto done;
380 : }
381 :
382 66 : gst_tensor_buffer_append_memory (outbuf, out_mem, _info);
383 : }
384 :
385 35 : ret = gst_pad_push (self->srcpad, outbuf);
386 :
387 35 : done:
388 35 : gst_buffer_unref (buf);
389 35 : if (ret != GST_FLOW_OK)
390 0 : gst_buffer_unref (outbuf);
391 :
392 35 : return ret;
393 : }
|
