Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * Copyright (C) 2021 Samsung Electronics Co., Ltd.
4 : *
5 : * @file gsttensor_crop.c
6 : * @date 10 May 2021
7 : * @brief GStreamer element to crop the regions of incoming tensor
8 : * @see https://github.com/nnstreamer/nnstreamer
9 : * @author Jaeyun Jung <jy1210.jung@samsung.com>
10 : * @bug No known bugs except for NYI items
11 : */
12 :
13 : /**
14 : * SECTION:element-tensor_crop
15 : *
16 : * tensor_crop is a GStreamer element to crop the regions of incoming tensor.
17 : *
18 : * tensor_crop has two always sink pads - raw and info.
19 : * The raw pad accepts tensor (other/tensor) which will be cropped with crop info.
20 : * The info pad has capability for flexible tensor stream (other/tensors-flexible), that can have a various buffer size for crop info.
21 : * Incoming buffer on info pad should be an array of crop info.
22 : * Note that NNStreamer supports maximum NNS_TENSOR_SIZE_LIMIT memory blocks in a buffer.
23 : * So, when incoming buffer on info pad has more than NNS_TENSOR_SIZE_LIMIT crop-info array, tensor_crop will ignore the data.
24 : *
25 : * The output is always in the format of other/tensors-flexible.
26 : *
27 : * <refsect2>
28 : * <title>Example launch line</title>
29 : * |[
30 : * gst-launch-1.0 tensor_crop name=crop ! (cropped tensors) ... \
31 : * videotestsrc ! videoconvert ! video/x-raw,format=RGB ! tensor_converter ! tee name=t \
32 : * t. ! queue ! crop.raw \
33 : * t. ! queue ! (process raw video tensor and push buffer which includes crop info) ! crop.info
34 : * ]|
35 : * </refsect2>
36 : */
37 :
38 : #ifdef HAVE_CONFIG_H
39 : #include <config.h>
40 : #endif
41 :
42 : #include <string.h>
43 : #include <nnstreamer_util.h>
44 : #include "gsttensor_crop.h"
45 : #include "tensor_data.h"
46 :
47 : /**
48 : * @brief Internal data structure to describe tensor region.
49 : */
50 : typedef struct
51 : {
52 : guint x;
53 : guint y;
54 : guint w;
55 : guint h;
56 : } tensor_region_s;
57 :
58 : /**
59 : * @brief Internal data structure to describe cropping tensor data.
60 : * @todo Add various mode to crop tensor. Now tensor-crop handles NHWC data format only.
61 : */
62 : typedef struct
63 : {
64 : guint num;
65 : tensor_region_s region[NNS_TENSOR_SIZE_LIMIT];
66 : } tensor_crop_info_s;
67 :
68 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_crop_debug);
69 : #define GST_CAT_DEFAULT gst_tensor_crop_debug
70 :
71 : /**
72 : * @brief tensor_crop properties
73 : */
74 : enum
75 : {
76 : PROP_0,
77 : PROP_LATENESS,
78 : PROP_SILENT
79 : };
80 :
81 : /**
82 : * @brief Flag to print minimized log.
83 : */
84 : #define DEFAULT_SILENT TRUE
85 :
86 : /**
87 : * @brief Default value to compare timestamp of raw and info buffer, in milliseconds (-1 means no synchronization).
88 : */
89 : #define DEFAULT_LATENESS (-1)
90 :
91 : /**
92 : * @brief Template for sink pad (raw data).
93 : */
94 : static GstStaticPadTemplate raw_template = GST_STATIC_PAD_TEMPLATE ("raw",
95 : GST_PAD_SINK,
96 : GST_PAD_ALWAYS,
97 : GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT ";"
98 : GST_TENSORS_CAP_MAKE ("{ static, flexible }")));
99 :
100 : /**
101 : * @brief Template for sink pad (crop info).
102 : */
103 : static GstStaticPadTemplate info_template = GST_STATIC_PAD_TEMPLATE ("info",
104 : GST_PAD_SINK,
105 : GST_PAD_ALWAYS,
106 : GST_STATIC_CAPS (GST_TENSORS_FLEX_CAP_DEFAULT));
107 :
108 : /**
109 : * @brief Template for src pad.
110 : */
111 : static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
112 : GST_PAD_SRC,
113 : GST_PAD_ALWAYS,
114 : GST_STATIC_CAPS (GST_TENSORS_FLEX_CAP_DEFAULT));
115 :
116 : #define gst_tensor_crop_parent_class parent_class
117 953 : G_DEFINE_TYPE (GstTensorCrop, gst_tensor_crop, GST_TYPE_ELEMENT);
118 :
119 : static void gst_tensor_crop_finalize (GObject * object);
120 : static void gst_tensor_crop_set_property (GObject * object, guint prop_id,
121 : const GValue * value, GParamSpec * pspec);
122 : static void gst_tensor_crop_get_property (GObject * object, guint prop_id,
123 : GValue * value, GParamSpec * pspec);
124 : static GstStateChangeReturn gst_tensor_crop_change_state (GstElement * element,
125 : GstStateChange transition);
126 : static gboolean gst_tensor_crop_src_event (GstPad * pad, GstObject * parent,
127 : GstEvent * event);
128 : static gboolean gst_tensor_crop_sink_event (GstCollectPads * pads,
129 : GstCollectData * data, GstEvent * event, gpointer user_data);
130 : static GstFlowReturn gst_tensor_crop_collected (GstCollectPads * pads,
131 : gpointer user_data);
132 :
133 : /**
134 : * @brief Initialize the tensor_crop's class.
135 : */
136 : static void
137 2 : gst_tensor_crop_class_init (GstTensorCropClass * klass)
138 : {
139 : GObjectClass *object_class;
140 : GstElementClass *element_class;
141 :
142 2 : GST_DEBUG_CATEGORY_INIT (gst_tensor_crop_debug, "tensor_crop", 0,
143 : "Element to crop the regions of incoming tensor");
144 :
145 2 : object_class = (GObjectClass *) klass;
146 2 : element_class = (GstElementClass *) klass;
147 :
148 2 : object_class->set_property = gst_tensor_crop_set_property;
149 2 : object_class->get_property = gst_tensor_crop_get_property;
150 2 : object_class->finalize = gst_tensor_crop_finalize;
151 :
152 : /**
153 : * GstTensorCrop::lateness:
154 : *
155 : * The time difference between raw and info buffer, in milliseconds (-1 means no synchronization).
156 : * If raw and info buffers on the pads have different timestamp and time-diff is larger than 'lateness',
157 : * tensor-crop will drop old buffer and wait for next buffers.
158 : */
159 2 : g_object_class_install_property (object_class, PROP_LATENESS,
160 : g_param_spec_int ("lateness", "Lateness",
161 : "The time difference between raw and info buffer in milliseconds",
162 : -1, G_MAXINT, DEFAULT_LATENESS,
163 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
164 :
165 : /**
166 : * GstTensorCrop::silent:
167 : *
168 : * The flag to enable/disable debugging messages.
169 : */
170 2 : g_object_class_install_property (object_class, PROP_SILENT,
171 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
172 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
173 :
174 2 : element_class->change_state =
175 2 : GST_DEBUG_FUNCPTR (gst_tensor_crop_change_state);
176 :
177 2 : gst_element_class_add_pad_template (element_class,
178 : gst_static_pad_template_get (&src_template));
179 2 : gst_element_class_add_pad_template (element_class,
180 : gst_static_pad_template_get (&raw_template));
181 2 : gst_element_class_add_pad_template (element_class,
182 : gst_static_pad_template_get (&info_template));
183 :
184 2 : gst_element_class_set_static_metadata (element_class,
185 : "TensorCrop",
186 : "Filter/Tensor",
187 : "Element to crop the regions of incoming tensor",
188 : "Samsung Electronics Co., Ltd.");
189 2 : }
190 :
191 : /**
192 : * @brief Clear and reset old pad data.
193 : */
194 : static void
195 32 : gst_tensor_crop_pad_reset (GstTensorCropPadData * cpad)
196 : {
197 32 : gst_tensors_config_free (&cpad->config);
198 32 : gst_tensors_config_init (&cpad->config);
199 32 : }
200 :
201 : /**
202 : * @brief Clear and reset old data in tensor_crop.
203 : */
204 : static void
205 16 : gst_tensor_crop_reset (GstTensorCrop * self)
206 : {
207 : GstTensorCropPadData *cpad;
208 : GSList *walk;
209 :
210 16 : if (self->collect) {
211 16 : walk = self->collect->data;
212 :
213 48 : while (walk) {
214 32 : cpad = (GstTensorCropPadData *) walk->data;
215 :
216 32 : gst_tensor_crop_pad_reset (cpad);
217 32 : walk = g_slist_next (walk);
218 : }
219 : }
220 :
221 16 : self->send_stream_start = TRUE;
222 16 : }
223 :
224 : /**
225 : * @brief Initialize tensor_crop element.
226 : */
227 : static void
228 8 : gst_tensor_crop_init (GstTensorCrop * self)
229 : {
230 : /* setup sink pad */
231 8 : self->sinkpad_raw = gst_pad_new_from_static_template (&raw_template, "raw");
232 8 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad_raw);
233 :
234 8 : self->sinkpad_info =
235 8 : gst_pad_new_from_static_template (&info_template, "info");
236 8 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad_info);
237 :
238 8 : self->collect = gst_collect_pads_new ();
239 8 : gst_collect_pads_set_function (self->collect,
240 8 : GST_DEBUG_FUNCPTR (gst_tensor_crop_collected), self);
241 8 : gst_collect_pads_set_event_function (self->collect,
242 8 : GST_DEBUG_FUNCPTR (gst_tensor_crop_sink_event), self);
243 :
244 8 : gst_collect_pads_add_pad (self->collect, self->sinkpad_raw,
245 : sizeof (GstTensorCropPadData), NULL, TRUE);
246 8 : gst_collect_pads_add_pad (self->collect, self->sinkpad_info,
247 : sizeof (GstTensorCropPadData), NULL, TRUE);
248 :
249 : /* setup src pad */
250 8 : self->srcpad = gst_pad_new_from_static_template (&src_template, "src");
251 8 : gst_pad_set_event_function (self->srcpad,
252 : GST_DEBUG_FUNCPTR (gst_tensor_crop_src_event));
253 8 : gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
254 :
255 : /* init properties */
256 8 : self->lateness = DEFAULT_LATENESS;
257 8 : self->silent = DEFAULT_SILENT;
258 8 : self->send_stream_start = TRUE;
259 8 : }
260 :
261 : /**
262 : * @brief Function to finalize instance.
263 : */
264 : static void
265 8 : gst_tensor_crop_finalize (GObject * object)
266 : {
267 : GstTensorCrop *self;
268 :
269 8 : self = GST_TENSOR_CROP (object);
270 :
271 8 : gst_tensor_crop_reset (self);
272 :
273 8 : if (self->collect) {
274 8 : gst_object_unref (self->collect);
275 8 : self->collect = NULL;
276 : }
277 :
278 8 : G_OBJECT_CLASS (parent_class)->finalize (object);
279 8 : }
280 :
281 : /**
282 : * @brief Setter for tensor_crop properties.
283 : */
284 : static void
285 3 : gst_tensor_crop_set_property (GObject * object, guint prop_id,
286 : const GValue * value, GParamSpec * pspec)
287 : {
288 : GstTensorCrop *self;
289 :
290 3 : self = GST_TENSOR_CROP (object);
291 :
292 3 : switch (prop_id) {
293 2 : case PROP_LATENESS:
294 2 : self->lateness = g_value_get_int (value);
295 2 : break;
296 1 : case PROP_SILENT:
297 1 : self->silent = g_value_get_boolean (value);
298 1 : break;
299 0 : default:
300 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
301 0 : break;
302 : }
303 3 : }
304 :
305 : /**
306 : * @brief Getter for tensor_crop properties.
307 : */
308 : static void
309 4 : gst_tensor_crop_get_property (GObject * object, guint prop_id,
310 : GValue * value, GParamSpec * pspec)
311 : {
312 : GstTensorCrop *self;
313 :
314 4 : self = GST_TENSOR_CROP (object);
315 :
316 4 : switch (prop_id) {
317 2 : case PROP_LATENESS:
318 2 : g_value_set_int (value, self->lateness);
319 2 : break;
320 2 : case PROP_SILENT:
321 2 : g_value_set_boolean (value, self->silent);
322 2 : break;
323 0 : default:
324 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
325 0 : break;
326 : }
327 4 : }
328 :
329 : /**
330 : * @brief Handle state transition.
331 : */
332 : static GstStateChangeReturn
333 62 : gst_tensor_crop_change_state (GstElement * element, GstStateChange transition)
334 : {
335 : GstTensorCrop *self;
336 : GstStateChangeReturn ret;
337 :
338 62 : self = GST_TENSOR_CROP (element);
339 :
340 62 : switch (transition) {
341 8 : case GST_STATE_CHANGE_READY_TO_PAUSED:
342 8 : gst_collect_pads_start (self->collect);
343 8 : break;
344 8 : case GST_STATE_CHANGE_PAUSED_TO_READY:
345 8 : gst_collect_pads_stop (self->collect);
346 8 : break;
347 46 : default:
348 46 : break;
349 : }
350 :
351 62 : ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
352 :
353 62 : switch (transition) {
354 8 : case GST_STATE_CHANGE_PAUSED_TO_READY:
355 8 : gst_tensor_crop_reset (self);
356 8 : break;
357 54 : default:
358 54 : break;
359 : }
360 :
361 62 : return ret;
362 : }
363 :
364 : /**
365 : * @brief Handle event on src pad.
366 : */
367 : static gboolean
368 3 : gst_tensor_crop_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
369 : {
370 3 : g_return_val_if_fail (event != NULL, FALSE);
371 :
372 3 : switch (GST_EVENT_TYPE (event)) {
373 1 : case GST_EVENT_SEEK:
374 : /* disable seeking */
375 1 : gst_event_unref (event);
376 1 : return FALSE;
377 2 : default:
378 2 : break;
379 : }
380 :
381 2 : return gst_pad_event_default (pad, parent, event);
382 : }
383 :
384 : /**
385 : * @brief Handle event on sink pad.
386 : */
387 : static gboolean
388 60 : gst_tensor_crop_sink_event (GstCollectPads * pads, GstCollectData * data,
389 : GstEvent * event, gpointer user_data)
390 : {
391 : GstTensorCropPadData *cpad;
392 : UNUSED (user_data);
393 :
394 60 : g_return_val_if_fail (event != NULL, FALSE);
395 :
396 60 : cpad = (GstTensorCropPadData *) data;
397 :
398 60 : switch (GST_EVENT_TYPE (event)) {
399 15 : case GST_EVENT_CAPS:
400 : {
401 : GstCaps *caps;
402 : gboolean ret;
403 15 : gst_event_parse_caps (event, &caps);
404 :
405 15 : ret = gst_tensors_config_from_cap (&cpad->config, caps);
406 15 : gst_event_unref (event);
407 :
408 15 : return ret;
409 : }
410 45 : default:
411 45 : break;
412 : }
413 :
414 45 : return gst_collect_pads_event_default (pads, data, event, FALSE);
415 : }
416 :
417 : /**
418 : * @brief Set pad caps if not negotiated.
419 : */
420 : static GstFlowReturn
421 10 : gst_tensor_crop_negotiate (GstTensorCrop * self)
422 : {
423 10 : if (!gst_pad_has_current_caps (self->sinkpad_raw)) {
424 0 : GST_ERROR_OBJECT (self,
425 : "The raw pad of tensor_crop '%s' does not have pad caps.",
426 : GST_ELEMENT_NAME (self));
427 0 : return GST_FLOW_NOT_NEGOTIATED;
428 : }
429 :
430 10 : if (!gst_pad_has_current_caps (self->sinkpad_info)) {
431 0 : GST_ERROR_OBJECT (self,
432 : "The info pad of tensor_crop '%s' does not have pad caps.",
433 : GST_ELEMENT_NAME (self));
434 0 : return GST_FLOW_NOT_NEGOTIATED;
435 : }
436 :
437 10 : if (!gst_pad_has_current_caps (self->srcpad)) {
438 : GstCaps *caps;
439 : GstSegment segment;
440 : GstTensorsConfig config;
441 : GstTensorCropPadData *cpad;
442 : GSList *walk;
443 :
444 6 : if (self->send_stream_start) {
445 : /**
446 : * Cannot use gst-pad util to get stream ID (multiple sink pads).
447 : * Create stream ID using first sink pad.
448 : */
449 6 : g_autofree gchar *sink_stream_id = gst_pad_get_stream_id (self->sinkpad_raw);
450 6 : g_autofree gchar *element_name = gst_element_get_name (self);
451 6 : g_autofree gchar *pad_name = gst_pad_get_name (self->srcpad);
452 12 : g_autofree gchar *stream_id = g_strdup_printf ("%s-%s-nnscrop-%s-%08x",
453 6 : GST_STR_NULL (sink_stream_id), element_name, pad_name, g_random_int ());
454 :
455 6 : gst_pad_push_event (self->srcpad, gst_event_new_stream_start (stream_id));
456 6 : self->send_stream_start = FALSE;
457 : }
458 :
459 : /**
460 : * Get config from collect-pads and set framerate.
461 : * Output is always flexible tensor.
462 : */
463 6 : gst_tensors_config_init (&config);
464 6 : config.info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
465 :
466 6 : walk = self->collect->data;
467 18 : while (walk) {
468 12 : cpad = (GstTensorCropPadData *) walk->data;
469 :
470 12 : if (config.rate_n < 0 ||
471 6 : gst_util_fraction_compare (cpad->config.rate_n, cpad->config.rate_d,
472 : config.rate_n, config.rate_d) < 0) {
473 6 : config.rate_n = cpad->config.rate_n;
474 6 : config.rate_d = cpad->config.rate_d;
475 : }
476 :
477 12 : walk = g_slist_next (walk);
478 : }
479 :
480 6 : caps = gst_tensors_caps_from_config (&config);
481 6 : gst_pad_set_caps (self->srcpad, caps);
482 6 : gst_caps_unref (caps);
483 :
484 6 : gst_segment_init (&segment, GST_FORMAT_TIME);
485 6 : gst_pad_push_event (self->srcpad, gst_event_new_segment (&segment));
486 : }
487 :
488 10 : return GST_FLOW_OK;
489 : }
490 :
491 : /**
492 : * @brief Internal function to prepare output meta info.
493 : */
494 : static gboolean
495 6 : gst_tensor_crop_prepare_out_meta (GstTensorCrop * self, gpointer buffer,
496 : GstTensorMetaInfo * meta, GstTensorInfo * info, gboolean * is_flexible)
497 : {
498 : GstCaps *caps;
499 : GstStructure *structure;
500 : GstTensorsConfig config;
501 : GstTensorInfo *_info;
502 6 : gboolean ret = FALSE;
503 :
504 6 : gst_tensor_meta_info_init (meta);
505 6 : gst_tensor_info_init (info);
506 :
507 6 : caps = gst_pad_get_current_caps (self->sinkpad_raw);
508 6 : structure = gst_caps_get_structure (caps, 0);
509 :
510 6 : if (!gst_tensors_config_from_structure (&config, structure)) {
511 0 : GST_ERROR_OBJECT (self, "Failed to get the config from caps.");
512 0 : goto done;
513 : }
514 :
515 : /**
516 : * @note tensor-crop handles single tensor. Parse first one.
517 : */
518 6 : _info = gst_tensors_info_get_nth_info (&config.info, 0);
519 6 : *is_flexible = gst_tensors_config_is_flexible (&config);
520 :
521 6 : if (*is_flexible) {
522 : /* meta from buffer */
523 1 : if (gst_tensor_meta_info_parse_header (meta, buffer)) {
524 1 : ret = gst_tensor_meta_info_convert (meta, info);
525 : }
526 : } else {
527 : /* meta from caps */
528 5 : ret = gst_tensor_info_convert_to_meta (_info, meta);
529 5 : gst_tensor_info_copy (info, _info);
530 : }
531 :
532 : /* output is flex tensor */
533 6 : meta->format = _NNS_TENSOR_FORMAT_FLEXIBLE;
534 :
535 6 : done:
536 6 : gst_caps_unref (caps);
537 6 : gst_tensors_config_free (&config);
538 6 : return ret;
539 : }
540 :
541 : /**
542 : * @brief Internal function to parse buffer and fill crop info.
543 : */
544 : static gboolean
545 7 : gst_tensor_crop_get_crop_info (GstTensorCrop * self, GstBuffer * info,
546 : tensor_crop_info_s * cinfo)
547 : {
548 : GstMemory *mem;
549 : GstMapInfo map;
550 : GstTensorMetaInfo meta;
551 : gsize hsize, dsize, esize;
552 : guint i, j;
553 : guint8 *pos, *src, *desc;
554 7 : gboolean ret = FALSE;
555 :
556 7 : i = gst_tensor_buffer_get_count (info);
557 7 : g_assert (i > 0);
558 7 : if (i > 1) {
559 0 : GST_WARNING_OBJECT (self,
560 : "Info buffer has %u memories, parse first one.", i);
561 : }
562 :
563 7 : mem = gst_tensor_buffer_get_nth_memory (info, 0);
564 7 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
565 0 : GST_ERROR_OBJECT (self, "Failed to map the info buffer.");
566 0 : goto done;
567 : }
568 :
569 : /* parse crop-info from flex tensor */
570 7 : if (!gst_tensor_meta_info_parse_header (&meta, map.data)) {
571 0 : GST_ERROR_OBJECT (self, "Failed to get the meta from info buffer.");
572 0 : goto done;
573 : }
574 :
575 7 : hsize = gst_tensor_meta_info_get_header_size (&meta);
576 7 : dsize = gst_tensor_meta_info_get_data_size (&meta);
577 7 : esize = gst_tensor_get_element_size (meta.type);
578 :
579 7 : if (hsize + dsize != map.size) {
580 1 : GST_ERROR_OBJECT (self,
581 : "Invalid meta info, info buffer size is incorrect (received %zd, expected %zd).",
582 : map.size, hsize + dsize);
583 1 : goto done;
584 : }
585 :
586 : /**
587 : * @todo Add various mode to crop tensor.
588 : * Now tensor-crop handles NHWC data format only.
589 : */
590 6 : g_assert ((dsize % (esize * 4)) == 0);
591 :
592 6 : memset (cinfo, 0, sizeof (tensor_crop_info_s));
593 :
594 6 : cinfo->num = MIN (dsize / (esize * 4), NNS_TENSOR_SIZE_LIMIT);
595 :
596 17 : for (i = 0; i < cinfo->num; i++) {
597 11 : pos = map.data + hsize + (esize * 4 * i);
598 :
599 55 : for (j = 0; j < 4; j++) {
600 44 : src = pos + (esize * j);
601 44 : desc = (guint8 *) (&cinfo->region[i]) + sizeof (guint) * j;
602 :
603 44 : gst_tensor_data_raw_typecast (src, meta.type, desc, _NNS_UINT32);
604 : }
605 : }
606 :
607 6 : ret = TRUE;
608 :
609 7 : done:
610 7 : if (mem) {
611 7 : gst_memory_unmap (mem, &map);
612 7 : gst_memory_unref (mem);
613 : }
614 :
615 7 : return ret;
616 : }
617 :
618 : /**
619 : * @brief Internal function to crop incoming buffer.
620 : */
621 : static GstBuffer *
622 6 : gst_tensor_crop_do_cropping (GstTensorCrop * self, GstBuffer * raw,
623 : tensor_crop_info_s * cinfo)
624 : {
625 6 : GstBuffer *result = NULL;
626 : GstMemory *mem;
627 : GstMapInfo map;
628 : GstTensorMetaInfo meta;
629 : GstTensorInfo info;
630 : gboolean flexible;
631 : gsize hsize, esize, dsize;
632 : guint8 *cropped, *dpos, *desc, *src;
633 : guint i, j, ch, mw, mh, _x, _y, _w, _h;
634 :
635 6 : i = gst_tensor_buffer_get_count (raw);
636 6 : g_assert (i > 0);
637 6 : if (i > 1) {
638 0 : GST_WARNING_OBJECT (self,
639 : "Raw data buffer has %u memories, parse first one.", i);
640 : }
641 :
642 6 : mem = gst_tensor_buffer_get_nth_memory (raw, 0);
643 6 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
644 0 : GST_ERROR_OBJECT (self, "Failed to map the raw buffer.");
645 0 : goto done;
646 : }
647 :
648 6 : if (!gst_tensor_crop_prepare_out_meta (self, map.data, &meta,
649 : &info, &flexible)) {
650 0 : GST_ERROR_OBJECT (self, "Failed to get the output meta.");
651 0 : goto done;
652 : }
653 :
654 6 : hsize = flexible ? gst_tensor_meta_info_get_header_size (&meta) : 0;
655 6 : dsize = gst_tensor_meta_info_get_data_size (&meta);
656 6 : dpos = map.data + hsize;
657 6 : if ((hsize + dsize) != map.size) {
658 1 : GST_ERROR_OBJECT (self,
659 : "Raw buffer has invalid data size (received %zd, expected %zd).",
660 : map.size, dsize);
661 1 : goto done;
662 : }
663 :
664 5 : result = gst_buffer_new ();
665 :
666 : /** @todo Add various mode to crop tensor. */
667 5 : ch = info.dimension[0];
668 5 : mw = info.dimension[1];
669 5 : mh = info.dimension[2];
670 5 : esize = gst_tensor_get_element_size (info.type);
671 5 : hsize = gst_tensor_meta_info_get_header_size (&meta);
672 :
673 14 : for (i = 0; i < cinfo->num; i++) {
674 : GstTensorInfo crop_info;
675 : GstMemory *crop_mem;
676 :
677 9 : _x = (cinfo->region[i].x < mw) ? cinfo->region[i].x : mw;
678 9 : _y = (cinfo->region[i].y < mh) ? cinfo->region[i].y : mh;
679 9 : _w = (_x + cinfo->region[i].w - 1 < mw) ? cinfo->region[i].w : (mw - _x);
680 9 : _h = (_y + cinfo->region[i].h - 1 < mh) ? cinfo->region[i].h : (mh - _y);
681 :
682 9 : g_assert (_w > 0 && _h > 0);
683 9 : dsize = hsize + (esize * ch * _w * _h);
684 9 : cropped = (guint8 *) g_malloc0 (dsize);
685 :
686 : /* set header for flex tensor */
687 9 : meta.dimension[1] = _w;
688 9 : meta.dimension[2] = _h;
689 9 : meta.dimension[3] = 1;
690 9 : gst_tensor_meta_info_update_header (&meta, cropped);
691 :
692 232 : for (j = 0; j < _h; j++) {
693 223 : src = dpos + esize * ch * (_x + (j + _y) * mw);
694 223 : desc = cropped + hsize + (esize * ch * _w) * j;
695 223 : memcpy (desc, src, (esize * ch * _w));
696 : }
697 :
698 : /* Prepare tensors-info to append memory into gst-buffer. */
699 9 : gst_tensor_meta_info_convert (&meta, &crop_info);
700 : crop_mem =
701 9 : gst_memory_new_wrapped (0, cropped, dsize, 0, dsize, cropped, g_free);
702 :
703 9 : gst_tensor_buffer_append_memory (result, crop_mem, &crop_info);
704 9 : gst_tensor_info_free (&crop_info);
705 : }
706 :
707 : /* set timestamp from raw buffer */
708 5 : gst_buffer_copy_into (result, raw, GST_BUFFER_COPY_METADATA, 0, -1);
709 :
710 6 : done:
711 6 : if (mem) {
712 6 : gst_memory_unmap (mem, &map);
713 6 : gst_memory_unref (mem);
714 : }
715 :
716 6 : return result;
717 : }
718 :
719 : /**
720 : * @brief Internal function to transform the input buffer.
721 : */
722 : static GstFlowReturn
723 9 : gst_tensor_crop_chain (GstTensorCrop * self,
724 : GstCollectData * data_raw, GstCollectData * data_info)
725 : {
726 : GstFlowReturn ret;
727 : GstBuffer *buf_raw, *buf_info, *result;
728 : GstTensorCropPadData *cpad;
729 : tensor_crop_info_s cinfo;
730 : gboolean drop_raw, drop_info;
731 :
732 18 : g_return_val_if_fail (data_raw && data_info, GST_FLOW_ERROR);
733 :
734 9 : buf_raw = gst_collect_pads_peek (self->collect, data_raw);
735 9 : buf_info = gst_collect_pads_peek (self->collect, data_info);
736 9 : drop_raw = (buf_raw != NULL);
737 9 : drop_info = (buf_info != NULL);
738 :
739 9 : if (!buf_raw || !buf_info) {
740 0 : ret = GST_FLOW_EOS;
741 0 : goto done;
742 : }
743 :
744 9 : cpad = (GstTensorCropPadData *) data_raw;
745 9 : buf_raw = gst_tensor_buffer_from_config (buf_raw, &cpad->config);
746 9 : cpad = (GstTensorCropPadData *) data_info;
747 9 : buf_info = gst_tensor_buffer_from_config (buf_info, &cpad->config);
748 :
749 : /**
750 : * The case when raw and info have different timestamp.
751 : * Compare timestamp and if time diff is less than lateness, crop raw buffer.
752 : */
753 9 : if (self->lateness >= 0) {
754 : GstClockTime ts_raw, ts_info, lateness;
755 :
756 4 : ts_raw = GST_BUFFER_TIMESTAMP (buf_raw);
757 4 : ts_info = GST_BUFFER_TIMESTAMP (buf_info);
758 4 : lateness = self->lateness * GST_MSECOND;
759 :
760 4 : if (GST_CLOCK_TIME_IS_VALID (ts_raw) && GST_CLOCK_TIME_IS_VALID (ts_info)) {
761 4 : if (((GstClockTime) ABS (GST_CLOCK_DIFF (ts_raw, ts_info))) > lateness) {
762 2 : GST_DEBUG_OBJECT (self, "Drop old buffer and wait for next.");
763 2 : GST_DEBUG_OBJECT (self, "Raw buffer ts: %" GST_TIME_FORMAT,
764 : GST_TIME_ARGS (ts_raw));
765 2 : GST_DEBUG_OBJECT (self, "Info buffer ts: %" GST_TIME_FORMAT,
766 : GST_TIME_ARGS (ts_info));
767 :
768 : /* clear old buffer and return ok to get next buffer */
769 2 : if (ts_raw > ts_info)
770 1 : drop_raw = FALSE;
771 : else
772 1 : drop_info = FALSE;
773 :
774 2 : ret = GST_FLOW_OK;
775 2 : goto done;
776 : }
777 : } else {
778 0 : GST_WARNING_OBJECT (self,
779 : "Incoming buffer has invalid timestamp, continue cropping data.");
780 : }
781 : }
782 :
783 7 : if (!gst_tensor_crop_get_crop_info (self, buf_info, &cinfo)) {
784 1 : ret = GST_FLOW_ERROR;
785 1 : goto done;
786 : }
787 :
788 6 : result = gst_tensor_crop_do_cropping (self, buf_raw, &cinfo);
789 6 : ret = gst_pad_push (self->srcpad, result);
790 :
791 9 : done:
792 9 : if (buf_raw)
793 9 : gst_buffer_unref (buf_raw);
794 9 : if (buf_info)
795 9 : gst_buffer_unref (buf_info);
796 :
797 : /* clear buffer in collect pads */
798 9 : if (drop_raw)
799 8 : gst_buffer_unref (gst_collect_pads_pop (self->collect, data_raw));
800 9 : if (drop_info)
801 8 : gst_buffer_unref (gst_collect_pads_pop (self->collect, data_info));
802 :
803 9 : return ret;
804 : }
805 :
806 : /**
807 : * @brief Chain function called when the buffer is available on all of the collect pads.
808 : */
809 : static GstFlowReturn
810 10 : gst_tensor_crop_collected (GstCollectPads * pads, gpointer user_data)
811 : {
812 : GstTensorCrop *self;
813 : GstCollectData *data_raw, *data_info;
814 : GSList *walk;
815 : GstFlowReturn ret;
816 :
817 10 : self = GST_TENSOR_CROP (user_data);
818 10 : data_raw = data_info = NULL;
819 :
820 10 : ret = gst_tensor_crop_negotiate (self);
821 10 : if (ret != GST_FLOW_OK)
822 0 : return ret;
823 :
824 28 : for (walk = pads->data; walk; walk = g_slist_next (walk)) {
825 : GstCollectData *data;
826 :
827 19 : data = (GstCollectData *) walk->data;
828 :
829 19 : if (GST_COLLECT_PADS_STATE_IS_SET (data, GST_COLLECT_PADS_STATE_EOS)) {
830 1 : gst_pad_push_event (self->srcpad, gst_event_new_eos ());
831 1 : return GST_FLOW_EOS;
832 : }
833 :
834 18 : if (data->pad == self->sinkpad_raw) {
835 9 : data_raw = data;
836 9 : } else if (data->pad == self->sinkpad_info) {
837 9 : data_info = data;
838 : }
839 : }
840 :
841 9 : return gst_tensor_crop_chain (self, data_raw, data_info);
842 : }
|
