Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * Copyright (C) 2021 Samsung Electronics Co., Ltd.
4 : *
5 : * @file gsttensor_crop.c
6 : * @date 10 May 2021
7 : * @brief GStreamer element to crop the regions of incoming tensor
8 : * @see https://github.com/nnstreamer/nnstreamer
9 : * @author Jaeyun Jung <jy1210.jung@samsung.com>
10 : * @bug No known bugs except for NYI items
11 : */
12 :
13 : /**
14 : * SECTION:element-tensor_crop
15 : *
16 : * tensor_crop is a GStreamer element to crop the regions of incoming tensor.
17 : *
18 : * tensor_crop has two always sink pads - raw and info.
19 : * The raw pad accepts tensor (other/tensor) which will be cropped with crop info.
20 : * The info pad has capability for flexible tensor stream (other/tensors-flexible), that can have a various buffer size for crop info.
21 : * Incoming buffer on info pad should be an array of crop info.
22 : * Note that NNStreamer supports maximum NNS_TENSOR_SIZE_LIMIT memory blocks in a buffer.
23 : * So, when incoming buffer on info pad has more than NNS_TENSOR_SIZE_LIMIT crop-info array, tensor_crop will ignore the data.
24 : *
25 : * The output is always in the format of other/tensors-flexible.
26 : *
27 : * <refsect2>
28 : * <title>Example launch line</title>
29 : * |[
30 : * gst-launch-1.0 tensor_crop name=crop ! (cropped tensors) ... \
31 : * videotestsrc ! videoconvert ! video/x-raw,format=RGB ! tensor_converter ! tee name=t \
32 : * t. ! queue ! crop.raw \
33 : * t. ! queue ! (process raw video tensor and push buffer which includes crop info) ! crop.info
34 : * ]|
35 : * </refsect2>
36 : */
37 :
38 : #ifdef HAVE_CONFIG_H
39 : #include <config.h>
40 : #endif
41 :
42 : #include <string.h>
43 : #include <nnstreamer_util.h>
44 : #include "gsttensor_crop.h"
45 : #include "tensor_data.h"
46 :
47 : /**
48 : * @brief Internal data structure to describe tensor region.
49 : */
50 : typedef struct
51 : {
52 : guint x;
53 : guint y;
54 : guint w;
55 : guint h;
56 : } tensor_region_s;
57 :
58 : /**
59 : * @brief Internal data structure to describe cropping tensor data.
60 : * @todo Add various mode to crop tensor. Now tensor-crop handles NHWC data format only.
61 : */
62 : typedef struct
63 : {
64 : guint num;
65 : tensor_region_s region[NNS_TENSOR_SIZE_LIMIT];
66 : } tensor_crop_info_s;
67 :
68 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_crop_debug);
69 : #define GST_CAT_DEFAULT gst_tensor_crop_debug
70 :
71 : /**
72 : * @brief tensor_crop properties
73 : */
74 : enum
75 : {
76 : PROP_0,
77 : PROP_LATENESS,
78 : PROP_SILENT
79 : };
80 :
81 : /**
82 : * @brief Flag to print minimized log.
83 : */
84 : #define DEFAULT_SILENT TRUE
85 :
86 : /**
87 : * @brief Default value to compare timestamp of raw and info buffer, in milliseconds (-1 means no synchronization).
88 : */
89 : #define DEFAULT_LATENESS (-1)
90 :
91 : /**
92 : * @brief Template for sink pad (raw data).
93 : */
94 : static GstStaticPadTemplate raw_template = GST_STATIC_PAD_TEMPLATE ("raw",
95 : GST_PAD_SINK,
96 : GST_PAD_ALWAYS,
97 : GST_STATIC_CAPS (GST_TENSOR_CAP_DEFAULT ";"
98 : GST_TENSORS_CAP_MAKE ("{ static, flexible }")));
99 :
100 : /**
101 : * @brief Template for sink pad (crop info).
102 : */
103 : static GstStaticPadTemplate info_template = GST_STATIC_PAD_TEMPLATE ("info",
104 : GST_PAD_SINK,
105 : GST_PAD_ALWAYS,
106 : GST_STATIC_CAPS (GST_TENSORS_FLEX_CAP_DEFAULT));
107 :
108 : /**
109 : * @brief Template for src pad.
110 : */
111 : static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
112 : GST_PAD_SRC,
113 : GST_PAD_ALWAYS,
114 : GST_STATIC_CAPS (GST_TENSORS_FLEX_CAP_DEFAULT));
115 :
116 : #define gst_tensor_crop_parent_class parent_class
117 983 : G_DEFINE_TYPE (GstTensorCrop, gst_tensor_crop, GST_TYPE_ELEMENT);
118 :
119 : static void gst_tensor_crop_finalize (GObject * object);
120 : static void gst_tensor_crop_set_property (GObject * object, guint prop_id,
121 : const GValue * value, GParamSpec * pspec);
122 : static void gst_tensor_crop_get_property (GObject * object, guint prop_id,
123 : GValue * value, GParamSpec * pspec);
124 : static GstStateChangeReturn gst_tensor_crop_change_state (GstElement * element,
125 : GstStateChange transition);
126 : static gboolean gst_tensor_crop_src_event (GstPad * pad, GstObject * parent,
127 : GstEvent * event);
128 : static gboolean gst_tensor_crop_sink_event (GstCollectPads * pads,
129 : GstCollectData * data, GstEvent * event, gpointer user_data);
130 : static GstFlowReturn gst_tensor_crop_collected (GstCollectPads * pads,
131 : gpointer user_data);
132 :
133 : /**
134 : * @brief Initialize the tensor_crop's class.
135 : */
136 : static void
137 12 : gst_tensor_crop_class_init (GstTensorCropClass * klass)
138 : {
139 : GObjectClass *object_class;
140 : GstElementClass *element_class;
141 :
142 12 : GST_DEBUG_CATEGORY_INIT (gst_tensor_crop_debug, "tensor_crop", 0,
143 : "Element to crop the regions of incoming tensor");
144 :
145 12 : object_class = (GObjectClass *) klass;
146 12 : element_class = (GstElementClass *) klass;
147 :
148 12 : object_class->set_property = gst_tensor_crop_set_property;
149 12 : object_class->get_property = gst_tensor_crop_get_property;
150 12 : object_class->finalize = gst_tensor_crop_finalize;
151 :
152 : /**
153 : * GstTensorCrop::lateness:
154 : *
155 : * The time difference between raw and info buffer, in milliseconds (-1 means no synchronization).
156 : * If raw and info buffers on the pads have different timestamp and time-diff is larger than 'lateness',
157 : * tensor-crop will drop old buffer and wait for next buffers.
158 : */
159 12 : g_object_class_install_property (object_class, PROP_LATENESS,
160 : g_param_spec_int ("lateness", "Lateness",
161 : "The time difference between raw and info buffer in milliseconds",
162 : -1, G_MAXINT, DEFAULT_LATENESS,
163 : G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
164 :
165 : /**
166 : * GstTensorCrop::silent:
167 : *
168 : * The flag to enable/disable debugging messages.
169 : */
170 12 : g_object_class_install_property (object_class, PROP_SILENT,
171 : g_param_spec_boolean ("silent", "Silent", "Produce verbose output",
172 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
173 :
174 12 : element_class->change_state =
175 12 : GST_DEBUG_FUNCPTR (gst_tensor_crop_change_state);
176 :
177 12 : gst_element_class_add_pad_template (element_class,
178 : gst_static_pad_template_get (&src_template));
179 12 : gst_element_class_add_pad_template (element_class,
180 : gst_static_pad_template_get (&raw_template));
181 12 : gst_element_class_add_pad_template (element_class,
182 : gst_static_pad_template_get (&info_template));
183 :
184 12 : gst_element_class_set_static_metadata (element_class,
185 : "TensorCrop",
186 : "Filter/Tensor",
187 : "Element to crop the regions of incoming tensor",
188 : "Samsung Electronics Co., Ltd.");
189 12 : }
190 :
191 : /**
192 : * @brief Clear and reset old pad data.
193 : */
194 : static void
195 32 : gst_tensor_crop_pad_reset (GstTensorCropPadData * cpad)
196 : {
197 32 : gst_tensors_config_free (&cpad->config);
198 32 : gst_tensors_config_init (&cpad->config);
199 32 : }
200 :
201 : /**
202 : * @brief Clear and reset old data in tensor_crop.
203 : */
204 : static void
205 16 : gst_tensor_crop_reset (GstTensorCrop * self)
206 : {
207 : GstTensorCropPadData *cpad;
208 : GSList *walk;
209 :
210 16 : if (self->collect) {
211 16 : walk = self->collect->data;
212 :
213 48 : while (walk) {
214 32 : cpad = (GstTensorCropPadData *) walk->data;
215 :
216 32 : gst_tensor_crop_pad_reset (cpad);
217 32 : walk = g_slist_next (walk);
218 : }
219 : }
220 :
221 16 : self->send_stream_start = TRUE;
222 16 : }
223 :
224 : /**
225 : * @brief Initialize tensor_crop element.
226 : */
227 : static void
228 8 : gst_tensor_crop_init (GstTensorCrop * self)
229 : {
230 : /* setup sink pad */
231 8 : self->sinkpad_raw = gst_pad_new_from_static_template (&raw_template, "raw");
232 8 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad_raw);
233 :
234 8 : self->sinkpad_info =
235 8 : gst_pad_new_from_static_template (&info_template, "info");
236 8 : gst_element_add_pad (GST_ELEMENT (self), self->sinkpad_info);
237 :
238 8 : self->collect = gst_collect_pads_new ();
239 8 : gst_collect_pads_set_function (self->collect,
240 8 : GST_DEBUG_FUNCPTR (gst_tensor_crop_collected), self);
241 8 : gst_collect_pads_set_event_function (self->collect,
242 8 : GST_DEBUG_FUNCPTR (gst_tensor_crop_sink_event), self);
243 :
244 8 : gst_collect_pads_add_pad (self->collect, self->sinkpad_raw,
245 : sizeof (GstTensorCropPadData), NULL, TRUE);
246 8 : gst_collect_pads_add_pad (self->collect, self->sinkpad_info,
247 : sizeof (GstTensorCropPadData), NULL, TRUE);
248 :
249 : /* setup src pad */
250 8 : self->srcpad = gst_pad_new_from_static_template (&src_template, "src");
251 8 : gst_pad_set_event_function (self->srcpad,
252 : GST_DEBUG_FUNCPTR (gst_tensor_crop_src_event));
253 8 : gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
254 :
255 : /* init properties */
256 8 : self->lateness = DEFAULT_LATENESS;
257 8 : self->silent = DEFAULT_SILENT;
258 8 : self->send_stream_start = TRUE;
259 8 : }
260 :
261 : /**
262 : * @brief Function to finalize instance.
263 : */
264 : static void
265 8 : gst_tensor_crop_finalize (GObject * object)
266 : {
267 : GstTensorCrop *self;
268 :
269 8 : self = GST_TENSOR_CROP (object);
270 :
271 8 : gst_tensor_crop_reset (self);
272 :
273 8 : if (self->collect) {
274 8 : gst_object_unref (self->collect);
275 8 : self->collect = NULL;
276 : }
277 :
278 8 : G_OBJECT_CLASS (parent_class)->finalize (object);
279 8 : }
280 :
281 : /**
282 : * @brief Setter for tensor_crop properties.
283 : */
284 : static void
285 3 : gst_tensor_crop_set_property (GObject * object, guint prop_id,
286 : const GValue * value, GParamSpec * pspec)
287 : {
288 : GstTensorCrop *self;
289 :
290 3 : self = GST_TENSOR_CROP (object);
291 :
292 3 : switch (prop_id) {
293 2 : case PROP_LATENESS:
294 2 : self->lateness = g_value_get_int (value);
295 2 : break;
296 1 : case PROP_SILENT:
297 1 : self->silent = g_value_get_boolean (value);
298 1 : break;
299 0 : default:
300 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
301 0 : break;
302 : }
303 3 : }
304 :
305 : /**
306 : * @brief Getter for tensor_crop properties.
307 : */
308 : static void
309 4 : gst_tensor_crop_get_property (GObject * object, guint prop_id,
310 : GValue * value, GParamSpec * pspec)
311 : {
312 : GstTensorCrop *self;
313 :
314 4 : self = GST_TENSOR_CROP (object);
315 :
316 4 : switch (prop_id) {
317 2 : case PROP_LATENESS:
318 2 : g_value_set_int (value, self->lateness);
319 2 : break;
320 2 : case PROP_SILENT:
321 2 : g_value_set_boolean (value, self->silent);
322 2 : break;
323 0 : default:
324 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
325 0 : break;
326 : }
327 4 : }
328 :
329 : /**
330 : * @brief Handle state transition.
331 : */
332 : static GstStateChangeReturn
333 62 : gst_tensor_crop_change_state (GstElement * element, GstStateChange transition)
334 : {
335 : GstTensorCrop *self;
336 : GstStateChangeReturn ret;
337 :
338 62 : self = GST_TENSOR_CROP (element);
339 :
340 62 : switch (transition) {
341 8 : case GST_STATE_CHANGE_READY_TO_PAUSED:
342 8 : gst_collect_pads_start (self->collect);
343 8 : break;
344 8 : case GST_STATE_CHANGE_PAUSED_TO_READY:
345 8 : gst_collect_pads_stop (self->collect);
346 8 : break;
347 46 : default:
348 46 : break;
349 : }
350 :
351 62 : ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
352 :
353 62 : switch (transition) {
354 8 : case GST_STATE_CHANGE_PAUSED_TO_READY:
355 8 : gst_tensor_crop_reset (self);
356 8 : break;
357 54 : default:
358 54 : break;
359 : }
360 :
361 62 : return ret;
362 : }
363 :
364 : /**
365 : * @brief Handle event on src pad.
366 : */
367 : static gboolean
368 3 : gst_tensor_crop_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
369 : {
370 3 : g_return_val_if_fail (event != NULL, FALSE);
371 :
372 3 : switch (GST_EVENT_TYPE (event)) {
373 1 : case GST_EVENT_SEEK:
374 : /* disable seeking */
375 1 : gst_event_unref (event);
376 1 : return FALSE;
377 2 : default:
378 2 : break;
379 : }
380 :
381 2 : return gst_pad_event_default (pad, parent, event);
382 : }
383 :
384 : /**
385 : * @brief Handle event on sink pad.
386 : */
387 : static gboolean
388 60 : gst_tensor_crop_sink_event (GstCollectPads * pads, GstCollectData * data,
389 : GstEvent * event, gpointer user_data)
390 : {
391 : GstTensorCropPadData *cpad;
392 : UNUSED (user_data);
393 :
394 60 : g_return_val_if_fail (event != NULL, FALSE);
395 :
396 60 : cpad = (GstTensorCropPadData *) data;
397 :
398 60 : switch (GST_EVENT_TYPE (event)) {
399 15 : case GST_EVENT_CAPS:
400 : {
401 : GstCaps *caps;
402 : gboolean ret;
403 :
404 15 : gst_event_parse_caps (event, &caps);
405 :
406 15 : ret = gst_tensors_config_from_caps (&cpad->config, caps, TRUE);
407 15 : gst_event_unref (event);
408 :
409 15 : return ret;
410 : }
411 45 : default:
412 45 : break;
413 : }
414 :
415 45 : return gst_collect_pads_event_default (pads, data, event, FALSE);
416 : }
417 :
418 : /**
419 : * @brief Set pad caps if not negotiated.
420 : */
421 : static GstFlowReturn
422 10 : gst_tensor_crop_negotiate (GstTensorCrop * self)
423 : {
424 10 : if (!gst_pad_has_current_caps (self->sinkpad_raw)) {
425 0 : GST_ERROR_OBJECT (self,
426 : "The raw pad of tensor_crop '%s' does not have pad caps.",
427 : GST_ELEMENT_NAME (self));
428 0 : return GST_FLOW_NOT_NEGOTIATED;
429 : }
430 :
431 10 : if (!gst_pad_has_current_caps (self->sinkpad_info)) {
432 0 : GST_ERROR_OBJECT (self,
433 : "The info pad of tensor_crop '%s' does not have pad caps.",
434 : GST_ELEMENT_NAME (self));
435 0 : return GST_FLOW_NOT_NEGOTIATED;
436 : }
437 :
438 10 : if (!gst_pad_has_current_caps (self->srcpad)) {
439 : GstCaps *caps;
440 : GstSegment segment;
441 : GstTensorsConfig config;
442 : GstTensorCropPadData *cpad;
443 : GSList *walk;
444 :
445 6 : if (self->send_stream_start) {
446 : /**
447 : * Cannot use gst-pad util to get stream ID (multiple sink pads).
448 : * Create stream ID using first sink pad.
449 : */
450 6 : g_autofree gchar *sink_stream_id = gst_pad_get_stream_id (self->sinkpad_raw);
451 6 : g_autofree gchar *element_name = gst_element_get_name (self);
452 6 : g_autofree gchar *pad_name = gst_pad_get_name (self->srcpad);
453 12 : g_autofree gchar *stream_id = g_strdup_printf ("%s-%s-nnscrop-%s-%08x",
454 6 : GST_STR_NULL (sink_stream_id), element_name, pad_name, g_random_int ());
455 :
456 6 : gst_pad_push_event (self->srcpad, gst_event_new_stream_start (stream_id));
457 6 : self->send_stream_start = FALSE;
458 : }
459 :
460 : /**
461 : * Get config from collect-pads and set framerate.
462 : * Output is always flexible tensor.
463 : */
464 6 : gst_tensors_config_init (&config);
465 6 : config.info.format = _NNS_TENSOR_FORMAT_FLEXIBLE;
466 :
467 6 : walk = self->collect->data;
468 18 : while (walk) {
469 12 : cpad = (GstTensorCropPadData *) walk->data;
470 :
471 12 : if (config.rate_n < 0 ||
472 6 : gst_util_fraction_compare (cpad->config.rate_n, cpad->config.rate_d,
473 : config.rate_n, config.rate_d) < 0) {
474 6 : config.rate_n = cpad->config.rate_n;
475 6 : config.rate_d = cpad->config.rate_d;
476 : }
477 :
478 12 : walk = g_slist_next (walk);
479 : }
480 :
481 6 : caps = gst_tensors_caps_from_config (&config);
482 6 : gst_pad_set_caps (self->srcpad, caps);
483 6 : gst_caps_unref (caps);
484 :
485 6 : gst_segment_init (&segment, GST_FORMAT_TIME);
486 6 : gst_pad_push_event (self->srcpad, gst_event_new_segment (&segment));
487 : }
488 :
489 10 : return GST_FLOW_OK;
490 : }
491 :
492 : /**
493 : * @brief Internal function to prepare output meta info.
494 : */
495 : static gboolean
496 6 : gst_tensor_crop_prepare_out_meta (GstTensorCrop * self, gpointer buffer,
497 : GstTensorMetaInfo * meta, GstTensorInfo * info, gboolean * is_flexible)
498 : {
499 : GstCaps *caps;
500 : GstTensorsConfig config;
501 : GstTensorInfo *_info;
502 6 : gboolean ret = FALSE;
503 :
504 6 : gst_tensor_meta_info_init (meta);
505 6 : gst_tensor_info_init (info);
506 :
507 6 : caps = gst_pad_get_current_caps (self->sinkpad_raw);
508 :
509 6 : if (!gst_tensors_config_from_caps (&config, caps, TRUE)) {
510 0 : GST_ERROR_OBJECT (self, "Failed to get the config from caps.");
511 0 : goto done;
512 : }
513 :
514 : /**
515 : * @note tensor-crop handles single tensor. Parse first one.
516 : */
517 6 : _info = gst_tensors_info_get_nth_info (&config.info, 0);
518 6 : *is_flexible = gst_tensors_config_is_flexible (&config);
519 :
520 6 : if (*is_flexible) {
521 : /* meta from buffer */
522 1 : if (gst_tensor_meta_info_parse_header (meta, buffer)) {
523 1 : ret = gst_tensor_meta_info_convert (meta, info);
524 : }
525 : } else {
526 : /* meta from caps */
527 5 : ret = gst_tensor_info_convert_to_meta (_info, meta);
528 5 : gst_tensor_info_copy (info, _info);
529 : }
530 :
531 : /* output is flex tensor */
532 6 : meta->format = _NNS_TENSOR_FORMAT_FLEXIBLE;
533 :
534 6 : done:
535 6 : gst_caps_unref (caps);
536 6 : gst_tensors_config_free (&config);
537 6 : return ret;
538 : }
539 :
540 : /**
541 : * @brief Internal function to parse buffer and fill crop info.
542 : */
543 : static gboolean
544 7 : gst_tensor_crop_get_crop_info (GstTensorCrop * self, GstBuffer * info,
545 : tensor_crop_info_s * cinfo)
546 : {
547 : GstMemory *mem;
548 : GstMapInfo map;
549 : GstTensorMetaInfo meta;
550 : gsize hsize, dsize, esize;
551 : guint i, j;
552 : guint8 *pos, *src, *desc;
553 7 : gboolean ret = FALSE;
554 :
555 7 : i = gst_tensor_buffer_get_count (info);
556 7 : g_assert (i > 0);
557 7 : if (i > 1) {
558 0 : GST_WARNING_OBJECT (self,
559 : "Info buffer has %u memories, parse first one.", i);
560 : }
561 :
562 7 : mem = gst_tensor_buffer_get_nth_memory (info, 0);
563 7 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
564 0 : GST_ERROR_OBJECT (self, "Failed to map the info buffer.");
565 0 : goto done;
566 : }
567 :
568 : /* parse crop-info from flex tensor */
569 7 : if (!gst_tensor_meta_info_parse_header (&meta, map.data)) {
570 0 : GST_ERROR_OBJECT (self, "Failed to get the meta from info buffer.");
571 0 : goto done;
572 : }
573 :
574 7 : hsize = gst_tensor_meta_info_get_header_size (&meta);
575 7 : dsize = gst_tensor_meta_info_get_data_size (&meta);
576 7 : esize = gst_tensor_get_element_size (meta.type);
577 :
578 7 : if (hsize + dsize != map.size) {
579 1 : GST_ERROR_OBJECT (self,
580 : "Invalid meta info, info buffer size is incorrect (received %zd, expected %zd).",
581 : map.size, hsize + dsize);
582 1 : goto done;
583 : }
584 :
585 : /**
586 : * @todo Add various mode to crop tensor.
587 : * Now tensor-crop handles NHWC data format only.
588 : */
589 6 : g_assert ((dsize % (esize * 4)) == 0);
590 :
591 6 : memset (cinfo, 0, sizeof (tensor_crop_info_s));
592 :
593 6 : cinfo->num = MIN (dsize / (esize * 4), NNS_TENSOR_SIZE_LIMIT);
594 :
595 17 : for (i = 0; i < cinfo->num; i++) {
596 11 : pos = map.data + hsize + (esize * 4 * i);
597 :
598 55 : for (j = 0; j < 4; j++) {
599 44 : src = pos + (esize * j);
600 44 : desc = (guint8 *) (&cinfo->region[i]) + sizeof (guint) * j;
601 :
602 44 : gst_tensor_data_raw_typecast (src, meta.type, desc, _NNS_UINT32);
603 : }
604 : }
605 :
606 6 : ret = TRUE;
607 :
608 7 : done:
609 7 : if (mem) {
610 7 : gst_memory_unmap (mem, &map);
611 7 : gst_memory_unref (mem);
612 : }
613 :
614 7 : return ret;
615 : }
616 :
617 : /**
618 : * @brief Internal function to crop incoming buffer.
619 : */
620 : static GstBuffer *
621 6 : gst_tensor_crop_do_cropping (GstTensorCrop * self, GstBuffer * raw,
622 : tensor_crop_info_s * cinfo)
623 : {
624 6 : GstBuffer *result = NULL;
625 : GstMemory *mem;
626 : GstMapInfo map;
627 : GstTensorMetaInfo meta;
628 : GstTensorInfo info;
629 : gboolean flexible;
630 : gsize hsize, esize, dsize;
631 : guint8 *cropped, *dpos, *desc, *src;
632 : guint i, j, ch, mw, mh, _x, _y, _w, _h;
633 :
634 6 : i = gst_tensor_buffer_get_count (raw);
635 6 : g_assert (i > 0);
636 6 : if (i > 1) {
637 0 : GST_WARNING_OBJECT (self,
638 : "Raw data buffer has %u memories, parse first one.", i);
639 : }
640 :
641 6 : mem = gst_tensor_buffer_get_nth_memory (raw, 0);
642 6 : if (!gst_memory_map (mem, &map, GST_MAP_READ)) {
643 0 : GST_ERROR_OBJECT (self, "Failed to map the raw buffer.");
644 0 : goto done;
645 : }
646 :
647 6 : if (!gst_tensor_crop_prepare_out_meta (self, map.data, &meta,
648 : &info, &flexible)) {
649 0 : GST_ERROR_OBJECT (self, "Failed to get the output meta.");
650 0 : goto done;
651 : }
652 :
653 6 : hsize = flexible ? gst_tensor_meta_info_get_header_size (&meta) : 0;
654 6 : dsize = gst_tensor_meta_info_get_data_size (&meta);
655 6 : dpos = map.data + hsize;
656 6 : if ((hsize + dsize) != map.size) {
657 1 : GST_ERROR_OBJECT (self,
658 : "Raw buffer has invalid data size (received %zd, expected %zd).",
659 : map.size, dsize);
660 1 : goto done;
661 : }
662 :
663 5 : result = gst_buffer_new ();
664 :
665 : /** @todo Add various mode to crop tensor. */
666 5 : ch = info.dimension[0];
667 5 : mw = info.dimension[1];
668 5 : mh = info.dimension[2];
669 5 : esize = gst_tensor_get_element_size (info.type);
670 5 : hsize = gst_tensor_meta_info_get_header_size (&meta);
671 :
672 14 : for (i = 0; i < cinfo->num; i++) {
673 : GstTensorInfo crop_info;
674 : GstMemory *crop_mem;
675 :
676 9 : _x = (cinfo->region[i].x < mw) ? cinfo->region[i].x : mw;
677 9 : _y = (cinfo->region[i].y < mh) ? cinfo->region[i].y : mh;
678 9 : _w = (_x + cinfo->region[i].w - 1 < mw) ? cinfo->region[i].w : (mw - _x);
679 9 : _h = (_y + cinfo->region[i].h - 1 < mh) ? cinfo->region[i].h : (mh - _y);
680 :
681 9 : g_assert (_w > 0 && _h > 0);
682 9 : dsize = hsize + (esize * ch * _w * _h);
683 9 : cropped = (guint8 *) g_malloc0 (dsize);
684 :
685 : /* set header for flex tensor */
686 9 : meta.dimension[1] = _w;
687 9 : meta.dimension[2] = _h;
688 9 : meta.dimension[3] = 1;
689 9 : gst_tensor_meta_info_update_header (&meta, cropped);
690 :
691 232 : for (j = 0; j < _h; j++) {
692 223 : src = dpos + esize * ch * (_x + (j + _y) * mw);
693 223 : desc = cropped + hsize + (esize * ch * _w) * j;
694 223 : memcpy (desc, src, (esize * ch * _w));
695 : }
696 :
697 : /* Prepare tensors-info to append memory into gst-buffer. */
698 9 : gst_tensor_meta_info_convert (&meta, &crop_info);
699 : crop_mem =
700 9 : gst_memory_new_wrapped (0, cropped, dsize, 0, dsize, cropped, g_free);
701 :
702 9 : gst_tensor_buffer_append_memory (result, crop_mem, &crop_info);
703 9 : gst_tensor_info_free (&crop_info);
704 : }
705 :
706 : /* set timestamp from raw buffer */
707 5 : gst_buffer_copy_into (result, raw, GST_BUFFER_COPY_METADATA, 0, -1);
708 :
709 6 : done:
710 6 : if (mem) {
711 6 : gst_memory_unmap (mem, &map);
712 6 : gst_memory_unref (mem);
713 : }
714 :
715 6 : return result;
716 : }
717 :
718 : /**
719 : * @brief Internal function to transform the input buffer.
720 : */
721 : static GstFlowReturn
722 9 : gst_tensor_crop_chain (GstTensorCrop * self,
723 : GstCollectData * data_raw, GstCollectData * data_info)
724 : {
725 : GstFlowReturn ret;
726 : GstBuffer *buf_raw, *buf_info, *result;
727 : GstTensorCropPadData *cpad;
728 : tensor_crop_info_s cinfo;
729 : gboolean drop_raw, drop_info;
730 :
731 18 : g_return_val_if_fail (data_raw && data_info, GST_FLOW_ERROR);
732 :
733 9 : buf_raw = gst_collect_pads_peek (self->collect, data_raw);
734 9 : buf_info = gst_collect_pads_peek (self->collect, data_info);
735 9 : drop_raw = (buf_raw != NULL);
736 9 : drop_info = (buf_info != NULL);
737 :
738 9 : if (!buf_raw || !buf_info) {
739 0 : ret = GST_FLOW_EOS;
740 0 : goto done;
741 : }
742 :
743 9 : cpad = (GstTensorCropPadData *) data_raw;
744 9 : buf_raw = gst_tensor_buffer_from_config (buf_raw, &cpad->config);
745 9 : cpad = (GstTensorCropPadData *) data_info;
746 9 : buf_info = gst_tensor_buffer_from_config (buf_info, &cpad->config);
747 :
748 : /**
749 : * The case when raw and info have different timestamp.
750 : * Compare timestamp and if time diff is less than lateness, crop raw buffer.
751 : */
752 9 : if (self->lateness >= 0) {
753 : GstClockTime ts_raw, ts_info, lateness;
754 :
755 4 : ts_raw = GST_BUFFER_TIMESTAMP (buf_raw);
756 4 : ts_info = GST_BUFFER_TIMESTAMP (buf_info);
757 4 : lateness = self->lateness * GST_MSECOND;
758 :
759 4 : if (GST_CLOCK_TIME_IS_VALID (ts_raw) && GST_CLOCK_TIME_IS_VALID (ts_info)) {
760 4 : if (((GstClockTime) ABS (GST_CLOCK_DIFF (ts_raw, ts_info))) > lateness) {
761 2 : GST_DEBUG_OBJECT (self, "Drop old buffer and wait for next.");
762 2 : GST_DEBUG_OBJECT (self, "Raw buffer ts: %" GST_TIME_FORMAT,
763 : GST_TIME_ARGS (ts_raw));
764 2 : GST_DEBUG_OBJECT (self, "Info buffer ts: %" GST_TIME_FORMAT,
765 : GST_TIME_ARGS (ts_info));
766 :
767 : /* clear old buffer and return ok to get next buffer */
768 2 : if (ts_raw > ts_info)
769 1 : drop_raw = FALSE;
770 : else
771 1 : drop_info = FALSE;
772 :
773 2 : ret = GST_FLOW_OK;
774 2 : goto done;
775 : }
776 : } else {
777 0 : GST_WARNING_OBJECT (self,
778 : "Incoming buffer has invalid timestamp, continue cropping data.");
779 : }
780 : }
781 :
782 7 : if (!gst_tensor_crop_get_crop_info (self, buf_info, &cinfo)) {
783 1 : ret = GST_FLOW_ERROR;
784 1 : goto done;
785 : }
786 :
787 6 : result = gst_tensor_crop_do_cropping (self, buf_raw, &cinfo);
788 6 : ret = gst_pad_push (self->srcpad, result);
789 :
790 9 : done:
791 9 : if (buf_raw)
792 9 : gst_buffer_unref (buf_raw);
793 9 : if (buf_info)
794 9 : gst_buffer_unref (buf_info);
795 :
796 : /* clear buffer in collect pads */
797 9 : if (drop_raw)
798 8 : gst_buffer_unref (gst_collect_pads_pop (self->collect, data_raw));
799 9 : if (drop_info)
800 8 : gst_buffer_unref (gst_collect_pads_pop (self->collect, data_info));
801 :
802 9 : return ret;
803 : }
804 :
805 : /**
806 : * @brief Chain function called when the buffer is available on all of the collect pads.
807 : */
808 : static GstFlowReturn
809 10 : gst_tensor_crop_collected (GstCollectPads * pads, gpointer user_data)
810 : {
811 : GstTensorCrop *self;
812 : GstCollectData *data_raw, *data_info;
813 : GSList *walk;
814 : GstFlowReturn ret;
815 :
816 10 : self = GST_TENSOR_CROP (user_data);
817 10 : data_raw = data_info = NULL;
818 :
819 10 : ret = gst_tensor_crop_negotiate (self);
820 10 : if (ret != GST_FLOW_OK)
821 0 : return ret;
822 :
823 28 : for (walk = pads->data; walk; walk = g_slist_next (walk)) {
824 : GstCollectData *data;
825 :
826 19 : data = (GstCollectData *) walk->data;
827 :
828 19 : if (GST_COLLECT_PADS_STATE_IS_SET (data, GST_COLLECT_PADS_STATE_EOS)) {
829 1 : gst_pad_push_event (self->srcpad, gst_event_new_eos ());
830 1 : return GST_FLOW_EOS;
831 : }
832 :
833 18 : if (data->pad == self->sinkpad_raw) {
834 9 : data_raw = data;
835 9 : } else if (data->pad == self->sinkpad_info) {
836 9 : data_info = data;
837 : }
838 : }
839 :
840 9 : return gst_tensor_crop_chain (self, data_raw, data_info);
841 : }
|
