Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer/NNStreamer Tensor-Rate
4 : * Copyright (C) 2020 Dongju Chae <dongju.chae@samsung.com>
5 : */
6 :
7 : /**
8 : * @file gsttensor_rate.c
9 : * @date 24 Sep 2020
10 : * @brief GStreamer plugin to adjust tensor rate
11 : * @see https://github.com/nnstreamer/nnstreamer
12 : * @author Dongju Chae <dongju.chae@samsung.com>
13 : * @bug No known bugs except for NYI items
14 : */
15 :
16 : /**
17 : * SECTION:element-tensor_rate
18 : *
19 : * This element controls a frame rate of tensor streams in the pipeline.
20 : *
21 : * Basically, this element takes an incoming stream of tensor frames, and
22 : * produces an adjusted stream that matches the source pad's framerate.
23 : * The adjustment is performed by dropping and duplicating tensor frames.
24 : * By default the element will simply negotiate the same framerate on its
25 : * source and sink pad.
26 : *
27 : * Also, when 'throttle' property is set, it propagates a specified frame-rate
28 : * to upstream elements by sending qos events, which prevents unnecessary
29 : * data from upstream elements.
30 : *
31 : * <refsect2>
32 : * <title>Example launch line with tensor rate</title>
33 : * gst-launch-1.0 videotestsrc
34 : * ! video/x-raw,width=640,height=480,framerate=15/1
35 : * ! tensor_converter
36 : * ! tensor_rate framerate=10/1 throttle=true
37 : * ! tensor_decoder mode=direct_video
38 : * ! videoconvert
39 : * ! autovideosink
40 : * </refsect2>
41 : */
42 :
43 : #ifdef HAVE_CONFIG_H
44 : #include "config.h"
45 : #endif
46 :
47 : #include <nnstreamer_log.h>
48 : #include <nnstreamer_util.h>
49 :
50 : #include "gsttensor_rate.h"
51 :
52 : /**
53 : * @brief Macro for debug mode.
54 : */
55 : #ifndef DBG
56 : #define DBG (!self->silent)
57 : #endif
58 :
59 : #ifndef ABSDIFF
60 : #define ABSDIFF(a, b) (((a) > (b)) ? (a) - (b) : (b) - (a))
61 : #endif
62 :
63 : GST_DEBUG_CATEGORY_STATIC (gst_tensor_rate_debug);
64 : #define GST_CAT_DEFAULT gst_tensor_rate_debug
65 :
66 : #define CAPS_STRING GST_TENSOR_CAP_DEFAULT "; " GST_TENSORS_CAP_DEFAULT
67 :
68 : #define GST_TENSOR_RATE_SCALED_TIME(self, count)\
69 : gst_util_uint64_scale (count,\
70 : self->to_rate_denominator * GST_SECOND, self->to_rate_numerator)
71 :
72 : /** @brief default parameters */
73 : #define DEFAULT_SILENT TRUE
74 : #define DEFAULT_THROTTLE TRUE
75 :
76 : /**
77 : * @brief tensor_rate properties
78 : */
79 : enum
80 : {
81 : PROP_0,
82 : PROP_IN,
83 : PROP_OUT,
84 : PROP_DUP,
85 : PROP_DROP,
86 : PROP_SILENT,
87 : PROP_THROTTLE,
88 : PROP_FRAMERATE,
89 : };
90 :
91 : /**
92 : * @brief The capabilities of the inputs
93 : */
94 : static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink",
95 : GST_PAD_SINK,
96 : GST_PAD_ALWAYS,
97 : GST_STATIC_CAPS (CAPS_STRING));
98 :
99 : /**
100 : * @brief The capabilities of the outputs
101 : */
102 : static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
103 : GST_PAD_SRC,
104 : GST_PAD_ALWAYS,
105 : GST_STATIC_CAPS (CAPS_STRING));
106 :
107 : static GParamSpec *pspec_drop = NULL;
108 : static GParamSpec *pspec_duplicate = NULL;
109 :
110 : #define gst_tensor_rate_parent_class parent_class
111 3012 : G_DEFINE_TYPE (GstTensorRate, gst_tensor_rate, GST_TYPE_BASE_TRANSFORM);
112 :
113 : /* GObject vmethod implementations */
114 : static void gst_tensor_rate_set_property (GObject * object, guint prop_id,
115 : const GValue * value, GParamSpec * pspec);
116 : static void gst_tensor_rate_get_property (GObject * object, guint prop_id,
117 : GValue * value, GParamSpec * pspec);
118 : static void gst_tensor_rate_finalize (GObject * object);
119 :
120 : /* GstBaseTransform vmethod implementations */
121 : static GstFlowReturn gst_tensor_rate_transform_ip (GstBaseTransform * trans,
122 : GstBuffer * buffer);
123 : static GstCaps *gst_tensor_rate_transform_caps (GstBaseTransform * trans,
124 : GstPadDirection direction, GstCaps * caps, GstCaps * _rate);
125 : static GstCaps *gst_tensor_rate_fixate_caps (GstBaseTransform * trans,
126 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
127 : static gboolean gst_tensor_rate_set_caps (GstBaseTransform * trans,
128 : GstCaps * incaps, GstCaps * outcaps);
129 : static void gst_tensor_rate_swap_prev (GstTensorRate * self,
130 : GstBuffer * buffer, gint64 time);
131 : static GstFlowReturn gst_tensor_rate_flush_prev (GstTensorRate * self,
132 : gboolean duplicate, GstClockTime next_intime);
133 :
134 : static void gst_tensor_rate_notify_drop (GstTensorRate * self);
135 : static void gst_tensor_rate_notify_duplicate (GstTensorRate * self);
136 :
137 : static gboolean gst_tensor_rate_start (GstBaseTransform * trans);
138 : static gboolean gst_tensor_rate_stop (GstBaseTransform * trans);
139 : static gboolean gst_tensor_rate_sink_event (GstBaseTransform * trans,
140 : GstEvent * event);
141 :
142 : static void gst_tensor_rate_install_properties (GObjectClass * gobject_class);
143 :
144 : /**
145 : * @brief initialize the tensor_rate's class (GST Standard)
146 : */
147 : static void
148 5 : gst_tensor_rate_class_init (GstTensorRateClass * klass)
149 : {
150 : GObjectClass *gobject_class;
151 : GstElementClass *gstelement_class;
152 : GstBaseTransformClass *trans_class;
153 :
154 5 : GST_DEBUG_CATEGORY_INIT (gst_tensor_rate_debug, "tensor_rate", 0,
155 : "Tensor Rate to control streams based on tensor(s) values");
156 :
157 5 : trans_class = (GstBaseTransformClass *) klass;
158 5 : gstelement_class = (GstElementClass *) trans_class;
159 5 : gobject_class = (GObjectClass *) gstelement_class;
160 :
161 5 : gobject_class->set_property = gst_tensor_rate_set_property;
162 5 : gobject_class->get_property = gst_tensor_rate_get_property;
163 5 : gobject_class->finalize = gst_tensor_rate_finalize;
164 :
165 5 : gst_tensor_rate_install_properties (gobject_class);
166 :
167 5 : gst_element_class_set_details_simple (gstelement_class,
168 : "TensorRate",
169 : "Filter/Tensor",
170 : "Adjusts a framerate of incoming tensors",
171 : "Dongju Chae <dongju.chae@samsung.com>");
172 :
173 5 : gst_element_class_add_pad_template (gstelement_class,
174 : gst_static_pad_template_get (&src_factory));
175 5 : gst_element_class_add_pad_template (gstelement_class,
176 : gst_static_pad_template_get (&sink_factory));
177 :
178 5 : trans_class->passthrough_on_same_caps = TRUE;
179 5 : trans_class->transform_ip_on_passthrough = TRUE;
180 :
181 : /* Processing units */
182 5 : trans_class->transform_ip = GST_DEBUG_FUNCPTR (gst_tensor_rate_transform_ip);
183 :
184 : /* Negotiation units */
185 5 : trans_class->transform_caps =
186 5 : GST_DEBUG_FUNCPTR (gst_tensor_rate_transform_caps);
187 5 : trans_class->fixate_caps = GST_DEBUG_FUNCPTR (gst_tensor_rate_fixate_caps);
188 5 : trans_class->set_caps = GST_DEBUG_FUNCPTR (gst_tensor_rate_set_caps);
189 :
190 : /* setup sink event */
191 5 : trans_class->sink_event = GST_DEBUG_FUNCPTR (gst_tensor_rate_sink_event);
192 :
193 : /* start/stop to call open/close */
194 5 : trans_class->start = GST_DEBUG_FUNCPTR (gst_tensor_rate_start);
195 5 : trans_class->stop = GST_DEBUG_FUNCPTR (gst_tensor_rate_stop);
196 5 : }
197 :
198 : /**
199 : * @brief push the buffer to src pad
200 : */
201 : static GstFlowReturn
202 1531 : gst_tensor_rate_push_buffer (GstTensorRate * self, GstBuffer * outbuf,
203 : gboolean duplicate, GstClockTime next_intime)
204 : {
205 : GstFlowReturn res;
206 : GstClockTime push_ts;
207 : UNUSED (next_intime);
208 :
209 1531 : GST_BUFFER_OFFSET (outbuf) = self->out;
210 1531 : GST_BUFFER_OFFSET_END (outbuf) = self->out + 1;
211 1531 : GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_DISCONT);
212 :
213 1531 : if (duplicate)
214 496 : GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_GAP);
215 : else
216 1035 : GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_GAP);
217 :
218 : /* this is the timestamp we put on the buffer */
219 1531 : push_ts = self->next_ts;
220 :
221 1531 : self->out++;
222 1531 : self->out_frame_count++;
223 :
224 1531 : if (self->to_rate_numerator) {
225 : GstClockTimeDiff duration;
226 :
227 1531 : duration = GST_TENSOR_RATE_SCALED_TIME (self, self->out_frame_count);
228 :
229 : /* interpolate next expected timestamp in the segment */
230 1531 : self->next_ts = self->segment.base + self->segment.start +
231 1531 : self->base_ts + duration;
232 :
233 1531 : GST_BUFFER_DURATION (outbuf) = self->next_ts - push_ts;
234 : } else {
235 : /** There must always be a valid duration on prevbuf if rate > 0,
236 : * it is ensured in the transform_ip function */
237 0 : g_assert (GST_BUFFER_PTS_IS_VALID (outbuf));
238 0 : g_assert (GST_BUFFER_DURATION_IS_VALID (outbuf));
239 0 : g_assert (GST_BUFFER_DURATION (outbuf) != 0);
240 :
241 0 : self->next_ts = GST_BUFFER_PTS (outbuf) + GST_BUFFER_DURATION (outbuf);
242 : }
243 :
244 : /* adapt for looping, bring back to time in current segment. */
245 1531 : GST_BUFFER_TIMESTAMP (outbuf) = push_ts - self->segment.base;
246 :
247 1531 : silent_debug (self, "old is best, dup, pushing buffer outgoing ts %"
248 : GST_TIME_FORMAT, GST_TIME_ARGS (push_ts));
249 :
250 1531 : res = gst_pad_push (GST_BASE_TRANSFORM_SRC_PAD (self), outbuf);
251 :
252 1531 : return res;
253 : }
254 :
255 : /**
256 : * @brief flush the oldest buffer
257 : */
258 : static GstFlowReturn
259 1531 : gst_tensor_rate_flush_prev (GstTensorRate * self, gboolean duplicate,
260 : GstClockTime next_intime)
261 : {
262 : GstBuffer *outbuf;
263 :
264 1531 : if (!self->prevbuf) {
265 0 : ml_logi ("got EOS before any buffer was received");
266 0 : return GST_FLOW_OK;
267 : }
268 :
269 1531 : outbuf = gst_buffer_ref (self->prevbuf);
270 : /* make sure we can write to the metadata */
271 1531 : outbuf = gst_buffer_make_writable (outbuf);
272 :
273 1531 : return gst_tensor_rate_push_buffer (self, outbuf, duplicate, next_intime);
274 : }
275 :
276 : /**
277 : * @brief swap a previous buffer
278 : */
279 : static void
280 1684 : gst_tensor_rate_swap_prev (GstTensorRate * self, GstBuffer * buffer,
281 : gint64 time)
282 : {
283 1684 : silent_debug (self, "swap_prev: storing buffer %p in prev", buffer);
284 :
285 1684 : if (self->prevbuf)
286 1652 : gst_buffer_unref (self->prevbuf);
287 1684 : self->prevbuf = buffer != NULL ? gst_buffer_ref (buffer) : NULL;
288 1684 : self->prev_ts = time;
289 1684 : }
290 :
291 : /**
292 : * @brief reset variables of the element (GST Standard)
293 : */
294 : static void
295 25 : gst_tensor_rate_reset (GstTensorRate * self)
296 : {
297 25 : self->in = 0;
298 25 : self->out = 0;
299 25 : self->drop = 0;
300 25 : self->dup = 0;
301 :
302 25 : self->out_frame_count = 0;
303 :
304 25 : self->base_ts = 0;
305 25 : self->next_ts = GST_CLOCK_TIME_NONE;
306 25 : self->last_ts = GST_CLOCK_TIME_NONE;
307 :
308 25 : self->sent_qos_on_passthrough = FALSE;
309 :
310 25 : gst_tensor_rate_swap_prev (self, NULL, 0);
311 25 : }
312 :
313 : /**
314 : * @brief initialize the new element (GST Standard)
315 : */
316 : static void
317 11 : gst_tensor_rate_init (GstTensorRate * self)
318 : {
319 11 : gst_tensor_rate_reset (self);
320 :
321 11 : self->silent = DEFAULT_SILENT;
322 11 : self->throttle = DEFAULT_THROTTLE;
323 :
324 : /* decided from caps negotiation */
325 11 : self->from_rate_numerator = 0;
326 11 : self->from_rate_denominator = 0;
327 11 : self->to_rate_numerator = 0;
328 11 : self->to_rate_denominator = 0;
329 :
330 : /* specified from property */
331 11 : self->rate_n = -1;
332 11 : self->rate_d = -1;
333 :
334 11 : gst_segment_init (&self->segment, GST_FORMAT_TIME);
335 11 : }
336 :
337 : /**
338 : * @brief Function to finalize instance. (GST Standard)
339 : */
340 : static void
341 11 : gst_tensor_rate_finalize (GObject * object)
342 : {
343 11 : G_OBJECT_CLASS (parent_class)->finalize (object);
344 11 : }
345 :
346 : /**
347 : * @brief Setter for tensor_rate properties.
348 : */
349 : static void
350 34 : gst_tensor_rate_set_property (GObject * object, guint prop_id,
351 : const GValue * value, GParamSpec * pspec)
352 : {
353 34 : GstTensorRate *self = GST_TENSOR_RATE (object);
354 :
355 34 : GST_OBJECT_LOCK (self);
356 :
357 34 : switch (prop_id) {
358 8 : case PROP_SILENT:
359 8 : self->silent = g_value_get_boolean (value);
360 8 : break;
361 12 : case PROP_THROTTLE:
362 12 : self->throttle = g_value_get_boolean (value);
363 12 : break;
364 14 : case PROP_FRAMERATE:
365 : {
366 14 : const gchar *str = g_value_get_string (value);
367 14 : gchar **strv = g_strsplit (str, "/", -1);
368 : gint rate_n, rate_d;
369 :
370 14 : if (g_strv_length (strv) != 2) {
371 1 : ml_loge ("Please specify a proper 'framerate' property");
372 1 : goto done;
373 : }
374 :
375 13 : rate_n = (gint) g_ascii_strtoll (strv[0], NULL, 10);
376 13 : if (errno == ERANGE || rate_n < 0) {
377 0 : ml_loge ("Invalid frame rate numerator in 'framerate'");
378 0 : goto done;
379 : }
380 :
381 13 : rate_d = (gint) g_ascii_strtoll (strv[1], NULL, 10);
382 13 : if (errno == ERANGE || rate_d <= 0) {
383 1 : ml_loge ("Invalid frame rate denominator in 'framerate'");
384 1 : goto done;
385 : }
386 :
387 12 : self->rate_n = rate_n;
388 12 : self->rate_d = rate_d;
389 :
390 14 : done:
391 14 : g_strfreev (strv);
392 14 : break;
393 : }
394 :
395 0 : default:
396 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
397 0 : break;
398 : }
399 :
400 34 : GST_OBJECT_UNLOCK (self);
401 34 : }
402 :
403 : /**
404 : * @brief Getter for tensor_rate properties.
405 : */
406 : static void
407 28 : gst_tensor_rate_get_property (GObject * object, guint prop_id,
408 : GValue * value, GParamSpec * pspec)
409 : {
410 28 : GstTensorRate *self = GST_TENSOR_RATE (object);
411 :
412 28 : GST_OBJECT_LOCK (self);
413 :
414 28 : switch (prop_id) {
415 5 : case PROP_IN:
416 5 : g_value_set_uint64 (value, self->in);
417 5 : break;
418 5 : case PROP_OUT:
419 5 : g_value_set_uint64 (value, self->out);
420 5 : break;
421 5 : case PROP_DROP:
422 5 : g_value_set_uint64 (value, self->drop);
423 5 : break;
424 5 : case PROP_DUP:
425 5 : g_value_set_uint64 (value, self->dup);
426 5 : break;
427 2 : case PROP_SILENT:
428 2 : g_value_set_boolean (value, self->silent);
429 2 : break;
430 2 : case PROP_THROTTLE:
431 2 : g_value_set_boolean (value, self->throttle);
432 2 : break;
433 4 : case PROP_FRAMERATE:
434 4 : if (self->rate_n < 0 || self->rate_d <= 0) {
435 0 : g_value_set_string (value, "");
436 : } else {
437 4 : gchar *str = g_strdup_printf ("%d/%d", self->rate_n, self->rate_d);
438 4 : g_value_take_string (value, str);
439 : }
440 4 : break;
441 0 : default:
442 0 : G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
443 0 : break;
444 : }
445 :
446 28 : GST_OBJECT_UNLOCK (self);
447 28 : }
448 :
449 : #define THROTTLE_DELAY_RATIO (0.999)
450 :
451 : /**
452 : * @brief send throttling qos event to upstream elements
453 : */
454 : static void
455 618 : gst_tensor_rate_send_qos_throttle (GstTensorRate * self, GstClockTime timestamp)
456 : {
457 618 : GstPad *sinkpad = GST_BASE_TRANSFORM_SINK_PAD (&self->element);
458 : GstClockTimeDiff delay;
459 : GstEvent *event;
460 :
461 618 : delay = GST_TENSOR_RATE_SCALED_TIME (self, 1);
462 618 : delay = (GstClockTimeDiff) (((gdouble) delay) * THROTTLE_DELAY_RATIO);
463 :
464 618 : event = gst_event_new_qos (GST_QOS_TYPE_THROTTLE,
465 : 0.9 /** unused */ , delay, timestamp);
466 :
467 618 : silent_debug (self, "Send throttling event with delay: %" GST_TIME_FORMAT,
468 : GST_TIME_ARGS (delay));
469 :
470 618 : gst_pad_push_event (sinkpad, event);
471 618 : }
472 :
473 : /**
474 : * @brief in-place transform
475 : */
476 : static GstFlowReturn
477 1952 : gst_tensor_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer)
478 : {
479 1952 : GstTensorRate *self = GST_TENSOR_RATE (trans);
480 1952 : GstFlowReturn res = GST_BASE_TRANSFORM_FLOW_DROPPED;
481 : GstClockTime intime, in_ts, in_dur;
482 :
483 : /* make sure the denominators are not 0 */
484 1952 : if (self->from_rate_denominator == 0 || self->to_rate_denominator == 0) {
485 0 : ml_loge ("No framerate negotiated");
486 0 : return GST_FLOW_NOT_NEGOTIATED;
487 : }
488 :
489 : /* tensor streams do not support reverse playback */
490 1952 : if (G_UNLIKELY (self->segment.rate < 0.0)) {
491 0 : ml_loge ("Unsupported reverse playback\n");
492 0 : return GST_FLOW_ERROR;
493 : }
494 :
495 1952 : in_ts = GST_BUFFER_TIMESTAMP (buffer);
496 1952 : in_dur = GST_BUFFER_DURATION (buffer);
497 :
498 1952 : if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (in_ts))) {
499 0 : in_ts = self->last_ts;
500 0 : if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (in_ts))) {
501 0 : ml_logw ("Discard an invalid buffer");
502 0 : return GST_BASE_TRANSFORM_FLOW_DROPPED;
503 : }
504 : }
505 :
506 1952 : self->in++;
507 :
508 : /* update the last timestamp */
509 1952 : self->last_ts = in_ts;
510 1952 : if (GST_CLOCK_TIME_IS_VALID (in_dur))
511 1952 : self->last_ts += in_dur;
512 :
513 1952 : silent_debug (self, "got buffer with timestamp %" GST_TIME_FORMAT,
514 : GST_TIME_ARGS (in_ts));
515 :
516 1952 : intime = in_ts + self->segment.base;
517 :
518 : /* let's send a QoS event even if pass-through is used on the same caps */
519 1952 : if (gst_base_transform_is_passthrough (trans)) {
520 300 : if (!self->sent_qos_on_passthrough) {
521 1 : self->sent_qos_on_passthrough = TRUE;
522 1 : gst_tensor_rate_send_qos_throttle (self, intime);
523 : }
524 :
525 300 : self->out++;
526 300 : return GST_FLOW_OK;
527 : }
528 :
529 : /* we need to have two buffers to compare */
530 1652 : if (self->prevbuf == NULL) {
531 6 : gst_tensor_rate_swap_prev (self, buffer, intime);
532 6 : if (!GST_CLOCK_TIME_IS_VALID (self->next_ts)) {
533 6 : self->next_ts = intime;
534 6 : self->base_ts = in_ts - self->segment.start;
535 6 : self->out_frame_count = 0;
536 : }
537 : } else {
538 : GstClockTime prevtime;
539 1646 : gint64 diff1 = 0, diff2 = 0;
540 1646 : guint count = 0;
541 :
542 1646 : prevtime = self->prev_ts;
543 :
544 1646 : silent_debug (self, "BEGINNING prev buf %" GST_TIME_FORMAT " new buf %"
545 : GST_TIME_FORMAT " outgoing ts %" GST_TIME_FORMAT,
546 : GST_TIME_ARGS (prevtime), GST_TIME_ARGS (intime),
547 : GST_TIME_ARGS (self->next_ts));
548 :
549 : /* drop new buffer if it's before previous one */
550 1646 : if (intime < prevtime) {
551 0 : silent_debug (self, "The new buffer (%" GST_TIME_FORMAT ") is before "
552 : "the previous buffer (%" GST_TIME_FORMAT
553 : "). Dropping new buffer.", GST_TIME_ARGS (intime),
554 : GST_TIME_ARGS (prevtime));
555 0 : self->drop++;
556 :
557 0 : if (!self->silent)
558 0 : gst_tensor_rate_notify_drop (self);
559 :
560 0 : return GST_BASE_TRANSFORM_FLOW_DROPPED;
561 : }
562 :
563 : /* got 2 buffers, see which one is the best */
564 : do {
565 : GstClockTime next_ts;
566 :
567 : /* Make sure that we have a duration for previous buffer */
568 3022 : if (!GST_BUFFER_DURATION_IS_VALID (self->prevbuf))
569 0 : GST_BUFFER_DURATION (self->prevbuf) =
570 0 : intime > prevtime ? intime - prevtime : 0;
571 :
572 3022 : next_ts = self->base_ts + (self->next_ts - self->base_ts);
573 :
574 3022 : diff1 = ABSDIFF (prevtime, next_ts);
575 3022 : diff2 = ABSDIFF (intime, next_ts);
576 :
577 3022 : silent_debug (self, "diff with prev %" GST_TIME_FORMAT
578 : " diff with new %" GST_TIME_FORMAT " outgoing ts %"
579 : GST_TIME_FORMAT, GST_TIME_ARGS (diff1),
580 : GST_TIME_ARGS (diff2), GST_TIME_ARGS (next_ts));
581 :
582 : /* output first one when its the best */
583 3022 : if (diff1 <= diff2) {
584 : GstFlowReturn r;
585 1525 : count++;
586 :
587 : /* on error the _flush function posted a warning already */
588 1525 : if ((r = gst_tensor_rate_flush_prev (self,
589 : count > 1, intime)) != GST_FLOW_OK) {
590 0 : return r;
591 : }
592 : }
593 :
594 : /**
595 : * continue while the first one was the best, if they were equal avoid
596 : * going into an infinite loop
597 : */
598 3022 : } while (diff1 < diff2);
599 :
600 : /* if we outputted the first buffer more then once, we have dups */
601 1646 : if (count > 1) {
602 397 : self->dup += count - 1;
603 397 : if (!self->silent)
604 0 : gst_tensor_rate_notify_duplicate (self);
605 : }
606 : /* if we didn't output the first buffer, we have a drop */
607 1249 : else if (count == 0) {
608 617 : self->drop++;
609 :
610 617 : if (!self->silent)
611 0 : gst_tensor_rate_notify_drop (self);
612 :
613 617 : gst_tensor_rate_send_qos_throttle (self, intime);
614 : }
615 :
616 : /* swap in new one when it's the best */
617 1646 : gst_tensor_rate_swap_prev (self, buffer, intime);
618 : }
619 :
620 1652 : return res;
621 : }
622 :
623 : /**
624 : * @brief configure tensor-srcpad cap from "proposed" cap. (GST Standard)
625 : *
626 : * @trans ("this" pointer)
627 : * @direction (why do we need this?)
628 : * @caps sinkpad cap (if direction GST_PAD_SINK)
629 : * @filter this element's cap (don't know specifically.)
630 : *
631 : * Be careful not to fix/set caps at this stage. Negotiation not completed yet.
632 : */
633 : static GstCaps *
634 80 : gst_tensor_rate_transform_caps (GstBaseTransform * trans,
635 : GstPadDirection direction, GstCaps * caps, GstCaps * filter)
636 : {
637 80 : GstTensorRate *self = GST_TENSOR_RATE (trans);
638 80 : GstCaps *result = gst_caps_new_empty ();
639 : guint i;
640 :
641 80 : silent_debug (self, "Direction = %d\n", direction);
642 80 : silent_debug_caps (self, caps, "from");
643 80 : silent_debug_caps (self, filter, "filter");
644 :
645 225 : for (i = 0; i < gst_caps_get_size (caps); i++) {
646 145 : GstStructure *s, *const_s = gst_caps_get_structure (caps, i);
647 :
648 145 : s = gst_structure_copy (const_s);
649 :
650 : /* when a target framerate is specified */
651 145 : if (direction == GST_PAD_SINK && self->rate_n >= 0 && self->rate_d > 0) {
652 61 : gst_structure_set (s, "framerate", GST_TYPE_FRACTION,
653 : self->rate_n, self->rate_d, NULL);
654 : } else {
655 84 : gst_structure_set (s, "framerate", GST_TYPE_FRACTION_RANGE,
656 : 0, 1, G_MAXINT, 1, NULL);
657 : }
658 :
659 145 : result = gst_caps_merge_structure_full (result, s,
660 145 : gst_caps_features_copy (gst_caps_get_features (caps, i)));
661 : }
662 :
663 80 : if (filter && gst_caps_get_size (filter) > 0) {
664 : GstCaps *intersection =
665 9 : gst_caps_intersect_full (filter, result, GST_CAPS_INTERSECT_FIRST);
666 :
667 9 : gst_caps_unref (result);
668 9 : result = intersection;
669 : }
670 :
671 80 : silent_debug_caps (self, result, "to");
672 :
673 80 : return result;
674 : }
675 :
676 : /**
677 : * @brief fixate caps. required vmethod of GstBaseTransform.
678 : */
679 : static GstCaps *
680 7 : gst_tensor_rate_fixate_caps (GstBaseTransform * trans,
681 : GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
682 : {
683 : GstStructure *s;
684 : gint num, denom;
685 : UNUSED (trans);
686 : UNUSED (direction);
687 :
688 7 : s = gst_caps_get_structure (caps, 0);
689 7 : if (G_UNLIKELY (!gst_structure_get_fraction (s, "framerate", &num, &denom)))
690 7 : return othercaps;
691 :
692 7 : othercaps = gst_caps_truncate (othercaps);
693 7 : othercaps = gst_caps_make_writable (othercaps);
694 :
695 7 : s = gst_caps_get_structure (othercaps, 0);
696 7 : gst_structure_fixate_field_nearest_fraction (s, "framerate", num, denom);
697 :
698 7 : return gst_caps_fixate (othercaps);
699 : }
700 :
701 : /**
702 : * @brief set caps. required vmethod of GstBaseTransform.
703 : */
704 : static gboolean
705 7 : gst_tensor_rate_set_caps (GstBaseTransform * trans,
706 : GstCaps * in_caps, GstCaps * out_caps)
707 : {
708 7 : GstTensorRate *self = GST_TENSOR_RATE (trans);
709 : GstStructure *structure;
710 : gint rate_numerator, rate_denominator;
711 :
712 7 : silent_debug (self,
713 : "setcaps called in: %" GST_PTR_FORMAT " out: %" GST_PTR_FORMAT, in_caps,
714 : out_caps);
715 :
716 7 : structure = gst_caps_get_structure (in_caps, 0);
717 :
718 7 : if (!gst_structure_get_fraction (structure, "framerate",
719 : &rate_numerator, &rate_denominator))
720 0 : goto no_framerate;
721 :
722 7 : self->from_rate_numerator = rate_numerator;
723 7 : self->from_rate_denominator = rate_denominator;
724 :
725 7 : structure = gst_caps_get_structure (out_caps, 0);
726 :
727 7 : if (!gst_structure_get_fraction (structure, "framerate",
728 : &rate_numerator, &rate_denominator))
729 0 : goto no_framerate;
730 :
731 7 : if (self->to_rate_numerator)
732 0 : self->base_ts += GST_TENSOR_RATE_SCALED_TIME (self, self->out_frame_count);
733 :
734 7 : self->out_frame_count = 0;
735 7 : self->to_rate_numerator = rate_numerator;
736 7 : self->to_rate_denominator = rate_denominator;
737 :
738 : /**
739 : * After a setcaps, our caps may have changed. In that case, we can't use
740 : * the old buffer, if there was one (it might have different dimensions)
741 : */
742 7 : silent_debug (self, "swapping old buffers");
743 7 : gst_tensor_rate_swap_prev (self, NULL, GST_CLOCK_TIME_NONE);
744 7 : self->last_ts = GST_CLOCK_TIME_NONE;
745 :
746 7 : return TRUE;
747 :
748 0 : no_framerate:
749 0 : silent_debug (self, "no framerate specified");
750 0 : return FALSE;
751 : }
752 :
753 : /**
754 : * @brief notify a frame drop event
755 : * @param[in] self "this" pointer
756 : */
757 : static void
758 0 : gst_tensor_rate_notify_drop (GstTensorRate * self)
759 : {
760 0 : g_object_notify_by_pspec ((GObject *) self, pspec_drop);
761 0 : }
762 :
763 : /**
764 : * @brief notify a frame duplicate event
765 : * @param[in] self "this" pointer
766 : */
767 : static void
768 0 : gst_tensor_rate_notify_duplicate (GstTensorRate * self)
769 : {
770 0 : g_object_notify_by_pspec ((GObject *) self, pspec_duplicate);
771 0 : }
772 :
773 : #define MAGIC_LIMIT 25
774 : /**
775 : * @brief Event handler for sink pad of tensor rate.
776 : * @param[in] trans "this" pointer
777 : * @param[in] event a passed event object
778 : * @return TRUE if there is no error.
779 : */
780 : static gboolean
781 28 : gst_tensor_rate_sink_event (GstBaseTransform * trans, GstEvent * event)
782 : {
783 28 : GstTensorRate *self = GST_TENSOR_RATE (trans);
784 :
785 28 : switch (GST_EVENT_TYPE (event)) {
786 7 : case GST_EVENT_SEGMENT:
787 : {
788 : GstSegment segment;
789 : gint seqnum;
790 :
791 7 : silent_debug (self, "Got %s",
792 : gst_event_type_get_name (GST_EVENT_TYPE (event)));
793 :
794 7 : gst_event_copy_segment (event, &segment);
795 7 : if (segment.format != GST_FORMAT_TIME) {
796 0 : ml_loge ("Got segment but doesn't have GST_FORMAT_TIME value");
797 0 : return FALSE;
798 : }
799 :
800 : /* close up the previous segment, if appropriate */
801 7 : if (self->prevbuf) {
802 0 : gint count = 0;
803 : GstFlowReturn res;
804 :
805 0 : res = GST_FLOW_OK;
806 : /**
807 : * fill up to the end of current segment,
808 : * or only send out the stored buffer if there is no specific stop.
809 : * regardless, prevent going loopy in strange cases
810 : */
811 0 : while (res == GST_FLOW_OK && count <= MAGIC_LIMIT
812 0 : && ((GST_CLOCK_TIME_IS_VALID (self->segment.stop)
813 0 : && GST_CLOCK_TIME_IS_VALID (self->next_ts)
814 0 : && self->next_ts - self->segment.base <
815 0 : self->segment.stop) || count < 1)) {
816 : res =
817 0 : gst_tensor_rate_flush_prev (self, count > 0, GST_CLOCK_TIME_NONE);
818 0 : count++;
819 : }
820 0 : if (count > 1) {
821 0 : self->dup += count - 1;
822 0 : if (!self->silent)
823 0 : gst_tensor_rate_notify_duplicate (self);
824 : }
825 : /* clean up for the new one; _chain will resume from the new start */
826 0 : gst_tensor_rate_swap_prev (self, NULL, 0);
827 : }
828 :
829 7 : self->base_ts = 0;
830 7 : self->out_frame_count = 0;
831 7 : self->next_ts = GST_CLOCK_TIME_NONE;
832 :
833 7 : gst_segment_copy_into (&segment, &self->segment);
834 :
835 7 : silent_debug (self, "updated segment: %" GST_SEGMENT_FORMAT,
836 : &self->segment);
837 :
838 7 : seqnum = gst_event_get_seqnum (event);
839 7 : gst_event_unref (event);
840 7 : event = gst_event_new_segment (&segment);
841 7 : gst_event_set_seqnum (event, seqnum);
842 :
843 7 : break;
844 : }
845 7 : case GST_EVENT_SEGMENT_DONE:
846 : case GST_EVENT_EOS:
847 : {
848 7 : gint count = 0;
849 7 : GstFlowReturn res = GST_FLOW_OK;
850 :
851 7 : silent_debug (self, "Got %s",
852 : gst_event_type_get_name (GST_EVENT_TYPE (event)));
853 :
854 : /* If the segment has a stop position, fill the segment */
855 7 : if (GST_CLOCK_TIME_IS_VALID (self->segment.stop)) {
856 : /**
857 : * fill up to the end of current segment,
858 : * or only send out the stored buffer if there is no specific stop.
859 : * regardless, prevent going loopy in strange cases
860 : */
861 0 : while (res == GST_FLOW_OK && count <= MAGIC_LIMIT
862 0 : && (GST_CLOCK_TIME_IS_VALID (self->segment.stop)
863 0 : && GST_CLOCK_TIME_IS_VALID (self->next_ts)
864 0 : && (self->next_ts - self->segment.base < self->segment.stop))) {
865 0 : res = gst_tensor_rate_flush_prev (self, count > 0,
866 : GST_CLOCK_TIME_NONE);
867 0 : count++;
868 : }
869 7 : } else if (self->prevbuf) {
870 : /**
871 : * Output at least one frame but if the buffer duration is valid, output
872 : * enough frames to use the complete buffer duration
873 : */
874 6 : if (GST_BUFFER_DURATION_IS_VALID (self->prevbuf)) {
875 6 : GstClockTime end_ts =
876 6 : self->next_ts + GST_BUFFER_DURATION (self->prevbuf);
877 :
878 12 : while (res == GST_FLOW_OK && count <= MAGIC_LIMIT &&
879 12 : ((GST_CLOCK_TIME_IS_VALID (self->segment.stop)
880 0 : && GST_CLOCK_TIME_IS_VALID (self->next_ts)
881 0 : && self->next_ts - self->segment.base < end_ts)
882 12 : || count < 1)) {
883 : res =
884 6 : gst_tensor_rate_flush_prev (self, count > 0,
885 : GST_CLOCK_TIME_NONE);
886 6 : count++;
887 : }
888 : } else {
889 0 : res = gst_tensor_rate_flush_prev (self, FALSE, GST_CLOCK_TIME_NONE);
890 0 : count = 1;
891 : }
892 : }
893 :
894 7 : if (count > 1) {
895 0 : self->dup += count - 1;
896 0 : if (!self->silent)
897 0 : gst_tensor_rate_notify_duplicate (self);
898 : }
899 :
900 7 : break;
901 : }
902 0 : case GST_EVENT_FLUSH_STOP:
903 : /* also resets the segment */
904 0 : silent_debug (self, "Got %s",
905 : gst_event_type_get_name (GST_EVENT_TYPE (event)));
906 0 : gst_tensor_rate_reset (self);
907 0 : break;
908 0 : case GST_EVENT_GAP:
909 : /* no gaps after tensor rate, ignore the event */
910 0 : silent_debug (self, "Got %s",
911 : gst_event_type_get_name (GST_EVENT_TYPE (event)));
912 0 : gst_event_unref (event);
913 0 : return TRUE;
914 14 : default:
915 14 : break;
916 : }
917 :
918 : /* other events are handled in the default event handler */
919 28 : return GST_BASE_TRANSFORM_CLASS (parent_class)->sink_event (trans, event);
920 : }
921 :
922 : /**
923 : * @brief Called when the element starts processing. optional vmethod of BaseTransform
924 : * @param[in] trans "this" pointer
925 : * @return TRUE if there is no error.
926 : */
927 : static gboolean
928 7 : gst_tensor_rate_start (GstBaseTransform * trans)
929 : {
930 7 : GstTensorRate *self = GST_TENSOR_RATE (trans);
931 7 : gst_tensor_rate_reset (self);
932 7 : return TRUE;
933 : }
934 :
935 : /**
936 : * @brief Called when the element stops processing. optional vmethod of BaseTransform
937 : * @param[in] trans "this" pointer
938 : * @return TRUE if there is no error.
939 : */
940 : static gboolean
941 7 : gst_tensor_rate_stop (GstBaseTransform * trans)
942 : {
943 7 : GstTensorRate *self = GST_TENSOR_RATE (trans);
944 7 : gst_tensor_rate_reset (self);
945 7 : return TRUE;
946 : }
947 :
948 : /**
949 : * @brief Installs all the properties for tensor_rate
950 : * @param[in] gobject_class Glib object class whose properties will be set
951 : */
952 : static void
953 5 : gst_tensor_rate_install_properties (GObjectClass * object_class)
954 : {
955 : /* PROP_IN */
956 5 : g_object_class_install_property (object_class, PROP_IN,
957 : g_param_spec_uint64 ("in", "In",
958 : "Number of input frames",
959 : 0, G_MAXUINT64, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
960 :
961 : /* PROP_OUT */
962 5 : g_object_class_install_property (object_class, PROP_OUT,
963 : g_param_spec_uint64 ("out", "Out",
964 : "Number of output frames",
965 : 0, G_MAXUINT64, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
966 :
967 : /* PROP_DUP */
968 5 : pspec_duplicate = g_param_spec_uint64 ("duplicate", "Duplicate",
969 : "Number of duplicated frames", 0,
970 : G_MAXUINT64, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS);
971 5 : g_object_class_install_property (object_class, PROP_DUP, pspec_duplicate);
972 :
973 : /* PROP_DROP */
974 5 : pspec_drop =
975 5 : g_param_spec_uint64 ("drop", "Drop", "Number of dropped frames", 0,
976 : G_MAXUINT64, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS);
977 5 : g_object_class_install_property (object_class, PROP_DROP, pspec_drop);
978 :
979 : /* PROP_SILENT */
980 5 : g_object_class_install_property (object_class, PROP_SILENT,
981 : g_param_spec_boolean ("silent", "Silent",
982 : "Don't produce verbose output including dropped/duplicated frames",
983 : DEFAULT_SILENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
984 :
985 : /* PROP_THROTTLE */
986 5 : g_object_class_install_property (object_class, PROP_THROTTLE,
987 : g_param_spec_boolean ("throttle", "Throttle",
988 : "Send QoS events to upstream elements to limit a incoming data rate",
989 : DEFAULT_THROTTLE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
990 :
991 : /* PROP_FRAMERATE */
992 5 : g_object_class_install_property (object_class, PROP_FRAMERATE,
993 : g_param_spec_string ("framerate", "Framerate",
994 : "Specify a target framerate to adjust (e.g., framerate=10/1). "
995 : "Otherwise, the latest processing time will be a target interval.",
996 : "", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
997 5 : }
|
