Line data Source code
1 : /* SPDX-License-Identifier: Apache-2.0 */
2 : /**
3 : * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
4 : *
5 : * @file ml-api-inference-pipeline.c
6 : * @date 11 March 2019
7 : * @brief NNStreamer/Pipeline(main) C-API Wrapper.
8 : * This allows to construct and control NNStreamer pipelines.
9 : * @see https://github.com/nnstreamer/nnstreamer
10 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
11 : * @bug Thread safety for ml_tensors_data should be addressed.
12 : */
13 :
14 : #include <string.h>
15 : #include <glib.h>
16 : #include <gst/gstbuffer.h>
17 : #include <gst/app/app.h> /* To push data to pipeline */
18 : #include <nnstreamer_plugin_api.h>
19 : #include <tensor_if.h>
20 : #include <tensor_typedef.h>
21 : #include <tensor_filter_custom_easy.h>
22 :
23 : #include <nnstreamer.h>
24 : #include <nnstreamer-tizen-internal.h>
25 :
26 : #include "ml-api-internal.h"
27 : #include "ml-api-inference-internal.h"
28 : #include "ml-api-inference-pipeline-internal.h"
29 :
30 :
31 : #define handle_init(name, h) \
32 : ml_pipeline_common_elem *name= (h); \
33 : ml_pipeline *p; \
34 : ml_pipeline_element *elem; \
35 : int ret = ML_ERROR_NONE; \
36 : check_feature_state (ML_FEATURE_INFERENCE); \
37 : if ((h) == NULL) { \
38 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER, \
39 : "The parameter, %s, (handle) is invalid (NULL). Please provide a valid handle.", \
40 : #h); \
41 : } \
42 : p = name->pipe; \
43 : elem = name->element; \
44 : if (p == NULL) \
45 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER, \
46 : "Internal error. The contents of parameter, %s, (handle), is invalid. The pipeline entry (%s->pipe) is NULL. The handle (%s) is either not properly created or application threads may have touched its contents.", \
47 : #h, #h, #h); \
48 : if (elem == NULL) \
49 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER, \
50 : "Internal error. The contents of parameter, %s, (handle), is invalid. The element entry (%s->element) is NULL. The handle (%s) is either not properly created or application threads may have touched its contents.", \
51 : #h, #h, #h); \
52 : if (elem->pipe == NULL) \
53 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER, \
54 : "Internal error. The contents of parameter, %s, (handle), is invalid. The pipeline entry of the element entry (%s->element->pipe) is NULL. The handle (%s) is either not properly created or application threads may have touched its contents.", \
55 : #h, #h, #h); \
56 : g_mutex_lock (&p->lock); \
57 : g_mutex_lock (&elem->lock); \
58 : if (NULL == g_list_find (elem->handles, name)) { \
59 : _ml_error_report \
60 : ("Internal error. The handle name, %s, does not exists in the list of %s->element->handles.", \
61 : #h, #h); \
62 : ret = ML_ERROR_INVALID_PARAMETER; \
63 : goto unlock_return; \
64 : }
65 :
66 : #define handle_exit(h) \
67 : unlock_return: \
68 : g_mutex_unlock (&elem->lock); \
69 : g_mutex_unlock (&p->lock); \
70 : return ret;
71 :
72 : /**
73 : * @brief The enumeration for custom data type.
74 : */
75 : typedef enum
76 : {
77 : PIPE_CUSTOM_TYPE_NONE,
78 : PIPE_CUSTOM_TYPE_IF,
79 : PIPE_CUSTOM_TYPE_FILTER,
80 :
81 : PIPE_CUSTOM_TYPE_MAX
82 : } pipe_custom_type_e;
83 :
84 : /**
85 : * @brief The struct for custom data.
86 : */
87 : typedef struct
88 : {
89 : pipe_custom_type_e type;
90 : gchar *name;
91 : gpointer handle;
92 : } pipe_custom_data_s;
93 :
94 : static void ml_pipeline_custom_filter_ref (ml_custom_easy_filter_h custom);
95 : static void ml_pipeline_custom_filter_unref (ml_custom_easy_filter_h custom);
96 : static void ml_pipeline_if_custom_ref (ml_pipeline_if_h custom);
97 : static void ml_pipeline_if_custom_unref (ml_pipeline_if_h custom);
98 :
99 : /**
100 : * @brief Global lock for pipeline functions.
101 : */
102 : G_LOCK_DEFINE_STATIC (g_ml_pipe_lock);
103 :
104 : /**
105 : * @brief The list of custom data. This should be managed with lock.
106 : */
107 : static GList *g_ml_custom_data = NULL;
108 :
109 : /**
110 : * @brief Finds a position of custom data in the list.
111 : * @note This function should be called with lock.
112 : */
113 : static GList *
114 12 : pipe_custom_find_link (const pipe_custom_type_e type, const gchar * name)
115 : {
116 : pipe_custom_data_s *data;
117 : GList *link;
118 :
119 12 : g_return_val_if_fail (name != NULL, NULL);
120 :
121 12 : link = g_ml_custom_data;
122 12 : while (link) {
123 12 : data = (pipe_custom_data_s *) link->data;
124 :
125 12 : if (data->type == type && g_str_equal (data->name, name))
126 12 : break;
127 :
128 0 : link = link->next;
129 : }
130 :
131 12 : return link;
132 : }
133 :
134 : /**
135 : * @brief Finds custom data matched with data type and name.
136 : */
137 : static pipe_custom_data_s *
138 6 : pipe_custom_find_data (const pipe_custom_type_e type, const gchar * name)
139 : {
140 : pipe_custom_data_s *data;
141 : GList *link;
142 :
143 6 : G_LOCK (g_ml_pipe_lock);
144 :
145 6 : link = pipe_custom_find_link (type, name);
146 6 : data = (link != NULL) ? (pipe_custom_data_s *) link->data : NULL;
147 :
148 6 : G_UNLOCK (g_ml_pipe_lock);
149 6 : return data;
150 : }
151 :
152 : /**
153 : * @brief Adds new custom data into the list.
154 : */
155 : static void
156 6 : pipe_custom_add_data (const pipe_custom_type_e type, const gchar * name,
157 : gpointer handle)
158 : {
159 : pipe_custom_data_s *data;
160 :
161 6 : data = g_new0 (pipe_custom_data_s, 1);
162 6 : data->type = type;
163 6 : data->name = g_strdup (name);
164 6 : data->handle = handle;
165 :
166 6 : G_LOCK (g_ml_pipe_lock);
167 6 : g_ml_custom_data = g_list_prepend (g_ml_custom_data, data);
168 6 : G_UNLOCK (g_ml_pipe_lock);
169 6 : }
170 :
171 : /**
172 : * @brief Removes custom data from the list.
173 : */
174 : static void
175 6 : pipe_custom_remove_data (const pipe_custom_type_e type, const gchar * name)
176 : {
177 : pipe_custom_data_s *data;
178 : GList *link;
179 :
180 6 : G_LOCK (g_ml_pipe_lock);
181 :
182 6 : link = pipe_custom_find_link (type, name);
183 6 : if (link) {
184 6 : data = (pipe_custom_data_s *) link->data;
185 :
186 6 : g_ml_custom_data = g_list_delete_link (g_ml_custom_data, link);
187 :
188 6 : g_free (data->name);
189 6 : g_free (data);
190 : }
191 :
192 6 : G_UNLOCK (g_ml_pipe_lock);
193 6 : }
194 :
195 : /**
196 : * @brief The callback function called when the element node with custom data is released.
197 : */
198 : static int
199 6 : pipe_custom_destroy_cb (void *handle, void *user_data)
200 : {
201 : pipe_custom_data_s *custom_data;
202 :
203 6 : custom_data = (pipe_custom_data_s *) handle;
204 6 : if (custom_data == NULL)
205 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
206 : "The parameter, handle, is NULL. It should be a valid internal object. This is possibly a bug in ml-api-inference-pipeline.c along with tensor-if or tensor-filter/custom function. Please report to https://github.com/nnstreamer/nnstreamer/issues");
207 :
208 6 : switch (custom_data->type) {
209 3 : case PIPE_CUSTOM_TYPE_IF:
210 3 : ml_pipeline_if_custom_unref (custom_data->handle);
211 3 : break;
212 3 : case PIPE_CUSTOM_TYPE_FILTER:
213 3 : ml_pipeline_custom_filter_unref (custom_data->handle);
214 3 : break;
215 0 : default:
216 0 : break;
217 : }
218 :
219 6 : return ML_ERROR_NONE;
220 : }
221 :
222 : /**
223 : * @brief Internal function to create a referable element in a pipeline
224 : */
225 : static ml_pipeline_element *
226 404 : construct_element (GstElement * e, ml_pipeline * p, const char *name,
227 : ml_pipeline_element_e t)
228 : {
229 404 : ml_pipeline_element *ret = g_new0 (ml_pipeline_element, 1);
230 :
231 404 : if (ret == NULL)
232 0 : _ml_error_report_return (NULL,
233 : "Failed to allocate memory for the pipeline.");
234 :
235 404 : ret->element = e;
236 404 : ret->pipe = p;
237 404 : ret->name = g_strdup (name);
238 404 : ret->type = t;
239 404 : ret->handles = NULL;
240 404 : ret->src = NULL;
241 404 : ret->sink = NULL;
242 404 : ret->maxid = 0;
243 404 : ret->handle_id = 0;
244 404 : ret->is_media_stream = FALSE;
245 404 : ret->is_flexible_tensor = FALSE;
246 404 : g_mutex_init (&ret->lock);
247 404 : gst_tensors_info_init (&ret->tensors_info);
248 :
249 404 : return ret;
250 : }
251 :
252 : /**
253 : * @brief Internal function to get the tensors info from the element caps.
254 : */
255 : static gboolean
256 12172 : get_tensors_info_from_caps (const GstCaps * caps, GstTensorsInfo * info,
257 : gboolean * is_flexible)
258 : {
259 : GstTensorsConfig config;
260 12172 : gboolean found = FALSE;
261 :
262 12172 : found = gst_tensors_config_from_caps (&config, caps, TRUE);
263 :
264 12172 : if (found) {
265 12166 : gst_tensors_info_free (info);
266 12166 : gst_tensors_info_copy (info, &config.info);
267 :
268 12166 : if (is_flexible) {
269 12166 : *is_flexible = gst_tensors_config_is_flexible (&config);
270 : }
271 :
272 12166 : gst_tensors_config_free (&config);
273 : }
274 :
275 12172 : return found;
276 : }
277 :
278 : /**
279 : * @brief Handle a sink element for registered ml_pipeline_sink_cb
280 : */
281 : static void
282 6116 : cb_sink_event (GstElement * e, GstBuffer * b, gpointer user_data)
283 : {
284 6116 : ml_pipeline_element *elem = user_data;
285 :
286 : /** @todo CRITICAL if the pipeline is being killed, don't proceed! */
287 : GstMemory *mem[ML_TENSOR_SIZE_LIMIT];
288 : GstMapInfo map[ML_TENSOR_SIZE_LIMIT];
289 : guint i, num_tensors;
290 : GList *l;
291 6116 : ml_tensors_data_s *_data = NULL;
292 : GstTensorsInfo gst_info;
293 : int status;
294 :
295 6116 : gst_tensors_info_init (&gst_info);
296 6116 : gst_info.num_tensors = num_tensors = gst_tensor_buffer_get_count (b);
297 :
298 : /* Set tensor data. The handle for tensors-info in data should be added. */
299 : status =
300 6116 : _ml_tensors_data_create_no_alloc (NULL, (ml_tensors_data_h *) & _data);
301 6116 : if (status != ML_ERROR_NONE) {
302 0 : _ml_loge (_ml_detail
303 : ("Failed to allocate memory for tensors data in sink callback, which is registered by ml_pipeline_sink_register ()."));
304 6116 : return;
305 : }
306 :
307 6116 : g_mutex_lock (&elem->lock);
308 :
309 6116 : _data->num_tensors = num_tensors;
310 12709 : for (i = 0; i < num_tensors; i++) {
311 6593 : mem[i] = gst_tensor_buffer_get_nth_memory (b, i);
312 6593 : if (!gst_memory_map (mem[i], &map[i], GST_MAP_READ)) {
313 0 : _ml_loge (_ml_detail
314 : ("Failed to map the output in sink '%s' callback, which is registered by ml_pipeline_sink_register ()",
315 : elem->name));
316 0 : gst_memory_unref (mem[i]);
317 0 : num_tensors = i;
318 0 : goto error;
319 : }
320 :
321 6593 : _data->tensors[i].data = map[i].data;
322 6593 : _data->tensors[i].size = map[i].size;
323 : }
324 :
325 : /** @todo This assumes that padcap is static */
326 6116 : if (elem->sink == NULL) {
327 28 : gboolean found = FALSE;
328 28 : gboolean flexible = FALSE;
329 :
330 : /* Get the sink-pad-cap */
331 28 : elem->sink = gst_element_get_static_pad (elem->element, "sink");
332 :
333 28 : if (elem->sink) {
334 : /* sinkpadcap available (negotiated) */
335 28 : GstCaps *caps = gst_pad_get_current_caps (elem->sink);
336 :
337 28 : if (caps) {
338 28 : found = get_tensors_info_from_caps (caps, &elem->tensors_info,
339 : &flexible);
340 28 : gst_caps_unref (caps);
341 : }
342 : }
343 :
344 28 : if (found) {
345 28 : elem->is_flexible_tensor = flexible;
346 : } else {
347 : /* It is not valid */
348 0 : if (elem->sink) {
349 0 : gst_object_unref (elem->sink);
350 0 : elem->sink = NULL;
351 : }
352 :
353 0 : goto error;
354 : }
355 : }
356 :
357 : /* Prepare output and set data. */
358 6116 : if (elem->is_flexible_tensor) {
359 : GstTensorMetaInfo meta;
360 : gsize hsize;
361 :
362 : /* handle header for flex tensor */
363 12 : for (i = 0; i < num_tensors; i++) {
364 9 : gst_tensor_meta_info_parse_header (&meta, map[i].data);
365 9 : hsize = gst_tensor_meta_info_get_header_size (&meta);
366 :
367 9 : gst_tensor_meta_info_convert (&meta,
368 : gst_tensors_info_get_nth_info (&gst_info, i));
369 :
370 9 : _data->tensors[i].data = map[i].data + hsize;
371 9 : _data->tensors[i].size = map[i].size - hsize;
372 : }
373 : } else {
374 6113 : gst_tensors_info_copy (&gst_info, &elem->tensors_info);
375 :
376 : /* Compare output info and buffer if gst-buffer is not flexible. */
377 6113 : if (gst_info.num_tensors != num_tensors) {
378 0 : _ml_loge (_ml_detail
379 : ("The sink event of [%s] cannot be handled because the number of tensors mismatches.",
380 : elem->name));
381 :
382 0 : gst_object_unref (elem->sink);
383 0 : elem->sink = NULL;
384 0 : goto error;
385 : }
386 :
387 12697 : for (i = 0; i < num_tensors; i++) {
388 6584 : size_t sz = gst_tensors_info_get_size (&gst_info, i);
389 :
390 : /* Not configured, yet. */
391 6584 : if (sz == 0)
392 0 : _ml_loge (_ml_detail
393 : ("The caps for sink(%s) is not configured.", elem->name));
394 :
395 6584 : if (sz != map[i].size) {
396 0 : _ml_loge (_ml_detail
397 : ("The sink event of [%s] cannot be handled because the tensor dimension mismatches.",
398 : elem->name));
399 :
400 0 : gst_object_unref (elem->sink);
401 0 : elem->sink = NULL;
402 0 : goto error;
403 : }
404 : }
405 : }
406 :
407 : /* Create new output info, data handle should be updated here. */
408 6116 : _ml_tensors_info_create_from_gst (&_data->info, &gst_info);
409 :
410 : /* Iterate e->handles, pass the data to them */
411 12238 : for (l = elem->handles; l != NULL; l = l->next) {
412 : ml_pipeline_sink_cb callback;
413 6122 : ml_pipeline_common_elem *sink = l->data;
414 6122 : if (sink->callback_info == NULL)
415 3 : continue;
416 :
417 6119 : callback = sink->callback_info->sink_cb;
418 6119 : if (callback)
419 6119 : callback (_data, _data->info, sink->callback_info->sink_pdata);
420 :
421 : /** @todo Measure time. Warn if it takes long. Kill if it takes too long. */
422 : }
423 :
424 6116 : error:
425 6116 : g_mutex_unlock (&elem->lock);
426 :
427 12709 : for (i = 0; i < num_tensors; i++) {
428 6593 : gst_memory_unmap (mem[i], &map[i]);
429 6593 : gst_memory_unref (mem[i]);
430 : }
431 :
432 6116 : _ml_tensors_data_destroy_internal (_data, FALSE);
433 6116 : _data = NULL;
434 :
435 6116 : gst_tensors_info_free (&gst_info);
436 6116 : return;
437 : }
438 :
439 : /**
440 : * @brief Handle a appsink element for registered ml_pipeline_sink_cb
441 : */
442 : static GstFlowReturn
443 9 : cb_appsink_new_sample (GstElement * e, gpointer user_data)
444 : {
445 : GstSample *sample;
446 : GstBuffer *buffer;
447 :
448 : /* get the sample from appsink */
449 9 : sample = gst_app_sink_pull_sample (GST_APP_SINK (e));
450 9 : buffer = gst_sample_get_buffer (sample);
451 :
452 9 : cb_sink_event (e, buffer, user_data);
453 :
454 9 : gst_sample_unref (sample);
455 9 : return GST_FLOW_OK;
456 : }
457 :
458 : /**
459 : * @brief Callback for bus message.
460 : */
461 : static void
462 5408 : cb_bus_sync_message (GstBus * bus, GstMessage * message, gpointer user_data)
463 : {
464 : ml_pipeline *pipe_h;
465 :
466 5408 : pipe_h = (ml_pipeline *) user_data;
467 :
468 5408 : if (pipe_h == NULL)
469 0 : return;
470 :
471 5408 : switch (GST_MESSAGE_TYPE (message)) {
472 7 : case GST_MESSAGE_EOS:
473 7 : pipe_h->isEOS = TRUE;
474 7 : break;
475 4391 : case GST_MESSAGE_STATE_CHANGED:
476 4391 : if (GST_MESSAGE_SRC (message) == GST_OBJECT_CAST (pipe_h->element)) {
477 : GstState old_state, new_state;
478 :
479 507 : gst_message_parse_state_changed (message, &old_state, &new_state, NULL);
480 507 : pipe_h->pipe_state = (ml_pipeline_state_e) new_state;
481 :
482 507 : _ml_logd (_ml_detail ("The pipeline state changed from %s to %s.",
483 : gst_element_state_get_name (old_state),
484 : gst_element_state_get_name (new_state)));
485 :
486 507 : if (pipe_h->state_cb.cb) {
487 11 : pipe_h->state_cb.cb (pipe_h->pipe_state, pipe_h->state_cb.user_data);
488 : }
489 : }
490 4391 : break;
491 1010 : default:
492 1010 : break;
493 : }
494 : }
495 :
496 : /**
497 : * @brief Clean up each element of the pipeline.
498 : */
499 : static void
500 151 : free_element_handle (gpointer data)
501 : {
502 151 : ml_pipeline_common_elem *item = (ml_pipeline_common_elem *) data;
503 : ml_pipeline_element *elem;
504 :
505 151 : if (!(item && item->callback_info)) {
506 109 : g_free (item);
507 109 : return;
508 : }
509 :
510 : /* clear callbacks */
511 42 : item->callback_info->sink_cb = NULL;
512 42 : elem = item->element;
513 42 : if (elem && elem->type == ML_PIPELINE_ELEMENT_APP_SRC) {
514 2 : GstAppSrcCallbacks appsrc_cb = { 0, };
515 2 : gst_app_src_set_callbacks (GST_APP_SRC (elem->element), &appsrc_cb,
516 : NULL, NULL);
517 : }
518 :
519 42 : g_free (item->callback_info);
520 42 : item->callback_info = NULL;
521 42 : g_free (item);
522 : }
523 :
524 : /**
525 : * @brief Private function for ml_pipeline_destroy, cleaning up nodes in namednodes
526 : */
527 : static void
528 394 : cleanup_node (gpointer data)
529 : {
530 394 : ml_pipeline_element *e = data;
531 :
532 394 : g_mutex_lock (&e->lock);
533 : /** @todo CRITICAL. Stop the handle callbacks if they are running/ready */
534 394 : if (e->handle_id > 0) {
535 27 : g_signal_handler_disconnect (e->element, e->handle_id);
536 27 : e->handle_id = 0;
537 : }
538 :
539 : /* clear all handles first */
540 394 : if (e->handles)
541 63 : g_list_free_full (e->handles, free_element_handle);
542 394 : e->handles = NULL;
543 :
544 394 : if (e->type == ML_PIPELINE_ELEMENT_APP_SRC) {
545 55 : ml_pipeline *p = e->pipe;
546 :
547 55 : if (p && !p->isEOS) {
548 55 : int eos_check_cnt = 0;
549 :
550 : /** to push EOS event, the pipeline should be in PAUSED state */
551 55 : gst_element_set_state (p->element, GST_STATE_PAUSED);
552 :
553 55 : if (gst_app_src_end_of_stream (GST_APP_SRC (e->element)) != GST_FLOW_OK) {
554 0 : _ml_logw (_ml_detail
555 : ("Cleaning up a pipeline has failed to set End-Of-Stream for the pipeline element of %s",
556 : e->name));
557 : }
558 :
559 55 : g_mutex_unlock (&e->lock);
560 5500 : while (!p->isEOS) {
561 5500 : eos_check_cnt++;
562 : /** check EOS every 1ms */
563 5500 : g_usleep (1000);
564 5500 : if (eos_check_cnt >= EOS_MESSAGE_TIME_LIMIT) {
565 55 : _ml_loge (_ml_detail
566 : ("Cleaning up a pipeline has requested to set End-Of-Stream. However, the pipeline has not become EOS after the timeout. It has failed to become EOS with the element of %s.",
567 : e->name));
568 55 : break;
569 : }
570 : }
571 55 : g_mutex_lock (&e->lock);
572 : }
573 : }
574 :
575 394 : if (e->custom_destroy) {
576 6 : e->custom_destroy (e->custom_data, e);
577 : }
578 :
579 394 : g_free (e->name);
580 394 : if (e->src)
581 41 : gst_object_unref (e->src);
582 394 : if (e->sink)
583 28 : gst_object_unref (e->sink);
584 :
585 394 : gst_object_unref (e->element);
586 :
587 394 : gst_tensors_info_free (&e->tensors_info);
588 :
589 394 : g_mutex_unlock (&e->lock);
590 394 : g_mutex_clear (&e->lock);
591 :
592 394 : g_free (e);
593 394 : }
594 :
595 : /**
596 : * @brief Private function to release the pipeline resources
597 : */
598 : static void
599 0 : cleanup_resource (gpointer data)
600 : {
601 0 : pipeline_resource_s *res = data;
602 :
603 : /* check resource type and free data */
604 0 : if (g_str_has_prefix (res->type, "tizen")) {
605 0 : release_tizen_resource (res->handle, res->type);
606 : }
607 :
608 0 : g_free (res->type);
609 0 : g_free (res);
610 0 : }
611 :
612 : /**
613 : * @brief Converts predefined element in pipeline description.
614 : */
615 : static int
616 148 : convert_description (ml_pipeline_h pipe, const gchar * description,
617 : gchar ** result, gboolean is_internal)
618 : {
619 : gchar *converted;
620 148 : int status = ML_ERROR_NONE;
621 :
622 296 : g_return_val_if_fail (pipe, ML_ERROR_INVALID_PARAMETER);
623 148 : g_return_val_if_fail (description && result, ML_ERROR_INVALID_PARAMETER);
624 :
625 : /* init null */
626 148 : *result = NULL;
627 :
628 148 : converted = _ml_convert_predefined_entity (description);
629 :
630 : /* convert pre-defined element for Tizen */
631 148 : status = convert_tizen_element (pipe, &converted, is_internal);
632 :
633 148 : if (status == ML_ERROR_NONE) {
634 148 : _ml_logd (_ml_detail
635 : ("Pipeline element converted with aliases for gstreamer (Tizen element aliases): %s",
636 : converted));
637 148 : *result = converted;
638 : } else {
639 0 : g_free (converted);
640 0 : _ml_error_report_continue
641 : ("Failed to convert element: convert_tizen_element() returned %d",
642 : status);
643 : }
644 :
645 148 : return status;
646 : }
647 :
648 : /**
649 : * @brief Handle tensor-filter options.
650 : */
651 : static void
652 39 : process_tensor_filter_option (ml_pipeline_element * e)
653 : {
654 39 : gchar *fw = NULL;
655 39 : gchar *model = NULL;
656 : pipe_custom_data_s *custom_data;
657 :
658 39 : g_object_get (G_OBJECT (e->element), "framework", &fw, "model", &model, NULL);
659 :
660 39 : if (fw && g_ascii_strcasecmp (fw, "custom-easy") == 0) {
661 : /* ref to tensor-filter custom-easy handle. */
662 3 : custom_data = pipe_custom_find_data (PIPE_CUSTOM_TYPE_FILTER, model);
663 3 : if (custom_data) {
664 3 : ml_pipeline_custom_filter_ref (custom_data->handle);
665 :
666 3 : e->custom_destroy = pipe_custom_destroy_cb;
667 3 : e->custom_data = custom_data;
668 : }
669 : }
670 :
671 39 : g_free (fw);
672 39 : g_free (model);
673 39 : }
674 :
675 : /**
676 : * @brief Handle tensor-if options.
677 : */
678 : static void
679 3 : process_tensor_if_option (ml_pipeline_element * e)
680 : {
681 3 : gint cv = 0;
682 3 : gchar *cv_option = NULL;
683 : pipe_custom_data_s *custom_data;
684 :
685 3 : g_object_get (G_OBJECT (e->element), "compared-value", &cv,
686 : "compared-value-option", &cv_option, NULL);
687 :
688 3 : if (cv == 5) {
689 : /* cv is TIFCV_CUSTOM, ref to tensor-if custom handle. */
690 3 : custom_data = pipe_custom_find_data (PIPE_CUSTOM_TYPE_IF, cv_option);
691 3 : if (custom_data) {
692 3 : ml_pipeline_if_custom_ref (custom_data->handle);
693 :
694 3 : e->custom_destroy = pipe_custom_destroy_cb;
695 3 : e->custom_data = custom_data;
696 : }
697 : }
698 :
699 3 : g_free (cv_option);
700 3 : }
701 :
702 : /**
703 : * @brief Initializes the GStreamer library. This is internal function.
704 : */
705 : int
706 233 : _ml_initialize_gstreamer (void)
707 : {
708 233 : GError *err = NULL;
709 :
710 233 : if (!gst_init_check (NULL, NULL, &err)) {
711 0 : if (err) {
712 0 : _ml_error_report
713 : ("Initrializing ML-API failed: GStreamer has the following error from gst_init_check(): %s",
714 : err->message);
715 0 : g_clear_error (&err);
716 : } else {
717 0 : _ml_error_report ("Cannot initialize GStreamer. Unknown reason.");
718 : }
719 :
720 233 : return ML_ERROR_STREAMS_PIPE;
721 : }
722 :
723 233 : return ML_ERROR_NONE;
724 : }
725 :
726 : /**
727 : * @brief Checks the element is registered and available on the pipeline.
728 : */
729 : int
730 79 : ml_check_element_availability (const char *element_name, bool *available)
731 : {
732 : GstElementFactory *factory;
733 : int status;
734 :
735 79 : check_feature_state (ML_FEATURE_INFERENCE);
736 :
737 79 : if (!element_name)
738 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
739 : "The parameter, element_name, is NULL. It should be a name (string) to be queried if it exists as a GStreamer/NNStreamer element.");
740 :
741 78 : if (!available)
742 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
743 : "The parameter, available, is NULL. It should be a valid pointer to a bool entry so that the API (ml_check_element_availability) may return the queried result via \"available\" parameter. E.g., bool available; ml_check_element_availability (\"tensor_converter\", &available);");
744 :
745 77 : _ml_error_report_return_continue_iferr (_ml_initialize_gstreamer (),
746 : "Internal error of _ml_initialize_gstreamer(). Check the availability of gstreamer libraries in your system.");
747 :
748 : /* init false */
749 77 : *available = false;
750 :
751 77 : factory = gst_element_factory_find (element_name);
752 77 : if (factory) {
753 76 : GstPluginFeature *feature = GST_PLUGIN_FEATURE (factory);
754 76 : const gchar *plugin_name = gst_plugin_feature_get_plugin_name (feature);
755 :
756 : /* check restricted element */
757 76 : status = _ml_check_plugin_availability (plugin_name, element_name);
758 76 : if (status == ML_ERROR_NONE)
759 63 : *available = true;
760 :
761 76 : gst_object_unref (factory);
762 : }
763 :
764 77 : return ML_ERROR_NONE;
765 : }
766 :
767 : /**
768 : * @brief Checks the availability of the plugin.
769 : */
770 : int
771 927 : _ml_check_plugin_availability (const char *plugin_name,
772 : const char *element_name)
773 : {
774 : static gboolean list_loaded = FALSE;
775 : static gchar **allowed_elements = NULL;
776 :
777 927 : if (!plugin_name)
778 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
779 : "The parameter, plugin_name, is NULL. It should be a valid string.");
780 :
781 926 : if (!element_name)
782 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
783 : "The parameter, element_name, is NULL. It should be a valid string.");
784 :
785 925 : if (!list_loaded) {
786 : gboolean restricted;
787 :
788 : restricted =
789 7 : nnsconf_get_custom_value_bool ("element-restriction",
790 : "enable_element_restriction", FALSE);
791 7 : if (restricted) {
792 : gchar *elements;
793 :
794 : /* check white-list of available plugins */
795 : elements =
796 7 : nnsconf_get_custom_value_string ("element-restriction",
797 : "allowed_elements");
798 7 : if (elements) {
799 7 : allowed_elements = g_strsplit_set (elements, " ,;", -1);
800 7 : g_free (elements);
801 : }
802 : }
803 :
804 7 : list_loaded = TRUE;
805 : }
806 :
807 : /* nnstreamer elements */
808 925 : if (g_str_has_prefix (plugin_name, "nnstreamer") &&
809 272 : g_str_has_prefix (element_name, "tensor_")) {
810 272 : return ML_ERROR_NONE;
811 : }
812 :
813 653 : if (allowed_elements &&
814 653 : find_key_strv ((const gchar **) allowed_elements, element_name) < 0) {
815 13 : _ml_error_report_return (ML_ERROR_NOT_SUPPORTED,
816 : "The element %s is restricted.", element_name);
817 : }
818 :
819 640 : return ML_ERROR_NONE;
820 : }
821 :
822 : /**
823 : * @brief Get the ml_pipeline_element_e type from its element name
824 : */
825 : static ml_pipeline_element_e
826 860 : get_elem_type_from_name (GHashTable * table, const gchar * name)
827 : {
828 860 : gpointer value = g_hash_table_lookup (table, name);
829 860 : if (!value)
830 504 : return ML_PIPELINE_ELEMENT_UNKNOWN;
831 :
832 356 : return GPOINTER_TO_INT (value);
833 : }
834 :
835 : /**
836 : * @brief Iterate elements and prepare element handle.
837 : */
838 : static int
839 145 : iterate_element (ml_pipeline * pipe_h, GstElement * pipeline,
840 : gboolean is_internal)
841 : {
842 145 : GstIterator *it = NULL;
843 145 : int status = ML_ERROR_NONE;
844 :
845 145 : g_return_val_if_fail (pipe_h && pipeline, ML_ERROR_INVALID_PARAMETER);
846 :
847 145 : g_mutex_lock (&pipe_h->lock);
848 :
849 145 : it = gst_bin_iterate_elements (GST_BIN (pipeline));
850 145 : if (it != NULL) {
851 145 : gboolean done = FALSE;
852 145 : GValue item = G_VALUE_INIT;
853 : GObject *obj;
854 : gchar *name;
855 :
856 : /* Fill in the hashtable, "namednodes" with named Elements */
857 1295 : while (!done) {
858 1005 : switch (gst_iterator_next (it, &item)) {
859 860 : case GST_ITERATOR_OK:
860 860 : obj = g_value_get_object (&item);
861 :
862 860 : if (GST_IS_ELEMENT (obj)) {
863 860 : GstElement *elem = GST_ELEMENT (obj);
864 : GstPluginFeature *feature =
865 860 : GST_PLUGIN_FEATURE (gst_element_get_factory (elem));
866 : const gchar *plugin_name =
867 860 : gst_plugin_feature_get_plugin_name (feature);
868 860 : const gchar *element_name = gst_plugin_feature_get_name (feature);
869 :
870 : /* validate the availability of the plugin */
871 860 : if (!is_internal && _ml_check_plugin_availability (plugin_name,
872 : element_name) != ML_ERROR_NONE) {
873 0 : _ml_error_report_continue
874 : ("There is a pipeline element (filter) that is not allowed for applications via ML-API (privilege not granted) or now available: '%s'/'%s'.",
875 : plugin_name, element_name);
876 0 : status = ML_ERROR_NOT_SUPPORTED;
877 0 : done = TRUE;
878 0 : break;
879 : }
880 :
881 860 : name = gst_element_get_name (elem);
882 860 : if (name != NULL) {
883 : ml_pipeline_element_e element_type =
884 860 : get_elem_type_from_name (pipe_h->pipe_elm_type, element_name);
885 :
886 : /* check 'sync' property in sink element */
887 860 : if (element_type == ML_PIPELINE_ELEMENT_SINK ||
888 : element_type == ML_PIPELINE_ELEMENT_APP_SINK) {
889 137 : gboolean sync = FALSE;
890 :
891 137 : g_object_get (G_OBJECT (elem), "sync", &sync, NULL);
892 137 : if (sync) {
893 1 : _ml_logw (_ml_detail
894 : ("It is recommended to apply 'sync=false' property to a sink element in most AI applications. Otherwise, inference results of large neural networks will be frequently dropped by the synchronization mechanism at the sink element."));
895 : }
896 : }
897 :
898 860 : if (element_type != ML_PIPELINE_ELEMENT_UNKNOWN) {
899 : ml_pipeline_element *e;
900 :
901 356 : e = construct_element (gst_object_ref (elem), pipe_h, name,
902 : element_type);
903 356 : if (e != NULL) {
904 356 : if (g_str_equal (element_name, "tensor_if"))
905 3 : process_tensor_if_option (e);
906 353 : else if (g_str_equal (element_name, "tensor_filter"))
907 39 : process_tensor_filter_option (e);
908 :
909 356 : g_hash_table_insert (pipe_h->namednodes, g_strdup (name), e);
910 : } else {
911 : /* allocation failure */
912 0 : gst_object_unref (elem);
913 0 : _ml_error_report_continue
914 : ("Cannot allocate memory with construct_element().");
915 0 : status = ML_ERROR_OUT_OF_MEMORY;
916 0 : done = TRUE;
917 : }
918 : }
919 :
920 860 : g_free (name);
921 : }
922 : }
923 :
924 860 : g_value_reset (&item);
925 860 : break;
926 0 : case GST_ITERATOR_RESYNC:
927 : case GST_ITERATOR_ERROR:
928 0 : _ml_logw (_ml_detail
929 : ("There is an error or a resync-event while inspecting a pipeline. However, we can still execute the pipeline."));
930 : /* fallthrough */
931 145 : case GST_ITERATOR_DONE:
932 145 : done = TRUE;
933 : }
934 : }
935 :
936 145 : g_value_unset (&item);
937 : /** @todo CRITICAL check the validity of elem=item registered in e */
938 145 : gst_iterator_free (it);
939 : }
940 :
941 145 : g_mutex_unlock (&pipe_h->lock);
942 145 : return status;
943 : }
944 :
945 : /**
946 : * @brief Internal function to create the hash table for managing internal resources
947 : */
948 : static void
949 148 : create_internal_hash (ml_pipeline * pipe_h)
950 : {
951 148 : pipe_h->namednodes =
952 148 : g_hash_table_new_full (g_str_hash, g_str_equal, g_free, cleanup_node);
953 148 : pipe_h->resources =
954 148 : g_hash_table_new_full (g_str_hash, g_str_equal, g_free, cleanup_resource);
955 :
956 148 : pipe_h->pipe_elm_type =
957 148 : g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
958 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("tensor_sink"),
959 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_SINK));
960 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("appsrc"),
961 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_APP_SRC));
962 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("appsink"),
963 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_APP_SINK));
964 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("valve"),
965 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_VALVE));
966 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("input-selector"),
967 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_SWITCH_INPUT));
968 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("output-selector"),
969 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_SWITCH_OUTPUT));
970 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("tensor_if"),
971 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_COMMON));
972 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("tensor_filter"),
973 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_COMMON));
974 148 : }
975 :
976 : /**
977 : * @brief Internal function to construct the pipeline.
978 : * If is_internal is true, this will ignore the permission in Tizen.
979 : */
980 : static int
981 150 : construct_pipeline_internal (const char *pipeline_description,
982 : ml_pipeline_state_cb cb, void *user_data, ml_pipeline_h * pipe,
983 : gboolean is_internal)
984 : {
985 150 : GError *err = NULL;
986 : GstElement *pipeline;
987 150 : gchar *description = NULL;
988 150 : int status = ML_ERROR_NONE;
989 :
990 : ml_pipeline *pipe_h;
991 :
992 300 : check_feature_state (ML_FEATURE_INFERENCE);
993 :
994 150 : if (!pipe)
995 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
996 : "ml_pipeline_construct error: parameter pipe is NULL. It should be a valid ml_pipeline_h pointer. E.g., ml_pipeline_h pipe; ml_pipeline_construct (..., &pip);");
997 :
998 149 : if (!pipeline_description)
999 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1000 : "ml_pipeline_construct error: parameter pipeline_description is NULL. It should be a valid string of Gstreamer/NNStreamer pipeline description.");
1001 :
1002 : /* init null */
1003 148 : *pipe = NULL;
1004 :
1005 148 : _ml_error_report_return_continue_iferr (_ml_initialize_gstreamer (),
1006 : "ml_pipeline_construct error: it has failed to initialize gstreamer(). Please check if you have a valid GStreamer library installed in your system.");
1007 :
1008 : /* prepare pipeline handle */
1009 148 : pipe_h = g_new0 (ml_pipeline, 1);
1010 148 : if (pipe_h == NULL)
1011 0 : _ml_error_report_return (ML_ERROR_OUT_OF_MEMORY,
1012 : "ml_pipeline_construct error: failed to allocate memory for pipeline handle. Out of memory?");
1013 :
1014 148 : g_mutex_init (&pipe_h->lock);
1015 :
1016 148 : pipe_h->isEOS = FALSE;
1017 148 : pipe_h->pipe_state = ML_PIPELINE_STATE_UNKNOWN;
1018 :
1019 148 : create_internal_hash (pipe_h);
1020 :
1021 : /* convert predefined element and launch the pipeline */
1022 148 : status = convert_description ((ml_pipeline_h) pipe_h, pipeline_description,
1023 : &description, is_internal);
1024 148 : if (status != ML_ERROR_NONE) {
1025 0 : _ml_error_report_continue
1026 : ("ml_pipeline_construct error: failed while converting pipeline description for GStreamer w/ convert_description() function, which has returned %d",
1027 : status);
1028 0 : goto failed;
1029 : }
1030 :
1031 148 : pipeline = gst_parse_launch (description, &err);
1032 148 : g_free (description);
1033 :
1034 148 : if (pipeline == NULL || err) {
1035 3 : _ml_error_report
1036 : ("ml_pipeline_construct error: gst_parse_launch cannot parse and launch the given pipeline = [%s]. The error message from gst_parse_launch is '%s'.",
1037 : pipeline_description, (err) ? err->message : "unknown reason");
1038 3 : g_clear_error (&err);
1039 :
1040 3 : if (pipeline)
1041 3 : gst_object_unref (pipeline);
1042 :
1043 3 : status = ML_ERROR_STREAMS_PIPE;
1044 3 : goto failed;
1045 : }
1046 :
1047 145 : g_assert (GST_IS_PIPELINE (pipeline));
1048 145 : pipe_h->element = pipeline;
1049 :
1050 : /* bus and message callback */
1051 145 : pipe_h->bus = gst_element_get_bus (pipeline);
1052 145 : g_assert (pipe_h->bus);
1053 :
1054 145 : gst_bus_enable_sync_message_emission (pipe_h->bus);
1055 145 : pipe_h->signal_msg = g_signal_connect (pipe_h->bus, "sync-message",
1056 : G_CALLBACK (cb_bus_sync_message), pipe_h);
1057 :
1058 : /* state change callback */
1059 145 : pipe_h->state_cb.cb = cb;
1060 145 : pipe_h->state_cb.user_data = user_data;
1061 :
1062 : /* iterate elements and prepare element handle */
1063 145 : status = iterate_element (pipe_h, pipeline, is_internal);
1064 145 : if (status != ML_ERROR_NONE) {
1065 0 : _ml_error_report_continue ("ml_pipeline_construct error: ...");
1066 0 : goto failed;
1067 : }
1068 :
1069 : /* finally set pipeline state to PAUSED */
1070 145 : status = ml_pipeline_stop ((ml_pipeline_h) pipe_h);
1071 :
1072 145 : if (status == ML_ERROR_NONE) {
1073 : /**
1074 : * Let's wait until the pipeline state is changed to paused.
1075 : * Otherwise, the following APIs like 'set_property' may incur
1076 : * unintended behaviors. But, don't need to return any error
1077 : * even if this state change is not finished within the timeout,
1078 : * just replying on the caller.
1079 : */
1080 144 : gst_element_get_state (pipeline, NULL, NULL, 10 * GST_MSECOND);
1081 : } else {
1082 1 : _ml_error_report_continue
1083 : ("ml_pipeline_construct error: ml_pipeline_stop has failed with %d return. The pipeline should be able to be stopped when it is constructed.",
1084 : status);
1085 : }
1086 :
1087 148 : failed:
1088 148 : if (status != ML_ERROR_NONE) {
1089 : /* failed to construct the pipeline */
1090 4 : ml_pipeline_destroy ((ml_pipeline_h) pipe_h);
1091 : } else {
1092 144 : *pipe = pipe_h;
1093 : }
1094 :
1095 148 : return status;
1096 : }
1097 :
1098 : /**
1099 : * @brief Construct the pipeline (more info in nnstreamer.h)
1100 : */
1101 : int
1102 148 : ml_pipeline_construct (const char *pipeline_description,
1103 : ml_pipeline_state_cb cb, void *user_data, ml_pipeline_h * pipe)
1104 : {
1105 : /* not an internal pipeline construction */
1106 148 : return construct_pipeline_internal (pipeline_description, cb, user_data, pipe,
1107 : FALSE);
1108 : }
1109 :
1110 : #if defined (__TIZEN__)
1111 : /**
1112 : * @brief Construct the pipeline (Tizen internal, see nnstreamer-tizen-internal.h)
1113 : */
1114 : int
1115 2 : ml_pipeline_construct_internal (const char *pipeline_description,
1116 : ml_pipeline_state_cb cb, void *user_data, ml_pipeline_h * pipe)
1117 : {
1118 : /* Tizen internal pipeline construction */
1119 2 : return construct_pipeline_internal (pipeline_description, cb, user_data, pipe,
1120 : TRUE);
1121 : }
1122 : #endif /* __TIZEN__ */
1123 :
1124 : /**
1125 : * @brief Destroy the pipeline (more info in nnstreamer.h)
1126 : */
1127 : int
1128 143 : ml_pipeline_destroy (ml_pipeline_h pipe)
1129 : {
1130 143 : ml_pipeline *p = pipe;
1131 : GstStateChangeReturn scret;
1132 : GstState state;
1133 143 : guint check_paused_cnt = 0;
1134 :
1135 286 : check_feature_state (ML_FEATURE_INFERENCE);
1136 :
1137 143 : if (p == NULL)
1138 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1139 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h handle instance, usually created by ml_pipeline_construct().");
1140 :
1141 143 : g_mutex_lock (&p->lock);
1142 :
1143 : /* Before changing the state, remove all callbacks. */
1144 143 : p->state_cb.cb = NULL;
1145 :
1146 : /* Destroy registered callback handles and resources */
1147 143 : g_hash_table_destroy (p->namednodes);
1148 143 : g_hash_table_destroy (p->resources);
1149 143 : g_hash_table_destroy (p->pipe_elm_type);
1150 143 : p->namednodes = p->resources = p->pipe_elm_type = NULL;
1151 :
1152 143 : if (p->element) {
1153 : /* Pause the pipeline if it's playing */
1154 140 : scret = gst_element_get_state (p->element, &state, NULL, 10 * GST_MSECOND); /* 10ms */
1155 140 : if (scret != GST_STATE_CHANGE_FAILURE && state == GST_STATE_PLAYING) {
1156 3 : scret = gst_element_set_state (p->element, GST_STATE_PAUSED);
1157 3 : if (scret == GST_STATE_CHANGE_FAILURE) {
1158 0 : g_mutex_unlock (&p->lock);
1159 0 : _ml_error_report_return (ML_ERROR_STREAMS_PIPE,
1160 : "gst_element_get_state() has failed to wait until state changed from PLAYING to PAUSED and returned GST_STATE_CHANGE_FAILURE. For the detail, please check the GStreamer log messages (or dlog messages in Tizen). It is possible that there is a filter or neural network that is taking too much time to finish.");
1161 : }
1162 : }
1163 :
1164 140 : g_mutex_unlock (&p->lock);
1165 239 : while (p->pipe_state == ML_PIPELINE_STATE_PLAYING) {
1166 100 : check_paused_cnt++;
1167 : /** check PAUSED every 1ms */
1168 100 : g_usleep (1000);
1169 100 : if (check_paused_cnt >= WAIT_PAUSED_TIME_LIMIT) {
1170 1 : _ml_error_report
1171 : ("Timeout while waiting for a state change to 'PAUSED' from a 'sync-message' signal from the pipeline. It is possible that there is a filter or neural network that is taking too much time to finish.");
1172 1 : break;
1173 : }
1174 : }
1175 140 : g_mutex_lock (&p->lock);
1176 :
1177 : /* Stop (NULL State) the pipeline */
1178 140 : scret = gst_element_set_state (p->element, GST_STATE_NULL);
1179 140 : if (scret != GST_STATE_CHANGE_SUCCESS) {
1180 0 : g_mutex_unlock (&p->lock);
1181 0 : _ml_error_report_return (ML_ERROR_STREAMS_PIPE,
1182 : "gst_element_set_state to stop the pipeline has failed after trying to stop the pipeline with PAUSE and waiting for stopping. For the detail, please check the GStreamer log messages. It is possible that there is a filter of neural network that is taking too much time to finish.");
1183 : }
1184 :
1185 140 : if (p->bus) {
1186 140 : g_signal_handler_disconnect (p->bus, p->signal_msg);
1187 140 : gst_object_unref (p->bus);
1188 : }
1189 :
1190 140 : gst_object_unref (p->element);
1191 140 : p->element = NULL;
1192 : }
1193 :
1194 143 : g_mutex_unlock (&p->lock);
1195 143 : g_mutex_clear (&p->lock);
1196 :
1197 143 : g_free (p);
1198 143 : return ML_ERROR_NONE;
1199 : }
1200 :
1201 : /**
1202 : * @brief Get the pipeline state (more info in nnstreamer.h)
1203 : */
1204 : int
1205 43 : ml_pipeline_get_state (ml_pipeline_h pipe, ml_pipeline_state_e * state)
1206 : {
1207 43 : ml_pipeline *p = pipe;
1208 : GstState _state;
1209 : GstStateChangeReturn scret;
1210 :
1211 86 : check_feature_state (ML_FEATURE_INFERENCE);
1212 :
1213 43 : if (p == NULL)
1214 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1215 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h handle, which is usually created by ml_pipeline_construct ().");
1216 43 : if (state == NULL)
1217 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1218 : "The parameter, state, is NULL. It should be a valid pointer of ml_pipeline_state_e. E.g., ml_pipeline_state_e state; ml_pipeline_get_state(pipe, &state);");
1219 :
1220 43 : *state = ML_PIPELINE_STATE_UNKNOWN;
1221 :
1222 43 : g_mutex_lock (&p->lock);
1223 43 : scret = gst_element_get_state (p->element, &_state, NULL, GST_MSECOND); /* Do it within 1ms! */
1224 43 : g_mutex_unlock (&p->lock);
1225 :
1226 43 : if (scret == GST_STATE_CHANGE_FAILURE)
1227 0 : _ml_error_report_return (ML_ERROR_STREAMS_PIPE,
1228 : "Failed to get the state of the pipeline. For the detail, please check the GStreamer log messages.");
1229 :
1230 43 : *state = (ml_pipeline_state_e) _state;
1231 43 : return ML_ERROR_NONE;
1232 : }
1233 :
1234 : /****************************************************
1235 : ** NNStreamer Pipeline Start/Stop Control **
1236 : ****************************************************/
1237 : /**
1238 : * @brief Start/Resume the pipeline! (more info in nnstreamer.h)
1239 : */
1240 : int
1241 53 : ml_pipeline_start (ml_pipeline_h pipe)
1242 : {
1243 53 : ml_pipeline *p = pipe;
1244 : GstStateChangeReturn scret;
1245 53 : int status = ML_ERROR_NONE;
1246 :
1247 53 : check_feature_state (ML_FEATURE_INFERENCE);
1248 :
1249 53 : if (p == NULL)
1250 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1251 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h handle, which is usually created by ml_pipeline_construct ().");
1252 :
1253 53 : g_mutex_lock (&p->lock);
1254 :
1255 : /* check the resources when starting the pipeline */
1256 53 : if (g_hash_table_size (p->resources)) {
1257 : GHashTableIter iter;
1258 : gpointer key, value;
1259 :
1260 : /* iterate all handle and acquire res if released */
1261 0 : g_hash_table_iter_init (&iter, p->resources);
1262 0 : while (g_hash_table_iter_next (&iter, &key, &value)) {
1263 0 : if (g_str_has_prefix (key, "tizen")) {
1264 0 : status = get_tizen_resource (pipe, key);
1265 0 : if (status != ML_ERROR_NONE) {
1266 0 : _ml_error_report_continue
1267 : ("Internal API _ml_tizen_get_resource () has failed: Tizen mm resource manager has failed to acquire the resource of '%s'",
1268 : (gchar *) key);
1269 0 : goto done;
1270 : }
1271 : }
1272 : }
1273 : }
1274 :
1275 53 : scret = gst_element_set_state (p->element, GST_STATE_PLAYING);
1276 53 : if (scret == GST_STATE_CHANGE_FAILURE) {
1277 0 : _ml_error_report
1278 : ("Failed to set the state of the pipeline to PLAYING. For the detail, please check the GStreamer log messages.");
1279 0 : status = ML_ERROR_STREAMS_PIPE;
1280 : }
1281 :
1282 53 : done:
1283 53 : g_mutex_unlock (&p->lock);
1284 53 : return status;
1285 : }
1286 :
1287 : /**
1288 : * @brief Pause the pipeline! (more info in nnstreamer.h)
1289 : */
1290 : int
1291 193 : ml_pipeline_stop (ml_pipeline_h pipe)
1292 : {
1293 193 : ml_pipeline *p = pipe;
1294 : GstStateChangeReturn scret;
1295 :
1296 193 : check_feature_state (ML_FEATURE_INFERENCE);
1297 :
1298 193 : if (p == NULL)
1299 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1300 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
1301 :
1302 193 : g_mutex_lock (&p->lock);
1303 193 : scret = gst_element_set_state (p->element, GST_STATE_PAUSED);
1304 193 : g_mutex_unlock (&p->lock);
1305 :
1306 193 : if (scret == GST_STATE_CHANGE_FAILURE)
1307 1 : _ml_error_report_return (ML_ERROR_STREAMS_PIPE,
1308 : "Failed to set the state of the pipeline to PAUSED. For the detail, please check the GStreamer log messages.");
1309 :
1310 192 : return ML_ERROR_NONE;
1311 : }
1312 :
1313 : /**
1314 : * @brief Clears all data and resets the running-time of the pipeline (more info in nnstreamer.h)
1315 : */
1316 : int
1317 2 : ml_pipeline_flush (ml_pipeline_h pipe, bool start)
1318 : {
1319 2 : ml_pipeline *p = pipe;
1320 2 : int status = ML_ERROR_NONE;
1321 :
1322 2 : check_feature_state (ML_FEATURE_INFERENCE);
1323 :
1324 2 : if (p == NULL)
1325 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1326 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
1327 :
1328 1 : _ml_error_report_return_continue_iferr (ml_pipeline_stop (pipe),
1329 : "Failed to stop the pipeline with ml_pipeline_stop (). It has returned %d.",
1330 : _ERRNO);
1331 :
1332 1 : _ml_logi ("The pipeline is stopped, clear all data from the pipeline.");
1333 :
1334 : /* send flush event to pipeline */
1335 1 : g_mutex_lock (&p->lock);
1336 1 : if (!gst_element_send_event (p->element, gst_event_new_flush_start ())) {
1337 0 : _ml_logw ("Error occurs while sending flush_start event.");
1338 : }
1339 :
1340 1 : if (!gst_element_send_event (p->element, gst_event_new_flush_stop (TRUE))) {
1341 0 : _ml_logw ("Error occurs while sending flush_stop event.");
1342 : }
1343 1 : g_mutex_unlock (&p->lock);
1344 :
1345 1 : if (start && status == ML_ERROR_NONE)
1346 1 : status = ml_pipeline_start (pipe);
1347 :
1348 1 : return status;
1349 : }
1350 :
1351 : /****************************************************
1352 : ** NNStreamer Pipeline Sink/Src Control **
1353 : ****************************************************/
1354 : /**
1355 : * @brief Register a callback for sink (more info in nnstreamer.h)
1356 : */
1357 : int
1358 46 : ml_pipeline_sink_register (ml_pipeline_h pipe, const char *sink_name,
1359 : ml_pipeline_sink_cb cb, void *user_data, ml_pipeline_sink_h * h)
1360 : {
1361 : ml_pipeline_element *elem;
1362 46 : ml_pipeline *p = pipe;
1363 : ml_pipeline_common_elem *sink;
1364 46 : int ret = ML_ERROR_NONE;
1365 :
1366 46 : check_feature_state (ML_FEATURE_INFERENCE);
1367 :
1368 46 : if (h == NULL)
1369 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1370 : "The argument, h (ml_pipeline_sink_h), is NULL. It should be a valid pointer to a new ml_pipeline_sink_h instance. E.g., ml_pipeline_sink_h h; ml_pipeline_sink_register (...., &h);");
1371 :
1372 : /* init null */
1373 45 : *h = NULL;
1374 :
1375 45 : if (pipe == NULL)
1376 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1377 : "The argument, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h instance, usually created by ml_pipeline_construct.");
1378 :
1379 44 : if (sink_name == NULL)
1380 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1381 : "The argument, sink_name (const char *), is NULL. It should be a valid string naming the sink handle (h).");
1382 :
1383 43 : if (cb == NULL)
1384 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1385 : "The argument, cb (ml_pipeline_sink_cb), is NULL. It should be a call-back function called for data-sink events.");
1386 :
1387 42 : g_mutex_lock (&p->lock);
1388 42 : elem = g_hash_table_lookup (p->namednodes, sink_name);
1389 :
1390 42 : if (elem == NULL) {
1391 1 : _ml_error_report
1392 : ("There is no element named [%s](sink_name) in the pipeline. Please check your pipeline description.",
1393 : sink_name);
1394 1 : ret = ML_ERROR_INVALID_PARAMETER;
1395 1 : goto unlock_return;
1396 : }
1397 :
1398 41 : if (elem->type != ML_PIPELINE_ELEMENT_SINK &&
1399 5 : elem->type != ML_PIPELINE_ELEMENT_APP_SINK) {
1400 1 : _ml_error_report
1401 : ("The element [%s](sink_name) in the pipeline is not a sink element. Please supply the name of tensor_sink or appsink.",
1402 : sink_name);
1403 1 : ret = ML_ERROR_INVALID_PARAMETER;
1404 1 : goto unlock_return;
1405 : }
1406 :
1407 40 : if (elem->handle_id > 0) {
1408 : /* no need to connect signal to sink element */
1409 1 : _ml_logw ("Sink callback is already registered.");
1410 : } else {
1411 : /* set callback for new data */
1412 39 : if (elem->type == ML_PIPELINE_ELEMENT_SINK) {
1413 : /* tensor_sink */
1414 36 : g_object_set (G_OBJECT (elem->element), "emit-signal", (gboolean) TRUE,
1415 : NULL);
1416 36 : elem->handle_id =
1417 36 : g_signal_connect (elem->element, "new-data",
1418 : G_CALLBACK (cb_sink_event), elem);
1419 : } else {
1420 : /* appsink */
1421 3 : g_object_set (G_OBJECT (elem->element), "emit-signals", (gboolean) TRUE,
1422 : NULL);
1423 3 : elem->handle_id =
1424 3 : g_signal_connect (elem->element, "new-sample",
1425 : G_CALLBACK (cb_appsink_new_sample), elem);
1426 : }
1427 :
1428 39 : if (elem->handle_id == 0) {
1429 0 : _ml_error_report
1430 : ("Failed to connect a signal to the element [%s](sink_name). g_signal_connect has returned NULL.",
1431 : sink_name);
1432 0 : ret = ML_ERROR_STREAMS_PIPE;
1433 0 : goto unlock_return;
1434 : }
1435 : }
1436 :
1437 40 : sink = g_new0 (ml_pipeline_common_elem, 1);
1438 40 : if (sink == NULL) {
1439 0 : _ml_error_report
1440 : ("Failed to allocate memory for the sink handle of %s. Out of memory?",
1441 : sink_name);
1442 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1443 0 : goto unlock_return;
1444 : }
1445 :
1446 40 : sink->callback_info = g_new0 (callback_info_s, 1);
1447 40 : if (sink->callback_info == NULL) {
1448 0 : g_free (sink);
1449 0 : _ml_error_report
1450 : ("Failed to allocate memory for the sink handle of %s. Out of memory?",
1451 : sink_name);
1452 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1453 0 : goto unlock_return;
1454 : }
1455 :
1456 40 : sink->pipe = p;
1457 40 : sink->element = elem;
1458 40 : sink->callback_info->sink_cb = cb;
1459 40 : sink->callback_info->sink_pdata = user_data;
1460 40 : *h = sink;
1461 :
1462 40 : g_mutex_lock (&elem->lock);
1463 :
1464 40 : elem->maxid++;
1465 40 : sink->id = elem->maxid;
1466 40 : elem->handles = g_list_append (elem->handles, sink);
1467 :
1468 40 : g_mutex_unlock (&elem->lock);
1469 :
1470 42 : unlock_return:
1471 42 : g_mutex_unlock (&p->lock);
1472 42 : return ret;
1473 : }
1474 :
1475 : /**
1476 : * @brief Unregister a callback for sink (more info in nnstreamer.h)
1477 : */
1478 : int
1479 13 : ml_pipeline_sink_unregister (ml_pipeline_sink_h h)
1480 : {
1481 13 : handle_init (sink, h);
1482 :
1483 13 : if (elem->handle_id > 0) {
1484 12 : g_signal_handler_disconnect (elem->element, elem->handle_id);
1485 12 : elem->handle_id = 0;
1486 : }
1487 :
1488 13 : elem->handles = g_list_remove (elem->handles, sink);
1489 13 : free_element_handle (sink);
1490 :
1491 13 : handle_exit (h);
1492 : }
1493 :
1494 : /**
1495 : * @brief Parse tensors info of src element.
1496 : */
1497 : static int
1498 12144 : ml_pipeline_src_parse_tensors_info (ml_pipeline_element * elem)
1499 : {
1500 12144 : GstCaps *caps = NULL;
1501 12144 : gboolean found = FALSE, flexible = FALSE;
1502 :
1503 12144 : if (elem->src == NULL) {
1504 41 : elem->src = gst_element_get_static_pad (elem->element, "src");
1505 : }
1506 :
1507 12144 : if (elem->src == NULL) {
1508 0 : _ml_error_report
1509 : ("Failed to get the src pad of the element[%s]. The designated source element does not have available src pad? For the detail, please check the GStreamer log messages.",
1510 : elem->name);
1511 12144 : return ML_ERROR_STREAMS_PIPE;
1512 : }
1513 :
1514 : /* If caps is given, use it. e.g. Use cap "image/png" when the pipeline is */
1515 : /* given as "appsrc caps=image/png ! pngdec ! ... " */
1516 12144 : caps = gst_pad_get_current_caps (elem->src);
1517 12144 : if (!caps)
1518 12080 : caps = gst_pad_get_allowed_caps (elem->src);
1519 :
1520 12144 : if (!caps) {
1521 0 : _ml_logw
1522 : ("Cannot find caps. The pipeline is not yet negotiated for src element [%s].",
1523 : elem->name);
1524 0 : gst_object_unref (elem->src);
1525 0 : elem->src = NULL;
1526 0 : return ML_ERROR_TRY_AGAIN;
1527 : }
1528 :
1529 12144 : found = get_tensors_info_from_caps (caps, &elem->tensors_info, &flexible);
1530 :
1531 12144 : if (found) {
1532 12138 : elem->is_flexible_tensor = flexible;
1533 : } else {
1534 6 : if (gst_caps_is_fixed (caps)) {
1535 5 : GstStructure *st = gst_caps_get_structure (caps, 0);
1536 5 : elem->is_media_stream = !gst_structure_is_tensor_stream (st);
1537 : }
1538 : }
1539 :
1540 12144 : gst_caps_unref (caps);
1541 12144 : return ML_ERROR_NONE;
1542 : }
1543 :
1544 : /**
1545 : * @brief Get a handle to operate a src (more info in nnstreamer.h)
1546 : */
1547 : int
1548 48 : ml_pipeline_src_get_handle (ml_pipeline_h pipe, const char *src_name,
1549 : ml_pipeline_src_h * h)
1550 : {
1551 48 : ml_pipeline *p = pipe;
1552 : ml_pipeline_element *elem;
1553 : ml_pipeline_common_elem *src;
1554 48 : int ret = ML_ERROR_NONE;
1555 :
1556 48 : check_feature_state (ML_FEATURE_INFERENCE);
1557 :
1558 48 : if (h == NULL)
1559 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1560 : "The parameter, h (ml_pipeline_src_h), is NULL. It should be a valid pointer to a new (or to be cleared) instance. E.g., ml_pipeline_src_h h; ml_pipeline_src_get_handle (..., &h);");
1561 :
1562 : /* init null */
1563 47 : *h = NULL;
1564 :
1565 47 : if (pipe == NULL)
1566 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1567 : "The parameter, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
1568 :
1569 46 : if (src_name == NULL)
1570 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1571 : "The parameter, src_name (const char *), is NULL. This string is the name of source element (appsrc) you want to push data stream from your application threads.");
1572 :
1573 45 : g_mutex_lock (&p->lock);
1574 :
1575 45 : elem = g_hash_table_lookup (p->namednodes, src_name);
1576 :
1577 45 : if (elem == NULL) {
1578 1 : _ml_error_report
1579 : ("Cannot find the name, '%s': there is no element named [%s] in the given pipeline.",
1580 : src_name, src_name);
1581 1 : ret = ML_ERROR_INVALID_PARAMETER;
1582 1 : goto unlock_return;
1583 : }
1584 :
1585 44 : if (elem->type != ML_PIPELINE_ELEMENT_APP_SRC) {
1586 1 : _ml_error_report
1587 : ("The element designated by '%s' is not a source element (appsrc). Please provide a name of source element for ml_pipeline_src_get_handle API.",
1588 : src_name);
1589 1 : ret = ML_ERROR_INVALID_PARAMETER;
1590 1 : goto unlock_return;
1591 : }
1592 :
1593 43 : src = *h = g_new0 (ml_pipeline_common_elem, 1);
1594 43 : if (src == NULL) {
1595 0 : _ml_error_report
1596 : ("Failed to allocate the src handle for %s. Out of memory?", src_name);
1597 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1598 0 : goto unlock_return;
1599 : }
1600 :
1601 43 : src->pipe = p;
1602 43 : src->element = elem;
1603 :
1604 43 : g_mutex_lock (&elem->lock);
1605 :
1606 43 : elem->maxid++;
1607 43 : src->id = elem->maxid;
1608 43 : elem->handles = g_list_append (elem->handles, src);
1609 :
1610 43 : ml_pipeline_src_parse_tensors_info (elem);
1611 43 : g_mutex_unlock (&elem->lock);
1612 :
1613 45 : unlock_return:
1614 45 : g_mutex_unlock (&p->lock);
1615 :
1616 45 : return ret;
1617 : }
1618 :
1619 : /**
1620 : * @brief Close a src node (more info in nnstreamer.h)
1621 : */
1622 : int
1623 7 : ml_pipeline_src_release_handle (ml_pipeline_src_h h)
1624 : {
1625 7 : handle_init (src, h);
1626 :
1627 7 : elem->handles = g_list_remove (elem->handles, src);
1628 7 : free_element_handle (src);
1629 :
1630 7 : handle_exit (h);
1631 : }
1632 :
1633 : /**
1634 : * @brief Push a data frame to a src (more info in nnstreamer.h)
1635 : */
1636 : int
1637 6093 : ml_pipeline_src_input_data (ml_pipeline_src_h h, ml_tensors_data_h data,
1638 : ml_pipeline_buf_policy_e policy)
1639 : {
1640 : GstBuffer *buffer;
1641 : GstMemory *mem, *tmp;
1642 : gpointer mem_data;
1643 : gsize mem_size;
1644 : GstFlowReturn gret;
1645 : GstTensorsInfo gst_info;
1646 : ml_tensors_data_s *_data;
1647 : unsigned int i;
1648 :
1649 12186 : handle_init (src, h);
1650 :
1651 6093 : _data = (ml_tensors_data_s *) data;
1652 6093 : if (!_data) {
1653 1 : _ml_error_report
1654 : ("The given parameter, data (ml_tensors_data_h), is NULL. It should be a valid ml_tensor_data_h instance, which is usually created by ml_tensors_data_create().");
1655 1 : ret = ML_ERROR_INVALID_PARAMETER;
1656 1 : goto unlock_return;
1657 : }
1658 6092 : G_LOCK_UNLESS_NOLOCK (*_data);
1659 :
1660 6092 : if (_data->num_tensors < 1 || _data->num_tensors > ML_TENSOR_SIZE_LIMIT) {
1661 0 : _ml_error_report
1662 : ("The number of tensors of the given data (ml_tensors_data_h) is invalid. The number of tensors of data is %u. It should be between 1 and %u.",
1663 : _data->num_tensors, ML_TENSOR_SIZE_LIMIT);
1664 0 : ret = ML_ERROR_INVALID_PARAMETER;
1665 0 : goto dont_destroy_data;
1666 : }
1667 :
1668 6092 : ret = ml_pipeline_src_parse_tensors_info (elem);
1669 :
1670 6092 : if (ret != ML_ERROR_NONE) {
1671 0 : if (ret == ML_ERROR_TRY_AGAIN)
1672 0 : _ml_error_report_continue
1673 : ("The pipeline is not ready to accept input streams. The input is ignored.");
1674 : else
1675 0 : _ml_error_report_continue
1676 : ("The pipeline is either not ready to accept input streams, yet, or does not have appropriate source elements to accept input streams.");
1677 0 : goto dont_destroy_data;
1678 : }
1679 :
1680 6092 : if (!elem->is_media_stream && !elem->is_flexible_tensor) {
1681 6085 : if (elem->tensors_info.num_tensors != _data->num_tensors) {
1682 0 : _ml_error_report
1683 : ("The src push of [%s] cannot be handled because the number of tensors in a frame mismatches. %u != %u",
1684 : elem->name, elem->tensors_info.num_tensors, _data->num_tensors);
1685 :
1686 0 : ret = ML_ERROR_INVALID_PARAMETER;
1687 0 : goto dont_destroy_data;
1688 : }
1689 :
1690 12323 : for (i = 0; i < _data->num_tensors; i++) {
1691 6240 : size_t sz = gst_tensors_info_get_size (&elem->tensors_info, i);
1692 :
1693 6240 : if (sz != _data->tensors[i].size) {
1694 2 : _ml_error_report
1695 : ("The given input tensor size (%d'th, %zu bytes) mismatches the source pad (%zu bytes)",
1696 : i, _data->tensors[i].size, sz);
1697 :
1698 2 : ret = ML_ERROR_INVALID_PARAMETER;
1699 2 : goto dont_destroy_data;
1700 : }
1701 : }
1702 : }
1703 :
1704 : /* Create buffer to be pushed from buf[] */
1705 6090 : buffer = gst_buffer_new ();
1706 6090 : _ml_tensors_info_copy_from_ml (&gst_info, _data->info);
1707 :
1708 12341 : for (i = 0; i < _data->num_tensors; i++) {
1709 : GstTensorInfo *_gst_tensor_info =
1710 6251 : gst_tensors_info_get_nth_info (&gst_info, i);
1711 6251 : mem_data = _data->tensors[i].data;
1712 6251 : mem_size = _data->tensors[i].size;
1713 :
1714 6251 : mem = tmp = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY,
1715 : mem_data, mem_size, 0, mem_size, mem_data,
1716 : (policy == ML_PIPELINE_BUF_POLICY_AUTO_FREE) ? g_free : NULL);
1717 :
1718 : /* flex tensor, append header. */
1719 6251 : if (elem->is_flexible_tensor) {
1720 : GstTensorMetaInfo meta;
1721 :
1722 9 : gst_tensor_info_convert_to_meta (_gst_tensor_info, &meta);
1723 :
1724 9 : mem = gst_tensor_meta_info_append_header (&meta, tmp);
1725 9 : gst_memory_unref (tmp);
1726 : }
1727 :
1728 6251 : gst_tensor_buffer_append_memory (buffer, mem, _gst_tensor_info);
1729 : /** @todo Verify that gst_buffer_append lists tensors/gstmem in the correct order */
1730 : }
1731 :
1732 6090 : gst_tensors_info_free (&gst_info);
1733 :
1734 : /* Unlock if it's not auto-free. We do not know when it'll be freed. */
1735 6090 : if (policy != ML_PIPELINE_BUF_POLICY_AUTO_FREE)
1736 55 : G_UNLOCK_UNLESS_NOLOCK (*_data);
1737 :
1738 : /* Push the data! */
1739 6090 : gret = gst_app_src_push_buffer (GST_APP_SRC (elem->element), buffer);
1740 :
1741 : /* Free data ptr if buffer policy is auto-free */
1742 6090 : if (policy == ML_PIPELINE_BUF_POLICY_AUTO_FREE) {
1743 6035 : G_UNLOCK_UNLESS_NOLOCK (*_data);
1744 6035 : _ml_tensors_data_destroy_internal (_data, FALSE);
1745 6035 : _data = NULL;
1746 : }
1747 :
1748 6090 : if (gret == GST_FLOW_FLUSHING) {
1749 0 : _ml_logw
1750 : ("The pipeline is not in PAUSED/PLAYING. The input may be ignored.");
1751 0 : ret = ML_ERROR_TRY_AGAIN;
1752 6090 : } else if (gret == GST_FLOW_EOS) {
1753 0 : _ml_logw ("THe pipeline is in EOS state. The input is ignored.");
1754 0 : ret = ML_ERROR_STREAMS_PIPE;
1755 : }
1756 :
1757 6090 : goto unlock_return;
1758 :
1759 2 : dont_destroy_data:
1760 2 : G_UNLOCK_UNLESS_NOLOCK (*_data);
1761 :
1762 6093 : handle_exit (h);
1763 : }
1764 :
1765 : /**
1766 : * @brief Internal function to fetch ml_pipeline_src_callbacks_s pointer
1767 : */
1768 : static ml_pipeline_src_callbacks_s *
1769 6005 : get_app_src_callback (ml_pipeline_common_elem * src_h, void **data)
1770 : {
1771 6005 : ml_pipeline_src_callbacks_s *src_cb = NULL;
1772 :
1773 6005 : if (src_h->callback_info) {
1774 6005 : src_cb = &src_h->callback_info->src_cb;
1775 6005 : *data = src_h->callback_info->src_pdata;
1776 : }
1777 :
1778 6005 : return src_cb;
1779 : }
1780 :
1781 : /**
1782 : * @brief Internal function for appsrc callback - need_data.
1783 : */
1784 : static void
1785 6005 : _pipe_src_cb_need_data (GstAppSrc * src, guint length, gpointer user_data)
1786 : {
1787 : ml_pipeline_common_elem *src_h;
1788 6005 : ml_pipeline_src_callbacks_s *src_cb = NULL;
1789 6005 : void *pdata = NULL;
1790 :
1791 6005 : src_h = (ml_pipeline_common_elem *) user_data;
1792 6005 : if (!src_h)
1793 0 : return;
1794 :
1795 6005 : src_cb = get_app_src_callback (src_h, &pdata);
1796 6005 : if (src_cb && src_cb->need_data)
1797 6005 : src_cb->need_data (src_h, length, pdata);
1798 : }
1799 :
1800 : /**
1801 : * @brief Internal function for appsrc callback - enough_data.
1802 : */
1803 : static void
1804 0 : _pipe_src_cb_enough_data (GstAppSrc * src, gpointer user_data)
1805 : {
1806 : ml_pipeline_common_elem *src_h;
1807 0 : ml_pipeline_src_callbacks_s *src_cb = NULL;
1808 0 : void *pdata = NULL;
1809 :
1810 0 : src_h = (ml_pipeline_common_elem *) user_data;
1811 0 : if (!src_h)
1812 0 : return;
1813 :
1814 0 : src_cb = get_app_src_callback (src_h, &pdata);
1815 0 : if (src_cb && src_cb->enough_data)
1816 0 : src_cb->enough_data (src_h, pdata);
1817 : }
1818 :
1819 : /**
1820 : * @brief Internal function for appsrc callback - seek_data.
1821 : */
1822 : static gboolean
1823 0 : _pipe_src_cb_seek_data (GstAppSrc * src, guint64 offset, gpointer user_data)
1824 : {
1825 : ml_pipeline_common_elem *src_h;
1826 0 : ml_pipeline_src_callbacks_s *src_cb = NULL;
1827 0 : void *pdata = NULL;
1828 :
1829 0 : src_h = (ml_pipeline_common_elem *) user_data;
1830 0 : if (!src_h)
1831 0 : return TRUE;
1832 :
1833 0 : src_cb = get_app_src_callback (src_h, &pdata);
1834 0 : if (src_cb && src_cb->seek_data)
1835 0 : src_cb->seek_data (src_h, offset, pdata);
1836 :
1837 0 : return TRUE;
1838 : }
1839 :
1840 : /**
1841 : * @brief Register callbacks for src events (more info in nnstreamer.h)
1842 : */
1843 : int
1844 4 : ml_pipeline_src_set_event_cb (ml_pipeline_src_h src_handle,
1845 : ml_pipeline_src_callbacks_s * cb, void *user_data)
1846 : {
1847 4 : GstAppSrcCallbacks appsrc_cb = { 0, };
1848 :
1849 7 : handle_init (src, src_handle);
1850 :
1851 3 : if (cb == NULL) {
1852 1 : ret = ML_ERROR_INVALID_PARAMETER;
1853 1 : goto unlock_return;
1854 : }
1855 :
1856 2 : if (src->callback_info == NULL)
1857 2 : src->callback_info = g_new0 (callback_info_s, 1);
1858 2 : if (src->callback_info == NULL) {
1859 0 : _ml_error_report
1860 : ("Failed to allocate memory of the callback info for %s. Out of memory?",
1861 : elem->name);
1862 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1863 0 : goto unlock_return;
1864 : }
1865 :
1866 2 : src->callback_info->src_cb = *cb;
1867 2 : src->callback_info->src_pdata = user_data;
1868 :
1869 2 : appsrc_cb.need_data = _pipe_src_cb_need_data;
1870 2 : appsrc_cb.enough_data = _pipe_src_cb_enough_data;
1871 2 : appsrc_cb.seek_data = _pipe_src_cb_seek_data;
1872 :
1873 2 : gst_app_src_set_callbacks (GST_APP_SRC (elem->element), &appsrc_cb,
1874 : src_handle, NULL);
1875 :
1876 3 : handle_exit (src_handle);
1877 : }
1878 :
1879 : /**
1880 : * @brief Gets a handle for the tensors metadata of given src node.
1881 : */
1882 : int
1883 6009 : ml_pipeline_src_get_tensors_info (ml_pipeline_src_h h, ml_tensors_info_h * info)
1884 : {
1885 6009 : handle_init (src, h);
1886 :
1887 6009 : if (info == NULL) {
1888 0 : _ml_error_report
1889 : ("The parameter, info (ml_tensors_info_h *), is NULL. It should be a valid pointer to a ml_tensors_info_h instance, which is usually created by ml_tensors_info_create().");
1890 0 : ret = ML_ERROR_INVALID_PARAMETER;
1891 0 : goto unlock_return;
1892 : }
1893 :
1894 6009 : ret = ml_pipeline_src_parse_tensors_info (elem);
1895 :
1896 6009 : if (ret == ML_ERROR_NONE) {
1897 6009 : ret = _ml_tensors_info_create_from_gst (info, &elem->tensors_info);
1898 : } else {
1899 0 : _ml_error_report_continue
1900 : ("ml_pipeline_src_parse_tensors_info () has returned error; it cannot fetch input tensor info (metadata of input stream) for the given ml_pipeline_src_h handle (h). ml_pipeline_src_get_tensors_info () cannot continue.");
1901 : }
1902 :
1903 6009 : handle_exit (h);
1904 : }
1905 :
1906 : /****************************************************
1907 : ** NNStreamer Pipeline Switch/Valve Control **
1908 : ****************************************************/
1909 :
1910 : /**
1911 : * @brief Get a handle to operate a selector (more info in nnstreamer.h)
1912 : */
1913 : int
1914 10 : ml_pipeline_switch_get_handle (ml_pipeline_h pipe, const char *switch_name,
1915 : ml_pipeline_switch_e * type, ml_pipeline_switch_h * h)
1916 : {
1917 : ml_pipeline_element *elem;
1918 10 : ml_pipeline *p = pipe;
1919 : ml_pipeline_common_elem *swtc;
1920 10 : int ret = ML_ERROR_NONE;
1921 :
1922 10 : check_feature_state (ML_FEATURE_INFERENCE);
1923 :
1924 10 : if (h == NULL)
1925 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1926 : "The parameter, h (ml_pipeline_switch_h *), is NULL. It should be a new or to-be-cleared instance of ml_pipeline_switch_h. E.g., ml_pipeline_switch_h h; ml_pipeline_switch_get_handle (..., &h);");
1927 :
1928 : /* init null */
1929 9 : *h = NULL;
1930 :
1931 9 : if (pipe == NULL)
1932 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1933 : "The parameter, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h pipeline instance, which is usually created by ml_pipeline_construct().");
1934 :
1935 8 : if (switch_name == NULL)
1936 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1937 : "The parameter, switch_name, is NULL. It should be a valid string of the corresponding name of a switch element.");
1938 :
1939 7 : g_mutex_lock (&p->lock);
1940 7 : elem = g_hash_table_lookup (p->namednodes, switch_name);
1941 :
1942 7 : if (elem == NULL) {
1943 1 : _ml_error_report
1944 : ("The parameter, switch_name (%s), is invalid. An element with the name, '%s', cannot be found in the supplied pipeline (pipe)",
1945 : switch_name, switch_name);
1946 1 : ret = ML_ERROR_INVALID_PARAMETER;
1947 1 : goto unlock_return;
1948 : }
1949 :
1950 6 : if (elem->type == ML_PIPELINE_ELEMENT_SWITCH_INPUT) {
1951 4 : if (type)
1952 1 : *type = ML_PIPELINE_SWITCH_INPUT_SELECTOR;
1953 2 : } else if (elem->type == ML_PIPELINE_ELEMENT_SWITCH_OUTPUT) {
1954 1 : if (type)
1955 1 : *type = ML_PIPELINE_SWITCH_OUTPUT_SELECTOR;
1956 : } else {
1957 1 : _ml_error_report
1958 : ("An element with the given name, '%s', is found; however, it is not a 'switch' element. A switch-handle cannot be fetched from a non-switch element. It should be either input-selector or output-selector.",
1959 : switch_name);
1960 1 : ret = ML_ERROR_INVALID_PARAMETER;
1961 1 : goto unlock_return;
1962 : }
1963 :
1964 5 : swtc = *h = g_new0 (ml_pipeline_common_elem, 1);
1965 5 : if (swtc == NULL) {
1966 0 : _ml_error_report
1967 : ("Failed to allocate memory of the switch handle, %s. Out of memory?",
1968 : switch_name);
1969 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1970 0 : goto unlock_return;
1971 : }
1972 :
1973 5 : swtc->pipe = p;
1974 5 : swtc->element = elem;
1975 :
1976 5 : g_mutex_lock (&elem->lock);
1977 :
1978 5 : elem->maxid++;
1979 5 : swtc->id = elem->maxid;
1980 5 : elem->handles = g_list_append (elem->handles, swtc);
1981 :
1982 5 : g_mutex_unlock (&elem->lock);
1983 :
1984 7 : unlock_return:
1985 7 : g_mutex_unlock (&p->lock);
1986 7 : return ret;
1987 : }
1988 :
1989 : /**
1990 : * @brief Close the given switch handle (more info in nnstreamer.h)
1991 : */
1992 : int
1993 5 : ml_pipeline_switch_release_handle (ml_pipeline_switch_h h)
1994 : {
1995 5 : handle_init (swtc, h);
1996 :
1997 5 : elem->handles = g_list_remove (elem->handles, swtc);
1998 5 : free_element_handle (swtc);
1999 :
2000 5 : handle_exit (h);
2001 : }
2002 :
2003 : /**
2004 : * @brief Control the switch (more info in nnstreamer.h)
2005 : */
2006 : int
2007 5 : ml_pipeline_switch_select (ml_pipeline_switch_h h, const char *pad_name)
2008 : {
2009 : GstPad *active_pad, *new_pad;
2010 : gchar *active_name;
2011 :
2012 9 : handle_init (swtc, h);
2013 :
2014 4 : if (pad_name == NULL) {
2015 1 : _ml_error_report
2016 : ("The parameter, pad_name (const char *), is NULL. It should be a valid name of a pad (GSTPAD) in the given switch, h.");
2017 1 : ret = ML_ERROR_INVALID_PARAMETER;
2018 1 : goto unlock_return;
2019 : }
2020 :
2021 3 : g_object_get (G_OBJECT (elem->element), "active-pad", &active_pad, NULL);
2022 3 : active_name = gst_pad_get_name (active_pad);
2023 :
2024 3 : if (g_strcmp0 (pad_name, active_name) == 0) {
2025 0 : _ml_logi ("Switch is called, but there is no effective changes: %s->%s.",
2026 : active_name, pad_name);
2027 0 : g_free (active_name);
2028 0 : gst_object_unref (active_pad);
2029 :
2030 0 : goto unlock_return;
2031 : }
2032 :
2033 3 : g_free (active_name);
2034 3 : gst_object_unref (active_pad);
2035 :
2036 3 : new_pad = gst_element_get_static_pad (elem->element, pad_name);
2037 3 : if (new_pad == NULL) {
2038 : /* Not Found! */
2039 1 : _ml_error_report
2040 : ("Cannot find the pad, [%s], from the switch, [%s]. Please check the pad name. You may use ml_pipeline_switch_pad_list() to fetch the valid pad names.",
2041 : pad_name, elem->name);
2042 1 : ret = ML_ERROR_INVALID_PARAMETER;
2043 1 : goto unlock_return;
2044 : }
2045 :
2046 2 : g_object_set (G_OBJECT (elem->element), "active-pad", new_pad, NULL);
2047 2 : gst_object_unref (new_pad);
2048 :
2049 2 : _ml_logi ("Switched to [%s] successfully at switch [%s].", pad_name,
2050 : elem->name);
2051 :
2052 4 : handle_exit (h);
2053 : }
2054 :
2055 : /**
2056 : * @brief Gets the pad names of a switch.
2057 : */
2058 : int
2059 2 : ml_pipeline_switch_get_pad_list (ml_pipeline_switch_h h, char ***list)
2060 : {
2061 : GstIterator *it;
2062 2 : GValue item = G_VALUE_INIT;
2063 2 : gboolean done = FALSE;
2064 2 : GList *dllist = NULL;
2065 : GstPad *pad;
2066 2 : int counter = 0;
2067 :
2068 4 : handle_init (swtc, h);
2069 :
2070 2 : if (list == NULL) {
2071 0 : _ml_error_report
2072 : ("The parameter, list (char ***), is NULL. It should be a valid pointer to store a list of strings. E.g., char **list; ml_pipeline_switch_get_pad_list (h, &list);");
2073 0 : ret = ML_ERROR_INVALID_PARAMETER;
2074 0 : goto unlock_return;
2075 : }
2076 :
2077 : /* init null */
2078 2 : *list = NULL;
2079 :
2080 2 : if (elem->type == ML_PIPELINE_ELEMENT_SWITCH_INPUT)
2081 1 : it = gst_element_iterate_sink_pads (elem->element);
2082 1 : else if (elem->type == ML_PIPELINE_ELEMENT_SWITCH_OUTPUT)
2083 1 : it = gst_element_iterate_src_pads (elem->element);
2084 : else {
2085 0 : _ml_error_report
2086 : ("The element, [%s], is supposed to be input/output switch, but it is not. Internal data structure is broken.",
2087 : elem->name);
2088 0 : ret = ML_ERROR_STREAMS_PIPE;
2089 0 : goto unlock_return;
2090 : }
2091 :
2092 8 : while (!done) {
2093 6 : switch (gst_iterator_next (it, &item)) {
2094 4 : case GST_ITERATOR_OK:
2095 4 : pad = GST_PAD (g_value_get_object (&item));
2096 4 : dllist = g_list_append (dllist, gst_pad_get_name (pad));
2097 4 : counter++;
2098 4 : g_value_reset (&item);
2099 4 : break;
2100 0 : case GST_ITERATOR_RESYNC:
2101 0 : g_list_free_full (dllist, g_free); /* This frees all strings as well */
2102 0 : dllist = NULL;
2103 0 : counter = 0;
2104 0 : gst_iterator_resync (it);
2105 0 : break;
2106 0 : case GST_ITERATOR_ERROR:
2107 0 : _ml_error_report
2108 : ("Cannot access the list of pad properly of a switch, [%s]. Internal data structure is broken?",
2109 : elem->name);
2110 0 : ret = ML_ERROR_STREAMS_PIPE;
2111 0 : break;
2112 2 : case GST_ITERATOR_DONE:
2113 2 : done = TRUE;
2114 2 : break;
2115 : }
2116 : }
2117 :
2118 2 : gst_iterator_free (it);
2119 :
2120 : /* There has been no error with that "while" loop. */
2121 2 : if (ret == ML_ERROR_NONE) {
2122 2 : int i = 0;
2123 : GList *l;
2124 :
2125 2 : *list = g_malloc0 (sizeof (char *) * (counter + 1));
2126 2 : if (*list == NULL) {
2127 0 : _ml_error_report
2128 : ("Failed to allocate memory for pad list (parameter list). Out of memory?");
2129 0 : ret = ML_ERROR_OUT_OF_MEMORY;
2130 0 : g_list_free_full (dllist, g_free);
2131 0 : goto unlock_return;
2132 : }
2133 :
2134 6 : for (l = dllist; l != NULL; l = l->next) {
2135 4 : (*list)[i] = l->data; /* Allocated by gst_pad_get_name(). Caller has to free it */
2136 4 : i++;
2137 :
2138 4 : if (i > counter) {
2139 0 : g_list_free_full (dllist, g_free); /* This frees all strings as well */
2140 0 : g_free (*list);
2141 0 : *list = NULL;
2142 :
2143 0 : _ml_error_report
2144 : ("Internal data inconsistency. This could be a bug in nnstreamer. Switch [%s].",
2145 : elem->name);
2146 0 : ret = ML_ERROR_STREAMS_PIPE;
2147 0 : goto unlock_return;
2148 : }
2149 : }
2150 : }
2151 2 : g_list_free (dllist); /* This does not free the strings.. fortunately. */
2152 :
2153 2 : handle_exit (h);
2154 : }
2155 :
2156 : /**
2157 : * @brief Get a handle to operate a Valve (more info in nnstreamer.h)
2158 : */
2159 : int
2160 6 : ml_pipeline_valve_get_handle (ml_pipeline_h pipe, const char *valve_name,
2161 : ml_pipeline_valve_h * h)
2162 : {
2163 : ml_pipeline_element *elem;
2164 6 : ml_pipeline *p = pipe;
2165 : ml_pipeline_common_elem *valve;
2166 6 : int ret = ML_ERROR_NONE;
2167 :
2168 6 : check_feature_state (ML_FEATURE_INFERENCE);
2169 :
2170 6 : if (h == NULL)
2171 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2172 : "The parameter, h (ml_pipeline_valve_h), is NULL. It should be a valid pointer of ml_pipeline_valve_h. E.g., ml_pipeline_valve_h h; ml_pipeline_valve_get_handle (..., &h);");
2173 :
2174 : /* init null */
2175 5 : *h = NULL;
2176 :
2177 5 : if (pipe == NULL)
2178 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2179 : "The parameter, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
2180 :
2181 4 : if (valve_name == NULL)
2182 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2183 : "The parameter, valve_name (const char *), is NULL. It should be a valid string of the valve name.");
2184 :
2185 3 : g_mutex_lock (&p->lock);
2186 3 : elem = g_hash_table_lookup (p->namednodes, valve_name);
2187 :
2188 3 : if (elem == NULL) {
2189 1 : _ml_error_report
2190 : ("Cannot find the valve with the given name, '%s', in the pipeline. There is no element in the pipeline with such a name. Please check if you have a value with the appropriate name.",
2191 : valve_name);
2192 1 : ret = ML_ERROR_INVALID_PARAMETER;
2193 1 : goto unlock_return;
2194 : }
2195 :
2196 2 : if (elem->type != ML_PIPELINE_ELEMENT_VALVE) {
2197 1 : _ml_error_report
2198 : ("Cannot find the value with the given name, '%s', in the pipeline. There is an element with such a name; however, the element is not a valve. Please correct the names of element in the pipeline.",
2199 : valve_name);
2200 1 : ret = ML_ERROR_INVALID_PARAMETER;
2201 1 : goto unlock_return;
2202 : }
2203 :
2204 1 : valve = *h = g_new0 (ml_pipeline_common_elem, 1);
2205 1 : if (valve == NULL) {
2206 0 : _ml_error_report
2207 : ("Cannot allocate memory for valve handle of %s. Out of memory?",
2208 : valve_name);
2209 0 : ret = ML_ERROR_OUT_OF_MEMORY;
2210 0 : goto unlock_return;
2211 : }
2212 :
2213 1 : valve->pipe = p;
2214 1 : valve->element = elem;
2215 :
2216 1 : g_mutex_lock (&elem->lock);
2217 :
2218 1 : elem->maxid++;
2219 1 : valve->id = elem->maxid;
2220 1 : elem->handles = g_list_append (elem->handles, valve);
2221 :
2222 1 : g_mutex_unlock (&elem->lock);
2223 :
2224 3 : unlock_return:
2225 3 : g_mutex_unlock (&p->lock);
2226 3 : return ret;
2227 : }
2228 :
2229 : /**
2230 : * @brief Close the given valve handle (more info in nnstreamer.h)
2231 : */
2232 : int
2233 1 : ml_pipeline_valve_release_handle (ml_pipeline_valve_h h)
2234 : {
2235 1 : handle_init (valve, h);
2236 :
2237 1 : elem->handles = g_list_remove (elem->handles, valve);
2238 1 : free_element_handle (valve);
2239 :
2240 1 : handle_exit (h);
2241 : }
2242 :
2243 : /**
2244 : * @brief Control the valve with the given handle (more info in nnstreamer.h)
2245 : */
2246 : int
2247 2 : ml_pipeline_valve_set_open (ml_pipeline_valve_h h, bool open)
2248 : {
2249 2 : gboolean drop = FALSE;
2250 4 : handle_init (valve, h);
2251 :
2252 2 : g_object_get (G_OBJECT (elem->element), "drop", &drop, NULL);
2253 :
2254 2 : if ((open != false) != (drop != FALSE)) {
2255 : /* Nothing to do */
2256 0 : _ml_logi ("Valve is called, but there is no effective changes");
2257 0 : goto unlock_return;
2258 : }
2259 :
2260 2 : drop = (open) ? FALSE : TRUE;
2261 2 : g_object_set (G_OBJECT (elem->element), "drop", drop, NULL);
2262 :
2263 2 : handle_exit (h);
2264 : }
2265 :
2266 : /********************************************************
2267 : ** NNStreamer Element Property Control in Pipeline **
2268 : ********************************************************/
2269 : /**
2270 : * @brief Gets an element handle in NNStreamer pipelines to control its properties.
2271 : */
2272 : int
2273 64 : ml_pipeline_element_get_handle (ml_pipeline_h pipe, const char *element_name,
2274 : ml_pipeline_element_h * elem_h)
2275 : {
2276 64 : int ret = ML_ERROR_NONE;
2277 : ml_pipeline_element *elem;
2278 : ml_pipeline_common_elem *common_elem;
2279 64 : ml_pipeline *p = pipe;
2280 :
2281 : /* Check input parameter */
2282 64 : if (pipe == NULL)
2283 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2284 : "The parameter, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
2285 :
2286 64 : if (element_name == NULL)
2287 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2288 : "The parameter, element_name (const char *), is NULL. It should be a valid string of the element name to be searched.");
2289 :
2290 63 : if (elem_h == NULL)
2291 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2292 : "The parameter, elem_h (ml_pipeline_element_h), is NULL. It should be a valid pointer of ml_pipeline_element_h. E.g., ml_pipeline_element_h eh; ml_pipeline_element_get_handle (..., &eh);");
2293 63 : *elem_h = NULL;
2294 :
2295 63 : g_mutex_lock (&p->lock);
2296 :
2297 : /* 1. Search element in lookup table first */
2298 63 : elem = g_hash_table_lookup (p->namednodes, element_name);
2299 63 : if (elem == NULL) {
2300 : /* 2. Search element in pipeline itself */
2301 : GstElement *gst_elem;
2302 :
2303 49 : gst_elem = gst_bin_get_by_name (GST_BIN (p->element), element_name);
2304 49 : if (gst_elem == NULL) {
2305 1 : _ml_error_report
2306 : ("Cannot find the element with the given name, '%s', in the pipeline. There is no element in the pipeline with such a name. Please check if you have an element with the appropriate name.",
2307 : element_name);
2308 1 : ret = ML_ERROR_INVALID_PARAMETER;
2309 1 : goto unlock_return;
2310 : }
2311 :
2312 : /* Caching for next search */
2313 48 : elem = construct_element (gst_elem, pipe, element_name,
2314 : ML_PIPELINE_ELEMENT_COMMON);
2315 48 : if (elem == NULL) {
2316 0 : _ml_error_report
2317 : ("Cannot allocate memory for element handle of %s. Out of memory?",
2318 : element_name);
2319 0 : ret = ML_ERROR_OUT_OF_MEMORY;
2320 0 : goto unlock_return;
2321 : }
2322 48 : g_hash_table_insert (p->namednodes, g_strdup (element_name), elem);
2323 : }
2324 :
2325 : /* Type checking */
2326 62 : if (elem->type == ML_PIPELINE_ELEMENT_UNKNOWN) {
2327 0 : _ml_error_report
2328 : ("There is an element named [%s] in the pipeline, but its type is unknown. It is possible that the app thread has touched ML-API's internal data structure.",
2329 : element_name);
2330 0 : ret = ML_ERROR_INVALID_PARAMETER;
2331 0 : goto unlock_return;
2332 : }
2333 :
2334 62 : common_elem = *elem_h = g_new0 (ml_pipeline_common_elem, 1);
2335 62 : if (common_elem == NULL) {
2336 0 : _ml_error_report
2337 : ("Failed to allocate the internal handler for %s. Out of memory?",
2338 : element_name);
2339 0 : ret = ML_ERROR_OUT_OF_MEMORY;
2340 0 : goto unlock_return;
2341 : }
2342 :
2343 62 : common_elem->pipe = p;
2344 62 : common_elem->element = elem;
2345 :
2346 62 : g_mutex_lock (&elem->lock);
2347 62 : elem->maxid++;
2348 62 : common_elem->id = elem->maxid;
2349 62 : elem->handles = g_list_append (elem->handles, common_elem);
2350 62 : g_mutex_unlock (&elem->lock);
2351 :
2352 63 : unlock_return:
2353 63 : g_mutex_unlock (&p->lock);
2354 63 : return ret;
2355 : }
2356 :
2357 : /**
2358 : * @brief Releases the given element handle.
2359 : */
2360 : int
2361 62 : ml_pipeline_element_release_handle (ml_pipeline_element_h elem_h)
2362 : {
2363 62 : handle_init (common_elem, elem_h);
2364 :
2365 61 : elem->handles = g_list_remove (elem->handles, common_elem);
2366 61 : free_element_handle (common_elem);
2367 :
2368 61 : handle_exit (elem_h);
2369 : }
2370 :
2371 : /**
2372 : * @brief Check property existence and its type.
2373 : */
2374 : static bool
2375 109 : ml_pipeline_element_check_property (GObjectClass * klass,
2376 : const char *property_name, const GType type)
2377 : {
2378 109 : GParamSpec *pspec = NULL;
2379 :
2380 : /* Check property existence */
2381 109 : pspec = g_object_class_find_property (klass, property_name);
2382 109 : if (pspec == NULL) {
2383 16 : _ml_logw ("The property name [%s] does not exist.", property_name);
2384 16 : return FALSE;
2385 : }
2386 :
2387 : /* Compare property's type with given type */
2388 109 : if (!((pspec->value_type == type) ||
2389 33 : (type == G_TYPE_ENUM && G_TYPE_IS_ENUM (pspec->value_type)) ||
2390 22 : (type == G_TYPE_INT64 && pspec->value_type == G_TYPE_LONG) ||
2391 22 : (type == G_TYPE_UINT64 && pspec->value_type == G_TYPE_ULONG) ||
2392 22 : (type == G_TYPE_INT && G_TYPE_IS_ENUM (pspec->value_type)) ||
2393 5 : (type == G_TYPE_UINT && G_TYPE_IS_ENUM (pspec->value_type)) ||
2394 2 : (type == G_TYPE_DOUBLE && pspec->value_type == G_TYPE_FLOAT))) {
2395 16 : _ml_logw ("The type of property name [%s] is '%s'", property_name,
2396 : g_type_name (pspec->value_type));
2397 16 : return FALSE;
2398 : }
2399 77 : return TRUE;
2400 : }
2401 :
2402 : /**
2403 : * @brief Sets the value of given element's property in NNStreamer pipelines.
2404 : */
2405 : static int
2406 83 : ml_pipeline_element_set_property (ml_pipeline_element_h elem_h,
2407 : const char *property_name, gpointer value, GType type)
2408 : {
2409 83 : handle_init (common_elem, elem_h);
2410 :
2411 : /* Check the input parameter */
2412 75 : if (property_name == NULL) {
2413 0 : _ml_error_report
2414 : ("The parameter, property_name (const char *), is NULL. It should be a valid string of property name.");
2415 0 : ret = ML_ERROR_INVALID_PARAMETER;
2416 0 : goto unlock_return;
2417 : }
2418 :
2419 : /* Check property existence & its type */
2420 75 : if (!ml_pipeline_element_check_property (G_OBJECT_GET_CLASS (elem->element),
2421 : property_name, type)) {
2422 16 : _ml_error_report
2423 : ("The property ('%s') of the element, '%s', cannot be checked. It looks like this property does not exist in this element.",
2424 : property_name, elem->name);
2425 16 : ret = ML_ERROR_INVALID_PARAMETER;
2426 16 : goto unlock_return;
2427 : }
2428 :
2429 : /* Set property */
2430 59 : if (type == G_TYPE_DOUBLE || type == G_TYPE_FLOAT) {
2431 7 : g_object_set (G_OBJECT (elem->element), property_name,
2432 : *(double *) value, NULL);
2433 52 : } else if (type == G_TYPE_INT64) {
2434 7 : g_object_set (G_OBJECT (elem->element), property_name,
2435 : *(int64_t *) value, NULL);
2436 45 : } else if (type == G_TYPE_UINT64) {
2437 7 : g_object_set (G_OBJECT (elem->element), property_name,
2438 : *(uint64_t *) value, NULL);
2439 : } else {
2440 38 : g_object_set (G_OBJECT (elem->element), property_name, value, NULL);
2441 : }
2442 :
2443 75 : handle_exit (elem_h);
2444 : }
2445 :
2446 : /**
2447 : * @brief Gets the value of given element's property in NNStreamer pipelines.
2448 : */
2449 : static int
2450 50 : ml_pipeline_element_get_property (ml_pipeline_element_h elem_h,
2451 : const char *property_name, GType type, gpointer pvalue)
2452 : {
2453 50 : handle_init (common_elem, elem_h);
2454 :
2455 : /* Check the input parameter */
2456 42 : if (property_name == NULL) {
2457 0 : _ml_error_report
2458 : ("The parameter, property_name (const char *), is NULL. It should be a valid string of the property name of an element.");
2459 0 : ret = ML_ERROR_INVALID_PARAMETER;
2460 0 : goto unlock_return;
2461 : }
2462 :
2463 42 : if (pvalue == NULL) {
2464 8 : _ml_error_report
2465 : ("The parameter, pvalue (gpointer / a pointer of a value), is NULL. It should be a valid gpointer (a pointer of a value). E.g., char *str; ... ml_pipeline_get_property_string (... &str); ... int32_t val; ... ml_pipeline_get_property_int32 (..., &val);");
2466 8 : ret = ML_ERROR_INVALID_PARAMETER;
2467 8 : goto unlock_return;
2468 : }
2469 :
2470 : /* Check property existence & its type */
2471 34 : if (!ml_pipeline_element_check_property (G_OBJECT_GET_CLASS (elem->element),
2472 : property_name, type)) {
2473 16 : _ml_error_report
2474 : ("Cannot check the property ('%s') or the element ('%s'). Please check if you have the corresponding element in the pipeline.",
2475 : property_name, elem->name);
2476 16 : ret = ML_ERROR_INVALID_PARAMETER;
2477 16 : goto unlock_return;
2478 : }
2479 :
2480 : /* Get property */
2481 18 : g_object_get (G_OBJECT (elem->element), property_name, pvalue, NULL);
2482 :
2483 42 : handle_exit (elem_h);
2484 : }
2485 :
2486 : /**
2487 : * @brief Sets the boolean value of element's property in NNStreamer pipelines.
2488 : */
2489 : int
2490 11 : ml_pipeline_element_set_property_bool (ml_pipeline_element_h elem_h,
2491 : const char *property_name, const int32_t value)
2492 : {
2493 11 : return ml_pipeline_element_set_property (elem_h, property_name,
2494 11 : GINT_TO_POINTER (value), G_TYPE_BOOLEAN);
2495 : }
2496 :
2497 : /**
2498 : * @brief Sets the string value of element's property in NNStreamer pipelines.
2499 : */
2500 : int
2501 6 : ml_pipeline_element_set_property_string (ml_pipeline_element_h elem_h,
2502 : const char *property_name, const char *value)
2503 : {
2504 6 : return ml_pipeline_element_set_property (elem_h, property_name,
2505 : (gpointer) value, G_TYPE_STRING);
2506 : }
2507 :
2508 : /**
2509 : * @brief Sets the integer value of element's property in NNStreamer pipelines.
2510 : */
2511 : int
2512 12 : ml_pipeline_element_set_property_int32 (ml_pipeline_element_h elem_h,
2513 : const char *property_name, const int32_t value)
2514 : {
2515 12 : return ml_pipeline_element_set_property (elem_h, property_name,
2516 12 : GINT_TO_POINTER (value), G_TYPE_INT);
2517 : }
2518 :
2519 : /**
2520 : * @brief Sets the integer 64bit value of element's property in NNStreamer pipelines.
2521 : */
2522 : int
2523 10 : ml_pipeline_element_set_property_int64 (ml_pipeline_element_h elem_h,
2524 : const char *property_name, const int64_t value)
2525 : {
2526 10 : return ml_pipeline_element_set_property (elem_h, property_name,
2527 : (gpointer) (&value), G_TYPE_INT64);
2528 : }
2529 :
2530 : /**
2531 : * @brief Sets the unsigned integer value of element's property in NNStreamer pipelines.
2532 : */
2533 : int
2534 12 : ml_pipeline_element_set_property_uint32 (ml_pipeline_element_h elem_h,
2535 : const char *property_name, const uint32_t value)
2536 : {
2537 12 : return ml_pipeline_element_set_property (elem_h, property_name,
2538 12 : GUINT_TO_POINTER (value), G_TYPE_UINT);
2539 : }
2540 :
2541 : /**
2542 : * @brief Sets the unsigned integer 64bit value of element's property in NNStreamer pipelines.
2543 : */
2544 : int
2545 10 : ml_pipeline_element_set_property_uint64 (ml_pipeline_element_h elem_h,
2546 : const char *property_name, const uint64_t value)
2547 : {
2548 10 : return ml_pipeline_element_set_property (elem_h, property_name,
2549 : (gpointer) (&value), G_TYPE_UINT64);
2550 : }
2551 :
2552 : /**
2553 : * @brief Sets the floating point value of element's property in NNStreamer pipelines.
2554 : */
2555 : int
2556 10 : ml_pipeline_element_set_property_double (ml_pipeline_element_h elem_h,
2557 : const char *property_name, const double value)
2558 : {
2559 10 : return ml_pipeline_element_set_property (elem_h, property_name,
2560 : (gpointer) (&value), G_TYPE_DOUBLE);
2561 : }
2562 :
2563 : /**
2564 : * @brief Sets the enumeration value of element's property in NNStreamer pipelines.
2565 : */
2566 : int
2567 12 : ml_pipeline_element_set_property_enum (ml_pipeline_element_h elem_h,
2568 : const char *property_name, const uint32_t value)
2569 : {
2570 12 : return ml_pipeline_element_set_property (elem_h, property_name,
2571 12 : GUINT_TO_POINTER (value), G_TYPE_ENUM);
2572 : }
2573 :
2574 : /**
2575 : * @brief Gets the boolean value of element's property in NNStreamer pipelines.
2576 : */
2577 : int
2578 6 : ml_pipeline_element_get_property_bool (ml_pipeline_element_h elem_h,
2579 : const char *property_name, int32_t * value)
2580 : {
2581 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2582 : G_TYPE_BOOLEAN, (gpointer) value);
2583 : }
2584 :
2585 : /**
2586 : * @brief Gets the string value of element's property in NNStreamer pipelines.
2587 : */
2588 : int
2589 6 : ml_pipeline_element_get_property_string (ml_pipeline_element_h elem_h,
2590 : const char *property_name, char **value)
2591 : {
2592 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2593 : G_TYPE_STRING, (gpointer) value);
2594 : }
2595 :
2596 : /**
2597 : * @brief Gets the integer value of element's property in NNStreamer pipelines.
2598 : */
2599 : int
2600 7 : ml_pipeline_element_get_property_int32 (ml_pipeline_element_h elem_h,
2601 : const char *property_name, int32_t * value)
2602 : {
2603 7 : return ml_pipeline_element_get_property (elem_h, property_name,
2604 : G_TYPE_INT, (gpointer) value);
2605 : }
2606 :
2607 : /**
2608 : * @brief Gets the integer 64bit value of element's property in NNStreamer pipelines.
2609 : */
2610 : int
2611 6 : ml_pipeline_element_get_property_int64 (ml_pipeline_element_h elem_h,
2612 : const char *property_name, int64_t * value)
2613 : {
2614 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2615 : G_TYPE_INT64, (gpointer) value);
2616 : }
2617 :
2618 : /**
2619 : * @brief Gets the unsigned integer value of element's property in NNStreamer pipelines.
2620 : */
2621 : int
2622 7 : ml_pipeline_element_get_property_uint32 (ml_pipeline_element_h elem_h,
2623 : const char *property_name, uint32_t * value)
2624 : {
2625 7 : return ml_pipeline_element_get_property (elem_h, property_name,
2626 : G_TYPE_UINT, (gpointer) value);
2627 : }
2628 :
2629 : /**
2630 : * @brief Gets the unsigned integer 64bit value of element's property in NNStreamer pipelines.
2631 : */
2632 : int
2633 6 : ml_pipeline_element_get_property_uint64 (ml_pipeline_element_h elem_h,
2634 : const char *property_name, uint64_t * value)
2635 : {
2636 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2637 : G_TYPE_UINT64, (gpointer) value);
2638 : }
2639 :
2640 : /**
2641 : * @brief Gets the floating point value of element's property in NNStreamer pipelines.
2642 : */
2643 : int
2644 6 : ml_pipeline_element_get_property_double (ml_pipeline_element_h elem_h,
2645 : const char *property_name, double *value)
2646 : {
2647 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2648 : G_TYPE_DOUBLE, (gpointer) value);
2649 : }
2650 :
2651 : /**
2652 : * @brief Gets the enumeration value of element's property in NNStreamer pipelines.
2653 : */
2654 : int
2655 6 : ml_pipeline_element_get_property_enum (ml_pipeline_element_h elem_h,
2656 : const char *property_name, uint32_t * value)
2657 : {
2658 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2659 : G_TYPE_ENUM, (gpointer) value);
2660 : }
2661 :
2662 : /**
2663 : * @brief Gets the element of pipeline itself (GstElement).
2664 : */
2665 : GstElement *
2666 0 : _ml_pipeline_get_gst_pipeline (ml_pipeline_h pipe)
2667 : {
2668 0 : ml_pipeline *p = (ml_pipeline *) pipe;
2669 0 : GstElement *element = NULL;
2670 :
2671 0 : if (p) {
2672 0 : g_mutex_lock (&p->lock);
2673 :
2674 0 : element = p->element;
2675 0 : if (element)
2676 0 : gst_object_ref (element);
2677 :
2678 0 : g_mutex_unlock (&p->lock);
2679 : }
2680 :
2681 0 : return element;
2682 : }
2683 :
2684 : /**
2685 : * @brief Gets the element in pipeline (GstElement).
2686 : */
2687 : GstElement *
2688 0 : _ml_pipeline_get_gst_element (ml_pipeline_element_h handle)
2689 : {
2690 0 : ml_pipeline_common_elem *e = (ml_pipeline_common_elem *) handle;
2691 0 : GstElement *element = NULL;
2692 :
2693 0 : if (e && e->element) {
2694 0 : ml_pipeline_element *elem = e->element;
2695 :
2696 0 : g_mutex_lock (&elem->lock);
2697 :
2698 0 : element = elem->element;
2699 0 : if (element)
2700 0 : gst_object_ref (element);
2701 :
2702 0 : g_mutex_unlock (&elem->lock);
2703 : }
2704 :
2705 0 : return element;
2706 : }
2707 :
2708 : /**
2709 : * @brief Increases ref count of custom-easy filter.
2710 : */
2711 : static void
2712 3 : ml_pipeline_custom_filter_ref (ml_custom_easy_filter_h custom)
2713 : {
2714 3 : ml_custom_filter_s *c = (ml_custom_filter_s *) custom;
2715 :
2716 3 : if (c) {
2717 3 : g_mutex_lock (&c->lock);
2718 3 : c->ref_count++;
2719 3 : g_mutex_unlock (&c->lock);
2720 : }
2721 3 : }
2722 :
2723 : /**
2724 : * @brief Decreases ref count of custom-easy filter.
2725 : */
2726 : static void
2727 3 : ml_pipeline_custom_filter_unref (ml_custom_easy_filter_h custom)
2728 : {
2729 3 : ml_custom_filter_s *c = (ml_custom_filter_s *) custom;
2730 :
2731 3 : if (!c)
2732 0 : return;
2733 :
2734 3 : g_mutex_lock (&c->lock);
2735 3 : if (c->ref_count > 0)
2736 3 : c->ref_count--;
2737 3 : g_mutex_unlock (&c->lock);
2738 : }
2739 :
2740 : /**
2741 : * @brief Releases custom filter handle.
2742 : */
2743 : static void
2744 4 : ml_pipeline_custom_free_handle (ml_custom_filter_s * custom)
2745 : {
2746 4 : if (custom) {
2747 4 : g_mutex_lock (&custom->lock);
2748 :
2749 4 : g_free (custom->name);
2750 4 : ml_tensors_info_destroy (custom->in_info);
2751 4 : ml_tensors_info_destroy (custom->out_info);
2752 :
2753 4 : g_mutex_unlock (&custom->lock);
2754 4 : g_mutex_clear (&custom->lock);
2755 :
2756 4 : g_free (custom);
2757 : }
2758 4 : }
2759 :
2760 : /**
2761 : * @brief Invoke callback for custom-easy filter.
2762 : */
2763 : static int
2764 5 : ml_pipeline_custom_invoke (void *data, const GstTensorFilterProperties * prop,
2765 : const GstTensorMemory * in, GstTensorMemory * out)
2766 : {
2767 : int status;
2768 : ml_custom_filter_s *c;
2769 : ml_tensors_data_h in_data, out_data;
2770 : ml_tensors_data_s *_data;
2771 : guint i;
2772 :
2773 5 : in_data = out_data = NULL;
2774 5 : c = (ml_custom_filter_s *) data;
2775 :
2776 : /* internal error? */
2777 5 : if (!c || !c->cb)
2778 5 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2779 : "Internal error of callback function, ml_pipeline_custom_invoke. Its internal data structure is broken.");
2780 :
2781 5 : g_mutex_lock (&c->lock);
2782 :
2783 : /* prepare invoke */
2784 5 : status = _ml_tensors_data_create_no_alloc (c->in_info, &in_data);
2785 5 : if (status != ML_ERROR_NONE) {
2786 0 : _ml_error_report_continue ("_ml_tensors_data_create_no_alloc has failed.");
2787 0 : goto done;
2788 : }
2789 :
2790 5 : _data = (ml_tensors_data_s *) in_data;
2791 10 : for (i = 0; i < _data->num_tensors; i++)
2792 5 : _data->tensors[i].data = in[i].data;
2793 :
2794 5 : status = _ml_tensors_data_create_no_alloc (c->out_info, &out_data);
2795 5 : if (status != ML_ERROR_NONE) {
2796 0 : _ml_error_report_continue ("_ml_tensors_data_create_no_alloc has failed.");
2797 0 : goto done;
2798 : }
2799 :
2800 5 : _data = (ml_tensors_data_s *) out_data;
2801 10 : for (i = 0; i < _data->num_tensors; i++)
2802 5 : _data->tensors[i].data = out[i].data;
2803 :
2804 : /* call invoke callback */
2805 5 : status = c->cb (in_data, out_data, c->pdata);
2806 :
2807 5 : done:
2808 5 : g_mutex_unlock (&c->lock);
2809 : /* NOTE: DO NOT free tensor data */
2810 5 : _ml_tensors_data_destroy_internal (in_data, FALSE);
2811 5 : _ml_tensors_data_destroy_internal (out_data, FALSE);
2812 :
2813 5 : return status;
2814 : }
2815 :
2816 : /**
2817 : * @brief Registers a custom filter.
2818 : */
2819 : int
2820 11 : ml_pipeline_custom_easy_filter_register (const char *name,
2821 : const ml_tensors_info_h in, const ml_tensors_info_h out,
2822 : ml_custom_easy_invoke_cb cb, void *user_data,
2823 : ml_custom_easy_filter_h * custom)
2824 : {
2825 11 : int status = ML_ERROR_NONE;
2826 : ml_custom_filter_s *c;
2827 : GstTensorsInfo in_info, out_info;
2828 :
2829 22 : check_feature_state (ML_FEATURE_INFERENCE);
2830 :
2831 11 : if (!name)
2832 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2833 : "The parameter, name (const char *), is NULL. It should be a valid string of the filter name.");
2834 10 : if (!cb)
2835 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2836 : "The parameter, cb (ml_custom_easy_invoke_cb), is NULL. It should be a valid call-back struct containing function pointer and its related data.");
2837 9 : if (!custom)
2838 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2839 : "The parameter, custom (ml_custom_easy_filter_h *), is NULL. It should be a valid pointer of ml_custom_easy_filter. E.g., ml_custom_easy_filter_h custom; ml_pipeline_custom_easy_filter_register (..., &custom);");
2840 :
2841 : /* init null */
2842 8 : *custom = NULL;
2843 :
2844 8 : if (!ml_tensors_info_is_valid (in))
2845 2 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2846 : "The parameter, in (const ml_tensors_info_h), is not valid. ml_tensors_info_is_valid(in) has returned FALSE. Please check if its cloned/fetched from a valid object or if you have configured it properly.");
2847 6 : if (!ml_tensors_info_is_valid (out))
2848 2 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2849 : "The parameter, out (const ml_tensors_info_h), is not valid. ml_tensors_info_is_valid(in) has returned FALSE. Please check if its cloned/fetched from a valid object or if you have configured it properly.");
2850 :
2851 : /* create and init custom handle */
2852 4 : if ((c = g_new0 (ml_custom_filter_s, 1)) == NULL)
2853 0 : _ml_error_report_return (ML_ERROR_OUT_OF_MEMORY,
2854 : "Cannot allocate memory. Out of memory?");
2855 :
2856 4 : g_mutex_init (&c->lock);
2857 :
2858 : /** no need to acquire c->lock as its created locally */
2859 4 : c->name = g_strdup (name);
2860 4 : c->ref_count = 0;
2861 4 : c->cb = cb;
2862 4 : c->pdata = user_data;
2863 4 : ml_tensors_info_create_extended (&c->in_info);
2864 4 : ml_tensors_info_create_extended (&c->out_info);
2865 :
2866 4 : status = ml_tensors_info_clone (c->in_info, in);
2867 4 : if (status != ML_ERROR_NONE) {
2868 0 : _ml_error_report_continue
2869 : ("ml_tensors_info_clone has failed with %d. Cannot fetch input tensor-info (metadata).",
2870 : status);
2871 0 : goto exit;
2872 : }
2873 :
2874 4 : status = ml_tensors_info_clone (c->out_info, out);
2875 4 : if (status != ML_ERROR_NONE) {
2876 0 : _ml_error_report_continue
2877 : ("ml_tensors_info_clone has filed with %d. Cannot fetch output tensor-info (metadata).",
2878 : status);
2879 0 : goto exit;
2880 : }
2881 :
2882 : /* register custom filter */
2883 4 : _ml_tensors_info_copy_from_ml (&in_info, c->in_info);
2884 4 : _ml_tensors_info_copy_from_ml (&out_info, c->out_info);
2885 :
2886 4 : status = NNS_custom_easy_register (name, ml_pipeline_custom_invoke, c,
2887 : &in_info, &out_info);
2888 4 : if (status != 0) {
2889 1 : char buf[255] = { 0 };
2890 1 : if (status == -EINVAL) {
2891 1 : status = ML_ERROR_INVALID_PARAMETER;
2892 1 : strncpy (buf, "invalid parameters are given.", 254);
2893 0 : } else if (status == -ENOMEM) {
2894 0 : status = ML_ERROR_OUT_OF_MEMORY;
2895 0 : strncpy (buf, "out of memory. cannot allocate.", 254);
2896 : } else {
2897 0 : status = ML_ERROR_UNKNOWN;
2898 0 : strncpy (buf, "unknown error.", 254);
2899 : }
2900 1 : _ml_error_report
2901 : ("Failed to register custom filter %s with NNStreamer API, NNS_custom_easy_register(). It has returned %d, which means '%s'.",
2902 : name, status, buf);
2903 : }
2904 :
2905 3 : exit:
2906 4 : if (status == ML_ERROR_NONE) {
2907 3 : pipe_custom_add_data (PIPE_CUSTOM_TYPE_FILTER, name, c);
2908 3 : *custom = c;
2909 : } else {
2910 1 : ml_pipeline_custom_free_handle (c);
2911 : }
2912 :
2913 4 : return status;
2914 : }
2915 :
2916 : /**
2917 : * @brief Unregisters the custom filter.
2918 : */
2919 : int
2920 6 : ml_pipeline_custom_easy_filter_unregister (ml_custom_easy_filter_h custom)
2921 : {
2922 : ml_custom_filter_s *c;
2923 6 : int status = ML_ERROR_NONE;
2924 :
2925 6 : check_feature_state (ML_FEATURE_INFERENCE);
2926 :
2927 6 : if (!custom)
2928 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2929 : "The parameter, custom (ml_custom_easy_filter_h), is NULL. It should be a valid ml_custom_easy_filter_h instance, usually created by ml_pipeline_custom_easy_filter_register().");
2930 :
2931 5 : c = (ml_custom_filter_s *) custom;
2932 5 : g_mutex_lock (&c->lock);
2933 :
2934 5 : if (c->ref_count > 0) {
2935 2 : _ml_error_report
2936 : ("Failed to unregister custom filter %s, it is used in the pipeline. Its reference counter value is %u.",
2937 : c->name, c->ref_count);
2938 2 : status = ML_ERROR_INVALID_PARAMETER;
2939 2 : goto done;
2940 : }
2941 :
2942 3 : status = NNS_custom_easy_unregister (c->name);
2943 3 : if (status != 0) {
2944 0 : _ml_error_report
2945 : ("Failed to unregister custom filter %s. It is possible that this is already unregistered or not registered.",
2946 : c->name);
2947 0 : status = ML_ERROR_INVALID_PARAMETER;
2948 0 : goto done;
2949 : }
2950 :
2951 3 : done:
2952 5 : g_mutex_unlock (&c->lock);
2953 :
2954 5 : if (status == ML_ERROR_NONE) {
2955 3 : pipe_custom_remove_data (PIPE_CUSTOM_TYPE_FILTER, c->name);
2956 3 : ml_pipeline_custom_free_handle (c);
2957 : }
2958 :
2959 5 : return status;
2960 : }
2961 :
2962 : /**
2963 : * @brief Increases ref count of tensor_if custom condition.
2964 : */
2965 : static void
2966 3 : ml_pipeline_if_custom_ref (ml_pipeline_if_h custom)
2967 : {
2968 3 : ml_if_custom_s *c = (ml_if_custom_s *) custom;
2969 :
2970 3 : if (c) {
2971 3 : g_mutex_lock (&c->lock);
2972 3 : c->ref_count++;
2973 3 : g_mutex_unlock (&c->lock);
2974 : }
2975 3 : }
2976 :
2977 : /**
2978 : * @brief Decreases ref count of tensor_if custom condition.
2979 : */
2980 : static void
2981 3 : ml_pipeline_if_custom_unref (ml_pipeline_if_h custom)
2982 : {
2983 3 : ml_if_custom_s *c = (ml_if_custom_s *) custom;
2984 :
2985 3 : if (c) {
2986 3 : g_mutex_lock (&c->lock);
2987 3 : if (c->ref_count > 0)
2988 3 : c->ref_count--;
2989 3 : g_mutex_unlock (&c->lock);
2990 : }
2991 3 : }
2992 :
2993 : /**
2994 : * @brief Callback for tensor_if custom condition.
2995 : */
2996 : static gboolean
2997 10 : ml_pipeline_if_custom (const GstTensorsInfo * info,
2998 : const GstTensorMemory * input, void *data, gboolean * result)
2999 : {
3000 10 : int status = 0;
3001 : guint i;
3002 : ml_if_custom_s *c;
3003 10 : ml_tensors_data_h in_data = NULL;
3004 : ml_tensors_data_s *_data;
3005 10 : ml_tensors_info_h ml_info = NULL;
3006 10 : gboolean ret = FALSE;
3007 :
3008 10 : c = (ml_if_custom_s *) data;
3009 :
3010 : /* internal error? */
3011 10 : if (!c || !c->cb)
3012 10 : _ml_error_report_return (FALSE,
3013 : "Internal error: the parameter, data, is not valid. App thread might have touched internal data structure.");
3014 :
3015 10 : status = _ml_tensors_info_create_from_gst (&ml_info, info);
3016 10 : if (status != ML_ERROR_NONE)
3017 0 : _ml_error_report_return_continue (FALSE,
3018 : "Cannot create tensors-info from the parameter, info (const GstTensorsInfo). _ml_tensors_info_create_from_gst has returned %d.",
3019 : status);
3020 10 : status = _ml_tensors_data_create_no_alloc (ml_info, &in_data);
3021 10 : if (status != ML_ERROR_NONE) {
3022 0 : _ml_error_report_continue
3023 : ("Cannot create data entry from the given metadata, info (const GstTensorMemory, although we could create tensor-info from info. _ml_tensors_data_create_no_alloc() has returned %d.",
3024 : status);
3025 0 : goto done;
3026 : }
3027 :
3028 10 : _data = (ml_tensors_data_s *) in_data;
3029 20 : for (i = 0; i < _data->num_tensors; i++)
3030 10 : _data->tensors[i].data = input[i].data;
3031 :
3032 : /* call invoke callback */
3033 10 : g_mutex_lock (&c->lock);
3034 10 : status = c->cb (in_data, ml_info, result, c->pdata);
3035 10 : g_mutex_unlock (&c->lock);
3036 :
3037 10 : ret = (status == ML_ERROR_NONE);
3038 10 : if (!ret)
3039 0 : _ml_error_report
3040 : ("The callback function of if-statement has returned error: %d.",
3041 : status);
3042 :
3043 10 : done:
3044 10 : ml_tensors_info_destroy (ml_info);
3045 10 : _ml_tensors_data_destroy_internal (in_data, FALSE);
3046 :
3047 10 : return ret;
3048 : }
3049 :
3050 : /**
3051 : * @brief Releases tensor_if custom condition.
3052 : */
3053 : static void
3054 4 : ml_pipeline_if_custom_free (ml_if_custom_s * custom)
3055 : {
3056 4 : if (custom) {
3057 4 : g_mutex_lock (&custom->lock);
3058 :
3059 4 : g_free (custom->name);
3060 :
3061 4 : g_mutex_unlock (&custom->lock);
3062 4 : g_mutex_clear (&custom->lock);
3063 :
3064 4 : g_free (custom);
3065 : }
3066 4 : }
3067 :
3068 : /**
3069 : * @brief Registers the tensor_if custom callback.
3070 : */
3071 : int
3072 7 : ml_pipeline_tensor_if_custom_register (const char *name,
3073 : ml_pipeline_if_custom_cb cb, void *user_data, ml_pipeline_if_h * if_custom)
3074 : {
3075 7 : int status = ML_ERROR_NONE;
3076 : ml_if_custom_s *c;
3077 :
3078 7 : check_feature_state (ML_FEATURE_INFERENCE);
3079 :
3080 7 : if (!name)
3081 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
3082 : "The parameter, name (const char *), is NULL. It should be a valid string of the tensor_if element in your pipeline.");
3083 6 : if (!cb)
3084 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
3085 : "The parameter, cb (ml_pipeline_if_custom_cb callback function pointer), is NULL. It should be a valid function pointer that determines if the 'if' statement is TRUE or FALSE from the given tensor data frame.");
3086 5 : if (!if_custom)
3087 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
3088 : "The parameter, if_custom (ml_pipeline_if_h *), is NULL. It should be a valid pointer to the pipeline-if handle instance. E.g., ml_pipeline_if_h h; ml_pipeline_tensor_if_custom_register (..., &h);");
3089 :
3090 : /* init null */
3091 4 : *if_custom = NULL;
3092 :
3093 : /* create and init custom handle */
3094 4 : if ((c = g_try_new0 (ml_if_custom_s, 1)) == NULL)
3095 0 : _ml_error_report_return (ML_ERROR_OUT_OF_MEMORY,
3096 : "Cannot allocate memory. Out of memory?");
3097 :
3098 4 : g_mutex_init (&c->lock);
3099 :
3100 4 : g_mutex_lock (&c->lock);
3101 4 : c->name = g_strdup (name);
3102 4 : c->ref_count = 0;
3103 4 : c->cb = cb;
3104 4 : c->pdata = user_data;
3105 :
3106 4 : status = nnstreamer_if_custom_register (name, ml_pipeline_if_custom, c);
3107 4 : if (status != 0) {
3108 1 : if (status == -ENOMEM) {
3109 0 : _ml_error_report
3110 : ("Failed to register tensor_if custom condition %s because nnstreamer_if_custom_register has failed to allocate memory. Out of memory?",
3111 : name);
3112 0 : status = ML_ERROR_OUT_OF_MEMORY;
3113 1 : } else if (status == -EINVAL) {
3114 1 : _ml_error_report
3115 : ("Failed to register tensor_if custom condition %s because nnstreamer_if_custom_register has reported that an invalid parameter is given to the API call. Please check if the given name is 0-length or duplicated (already registered), memory is full, or the name is not allowed ('any', 'auto' are not allowed).",
3116 : name);
3117 1 : status = ML_ERROR_INVALID_PARAMETER;
3118 : } else {
3119 0 : _ml_error_report
3120 : ("Failed to register tensor_if custom condition %s because nnstreamer_if_custom_register has returned unknown error.",
3121 : name);
3122 0 : status = ML_ERROR_UNKNOWN;
3123 : }
3124 : }
3125 4 : g_mutex_unlock (&c->lock);
3126 :
3127 4 : if (status == ML_ERROR_NONE) {
3128 3 : pipe_custom_add_data (PIPE_CUSTOM_TYPE_IF, name, c);
3129 3 : *if_custom = c;
3130 : } else {
3131 1 : ml_pipeline_if_custom_free (c);
3132 : }
3133 :
3134 4 : return status;
3135 : }
3136 :
3137 : /**
3138 : * @brief Unregisters the tensor_if custom callback.
3139 : */
3140 : int
3141 6 : ml_pipeline_tensor_if_custom_unregister (ml_pipeline_if_h if_custom)
3142 : {
3143 : ml_if_custom_s *c;
3144 6 : int status = ML_ERROR_NONE;
3145 :
3146 6 : check_feature_state (ML_FEATURE_INFERENCE);
3147 :
3148 6 : if (!if_custom)
3149 1 : return ML_ERROR_INVALID_PARAMETER;
3150 :
3151 5 : c = (ml_if_custom_s *) if_custom;
3152 5 : g_mutex_lock (&c->lock);
3153 :
3154 5 : if (c->ref_count > 0) {
3155 2 : _ml_error_report
3156 : ("Failed to unregister custom condition %s, it is used in the pipeline.",
3157 : c->name);
3158 2 : status = ML_ERROR_INVALID_PARAMETER;
3159 2 : goto done;
3160 : }
3161 :
3162 3 : status = nnstreamer_if_custom_unregister (c->name);
3163 3 : if (status != 0) {
3164 0 : if (status == -EINVAL)
3165 0 : _ml_error_report
3166 : ("Failed to unregister tensor_if custom condition %s. It appears that it is already unregistered or not yet registered.",
3167 : c->name);
3168 : else
3169 0 : _ml_error_report
3170 : ("Failed to unregister tensor_if custom condition %s with unknown reason. Internal error?",
3171 : c->name);
3172 0 : status = ML_ERROR_STREAMS_PIPE;
3173 0 : goto done;
3174 : }
3175 :
3176 3 : done:
3177 5 : g_mutex_unlock (&c->lock);
3178 :
3179 5 : if (status == ML_ERROR_NONE) {
3180 3 : pipe_custom_remove_data (PIPE_CUSTOM_TYPE_IF, c->name);
3181 3 : ml_pipeline_if_custom_free (c);
3182 : }
3183 :
3184 5 : return status;
3185 : }
|
