Line data Source code
1 : /* SPDX-License-Identifier: Apache-2.0 */
2 : /**
3 : * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
4 : *
5 : * @file ml-api-inference-pipeline.c
6 : * @date 11 March 2019
7 : * @brief NNStreamer/Pipeline(main) C-API Wrapper.
8 : * This allows to construct and control NNStreamer pipelines.
9 : * @see https://github.com/nnstreamer/nnstreamer
10 : * @author MyungJoo Ham <myungjoo.ham@samsung.com>
11 : * @bug Thread safety for ml_tensors_data should be addressed.
12 : */
13 :
14 : #include <string.h>
15 : #include <glib.h>
16 : #include <gst/gstbuffer.h>
17 : #include <gst/app/app.h> /* To push data to pipeline */
18 : #include <nnstreamer_plugin_api.h>
19 : #include <tensor_if.h>
20 : #include <tensor_typedef.h>
21 : #include <tensor_filter_custom_easy.h>
22 :
23 : #include <nnstreamer.h>
24 : #include <nnstreamer-tizen-internal.h>
25 :
26 : #include "ml-api-internal.h"
27 : #include "ml-api-inference-internal.h"
28 : #include "ml-api-inference-pipeline-internal.h"
29 :
30 :
31 : #define handle_init(name, h) \
32 : ml_pipeline_common_elem *name= (h); \
33 : ml_pipeline *p; \
34 : ml_pipeline_element *elem; \
35 : int ret = ML_ERROR_NONE; \
36 : check_feature_state (ML_FEATURE_INFERENCE); \
37 : if ((h) == NULL) { \
38 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER, \
39 : "The parameter, %s, (handle) is invalid (NULL). Please provide a valid handle.", \
40 : #h); \
41 : } \
42 : p = name->pipe; \
43 : elem = name->element; \
44 : if (p == NULL) \
45 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER, \
46 : "Internal error. The contents of parameter, %s, (handle), is invalid. The pipeline entry (%s->pipe) is NULL. The handle (%s) is either not properly created or application threads may have touched its contents.", \
47 : #h, #h, #h); \
48 : if (elem == NULL) \
49 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER, \
50 : "Internal error. The contents of parameter, %s, (handle), is invalid. The element entry (%s->element) is NULL. The handle (%s) is either not properly created or application threads may have touched its contents.", \
51 : #h, #h, #h); \
52 : if (elem->pipe == NULL) \
53 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER, \
54 : "Internal error. The contents of parameter, %s, (handle), is invalid. The pipeline entry of the element entry (%s->element->pipe) is NULL. The handle (%s) is either not properly created or application threads may have touched its contents.", \
55 : #h, #h, #h); \
56 : g_mutex_lock (&p->lock); \
57 : g_mutex_lock (&elem->lock); \
58 : if (NULL == g_list_find (elem->handles, name)) { \
59 : _ml_error_report \
60 : ("Internal error. The handle name, %s, does not exists in the list of %s->element->handles.", \
61 : #h, #h); \
62 : ret = ML_ERROR_INVALID_PARAMETER; \
63 : goto unlock_return; \
64 : }
65 :
66 : #define handle_exit(h) \
67 : unlock_return: \
68 : g_mutex_unlock (&elem->lock); \
69 : g_mutex_unlock (&p->lock); \
70 : return ret;
71 :
72 : /**
73 : * @brief The enumeration for custom data type.
74 : */
75 : typedef enum
76 : {
77 : PIPE_CUSTOM_TYPE_NONE,
78 : PIPE_CUSTOM_TYPE_IF,
79 : PIPE_CUSTOM_TYPE_FILTER,
80 :
81 : PIPE_CUSTOM_TYPE_MAX
82 : } pipe_custom_type_e;
83 :
84 : /**
85 : * @brief The struct for custom data.
86 : */
87 : typedef struct
88 : {
89 : pipe_custom_type_e type;
90 : gchar *name;
91 : gpointer handle;
92 : } pipe_custom_data_s;
93 :
94 : static void ml_pipeline_custom_filter_ref (ml_custom_easy_filter_h custom);
95 : static void ml_pipeline_custom_filter_unref (ml_custom_easy_filter_h custom);
96 : static void ml_pipeline_if_custom_ref (ml_pipeline_if_h custom);
97 : static void ml_pipeline_if_custom_unref (ml_pipeline_if_h custom);
98 :
99 : /**
100 : * @brief Global lock for pipeline functions.
101 : */
102 : G_LOCK_DEFINE_STATIC (g_ml_pipe_lock);
103 :
104 : /**
105 : * @brief The list of custom data. This should be managed with lock.
106 : */
107 : static GList *g_ml_custom_data = NULL;
108 :
109 : /**
110 : * @brief Finds a position of custom data in the list.
111 : * @note This function should be called with lock.
112 : */
113 : static GList *
114 12 : pipe_custom_find_link (const pipe_custom_type_e type, const gchar * name)
115 : {
116 : pipe_custom_data_s *data;
117 : GList *link;
118 :
119 12 : g_return_val_if_fail (name != NULL, NULL);
120 :
121 12 : link = g_ml_custom_data;
122 12 : while (link) {
123 12 : data = (pipe_custom_data_s *) link->data;
124 :
125 12 : if (data->type == type && g_str_equal (data->name, name))
126 12 : break;
127 :
128 0 : link = link->next;
129 : }
130 :
131 12 : return link;
132 : }
133 :
134 : /**
135 : * @brief Finds custom data matched with data type and name.
136 : */
137 : static pipe_custom_data_s *
138 6 : pipe_custom_find_data (const pipe_custom_type_e type, const gchar * name)
139 : {
140 : pipe_custom_data_s *data;
141 : GList *link;
142 :
143 6 : G_LOCK (g_ml_pipe_lock);
144 :
145 6 : link = pipe_custom_find_link (type, name);
146 6 : data = (link != NULL) ? (pipe_custom_data_s *) link->data : NULL;
147 :
148 6 : G_UNLOCK (g_ml_pipe_lock);
149 6 : return data;
150 : }
151 :
152 : /**
153 : * @brief Adds new custom data into the list.
154 : */
155 : static void
156 6 : pipe_custom_add_data (const pipe_custom_type_e type, const gchar * name,
157 : gpointer handle)
158 : {
159 : pipe_custom_data_s *data;
160 :
161 6 : data = g_new0 (pipe_custom_data_s, 1);
162 6 : data->type = type;
163 6 : data->name = g_strdup (name);
164 6 : data->handle = handle;
165 :
166 6 : G_LOCK (g_ml_pipe_lock);
167 6 : g_ml_custom_data = g_list_prepend (g_ml_custom_data, data);
168 6 : G_UNLOCK (g_ml_pipe_lock);
169 6 : }
170 :
171 : /**
172 : * @brief Removes custom data from the list.
173 : */
174 : static void
175 6 : pipe_custom_remove_data (const pipe_custom_type_e type, const gchar * name)
176 : {
177 : pipe_custom_data_s *data;
178 : GList *link;
179 :
180 6 : G_LOCK (g_ml_pipe_lock);
181 :
182 6 : link = pipe_custom_find_link (type, name);
183 6 : if (link) {
184 6 : data = (pipe_custom_data_s *) link->data;
185 :
186 6 : g_ml_custom_data = g_list_delete_link (g_ml_custom_data, link);
187 :
188 6 : g_free (data->name);
189 6 : g_free (data);
190 : }
191 :
192 6 : G_UNLOCK (g_ml_pipe_lock);
193 6 : }
194 :
195 : /**
196 : * @brief The callback function called when the element node with custom data is released.
197 : */
198 : static int
199 6 : pipe_custom_destroy_cb (void *handle, void *user_data)
200 : {
201 : pipe_custom_data_s *custom_data;
202 :
203 6 : custom_data = (pipe_custom_data_s *) handle;
204 6 : if (custom_data == NULL)
205 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
206 : "The parameter, handle, is NULL. It should be a valid internal object. This is possibly a bug in ml-api-inference-pipeline.c along with tensor-if or tensor-filter/custom function. Please report to https://github.com/nnstreamer/nnstreamer/issues");
207 :
208 6 : switch (custom_data->type) {
209 3 : case PIPE_CUSTOM_TYPE_IF:
210 3 : ml_pipeline_if_custom_unref (custom_data->handle);
211 3 : break;
212 3 : case PIPE_CUSTOM_TYPE_FILTER:
213 3 : ml_pipeline_custom_filter_unref (custom_data->handle);
214 3 : break;
215 0 : default:
216 0 : break;
217 : }
218 :
219 6 : return ML_ERROR_NONE;
220 : }
221 :
222 : /**
223 : * @brief Internal function to create a referable element in a pipeline
224 : */
225 : static ml_pipeline_element *
226 404 : construct_element (GstElement * e, ml_pipeline * p, const char *name,
227 : ml_pipeline_element_e t)
228 : {
229 404 : ml_pipeline_element *ret = g_new0 (ml_pipeline_element, 1);
230 :
231 404 : if (ret == NULL)
232 0 : _ml_error_report_return (NULL,
233 : "Failed to allocate memory for the pipeline.");
234 :
235 404 : ret->element = e;
236 404 : ret->pipe = p;
237 404 : ret->name = g_strdup (name);
238 404 : ret->type = t;
239 404 : ret->handles = NULL;
240 404 : ret->src = NULL;
241 404 : ret->sink = NULL;
242 404 : ret->maxid = 0;
243 404 : ret->handle_id = 0;
244 404 : ret->is_media_stream = FALSE;
245 404 : ret->is_flexible_tensor = FALSE;
246 404 : g_mutex_init (&ret->lock);
247 404 : gst_tensors_info_init (&ret->tensors_info);
248 :
249 404 : return ret;
250 : }
251 :
252 : /**
253 : * @brief Internal function to get the tensors info from the element caps.
254 : */
255 : static gboolean
256 12466 : get_tensors_info_from_caps (GstCaps * caps, GstTensorsInfo * info,
257 : gboolean * is_flexible)
258 : {
259 : GstStructure *s;
260 : GstTensorsConfig config;
261 : guint i, n_caps;
262 12466 : gboolean found = FALSE;
263 :
264 12466 : n_caps = gst_caps_get_size (caps);
265 :
266 12473 : for (i = 0; i < n_caps; i++) {
267 12467 : s = gst_caps_get_structure (caps, i);
268 12467 : found = gst_tensors_config_from_structure (&config, s);
269 :
270 12467 : if (found) {
271 12460 : gst_tensors_info_free (info);
272 12460 : gst_tensors_info_copy (info, &config.info);
273 12460 : *is_flexible = gst_tensors_config_is_flexible (&config);
274 : }
275 :
276 12467 : gst_tensors_config_free (&config);
277 12467 : if (found)
278 12460 : break;
279 : }
280 :
281 12466 : return found;
282 : }
283 :
284 : /**
285 : * @brief Handle a sink element for registered ml_pipeline_sink_cb
286 : */
287 : static void
288 6265 : cb_sink_event (GstElement * e, GstBuffer * b, gpointer user_data)
289 : {
290 6265 : ml_pipeline_element *elem = user_data;
291 :
292 : /** @todo CRITICAL if the pipeline is being killed, don't proceed! */
293 : GstMemory *mem[ML_TENSOR_SIZE_LIMIT];
294 : GstMapInfo map[ML_TENSOR_SIZE_LIMIT];
295 : guint i, num_tensors;
296 : GList *l;
297 6265 : ml_tensors_data_s *_data = NULL;
298 : GstTensorsInfo gst_info;
299 : int status;
300 :
301 6265 : gst_tensors_info_init (&gst_info);
302 6265 : gst_info.num_tensors = num_tensors = gst_tensor_buffer_get_count (b);
303 :
304 : /* Set tensor data. The handle for tensors-info in data should be added. */
305 : status =
306 6265 : _ml_tensors_data_create_no_alloc (NULL, (ml_tensors_data_h *) & _data);
307 6265 : if (status != ML_ERROR_NONE) {
308 0 : _ml_loge (_ml_detail
309 : ("Failed to allocate memory for tensors data in sink callback, which is registered by ml_pipeline_sink_register ()."));
310 6265 : return;
311 : }
312 :
313 6265 : g_mutex_lock (&elem->lock);
314 :
315 6265 : _data->num_tensors = num_tensors;
316 13007 : for (i = 0; i < num_tensors; i++) {
317 6742 : mem[i] = gst_tensor_buffer_get_nth_memory (b, i);
318 6742 : if (!gst_memory_map (mem[i], &map[i], GST_MAP_READ)) {
319 0 : _ml_loge (_ml_detail
320 : ("Failed to map the output in sink '%s' callback, which is registered by ml_pipeline_sink_register ()",
321 : elem->name));
322 0 : gst_memory_unref (mem[i]);
323 0 : num_tensors = i;
324 0 : goto error;
325 : }
326 :
327 6742 : _data->tensors[i].data = map[i].data;
328 6742 : _data->tensors[i].size = map[i].size;
329 : }
330 :
331 : /** @todo This assumes that padcap is static */
332 6265 : if (elem->sink == NULL) {
333 28 : gboolean found = FALSE;
334 28 : gboolean flexible = FALSE;
335 :
336 : /* Get the sink-pad-cap */
337 28 : elem->sink = gst_element_get_static_pad (elem->element, "sink");
338 :
339 28 : if (elem->sink) {
340 : /* sinkpadcap available (negotiated) */
341 28 : GstCaps *caps = gst_pad_get_current_caps (elem->sink);
342 :
343 28 : if (caps) {
344 28 : found = get_tensors_info_from_caps (caps, &elem->tensors_info,
345 : &flexible);
346 28 : gst_caps_unref (caps);
347 : }
348 : }
349 :
350 28 : if (found) {
351 28 : elem->is_flexible_tensor = flexible;
352 : } else {
353 : /* It is not valid */
354 0 : if (elem->sink) {
355 0 : gst_object_unref (elem->sink);
356 0 : elem->sink = NULL;
357 : }
358 :
359 0 : goto error;
360 : }
361 : }
362 :
363 : /* Prepare output and set data. */
364 6265 : if (elem->is_flexible_tensor) {
365 : GstTensorMetaInfo meta;
366 : gsize hsize;
367 :
368 : /* handle header for flex tensor */
369 12 : for (i = 0; i < num_tensors; i++) {
370 9 : gst_tensor_meta_info_parse_header (&meta, map[i].data);
371 9 : hsize = gst_tensor_meta_info_get_header_size (&meta);
372 :
373 9 : gst_tensor_meta_info_convert (&meta,
374 : gst_tensors_info_get_nth_info (&gst_info, i));
375 :
376 9 : _data->tensors[i].data = map[i].data + hsize;
377 9 : _data->tensors[i].size = map[i].size - hsize;
378 : }
379 : } else {
380 6262 : gst_tensors_info_copy (&gst_info, &elem->tensors_info);
381 :
382 : /* Compare output info and buffer if gst-buffer is not flexible. */
383 6262 : if (gst_info.num_tensors != num_tensors) {
384 0 : _ml_loge (_ml_detail
385 : ("The sink event of [%s] cannot be handled because the number of tensors mismatches.",
386 : elem->name));
387 :
388 0 : gst_object_unref (elem->sink);
389 0 : elem->sink = NULL;
390 0 : goto error;
391 : }
392 :
393 12995 : for (i = 0; i < num_tensors; i++) {
394 6733 : size_t sz = gst_tensors_info_get_size (&gst_info, i);
395 :
396 : /* Not configured, yet. */
397 6733 : if (sz == 0)
398 0 : _ml_loge (_ml_detail
399 : ("The caps for sink(%s) is not configured.", elem->name));
400 :
401 6733 : if (sz != map[i].size) {
402 0 : _ml_loge (_ml_detail
403 : ("The sink event of [%s] cannot be handled because the tensor dimension mismatches.",
404 : elem->name));
405 :
406 0 : gst_object_unref (elem->sink);
407 0 : elem->sink = NULL;
408 0 : goto error;
409 : }
410 : }
411 : }
412 :
413 : /* Create new output info, data handle should be updated here. */
414 6265 : _ml_tensors_info_create_from_gst (&_data->info, &gst_info);
415 :
416 : /* Iterate e->handles, pass the data to them */
417 12536 : for (l = elem->handles; l != NULL; l = l->next) {
418 : ml_pipeline_sink_cb callback;
419 6271 : ml_pipeline_common_elem *sink = l->data;
420 6271 : if (sink->callback_info == NULL)
421 3 : continue;
422 :
423 6268 : callback = sink->callback_info->sink_cb;
424 6268 : if (callback)
425 6268 : callback (_data, _data->info, sink->callback_info->sink_pdata);
426 :
427 : /** @todo Measure time. Warn if it takes long. Kill if it takes too long. */
428 : }
429 :
430 6265 : error:
431 6265 : g_mutex_unlock (&elem->lock);
432 :
433 13007 : for (i = 0; i < num_tensors; i++) {
434 6742 : gst_memory_unmap (mem[i], &map[i]);
435 6742 : gst_memory_unref (mem[i]);
436 : }
437 :
438 6265 : _ml_tensors_data_destroy_internal (_data, FALSE);
439 6265 : _data = NULL;
440 :
441 6265 : gst_tensors_info_free (&gst_info);
442 6265 : return;
443 : }
444 :
445 : /**
446 : * @brief Handle a appsink element for registered ml_pipeline_sink_cb
447 : */
448 : static GstFlowReturn
449 9 : cb_appsink_new_sample (GstElement * e, gpointer user_data)
450 : {
451 : GstSample *sample;
452 : GstBuffer *buffer;
453 :
454 : /* get the sample from appsink */
455 9 : sample = gst_app_sink_pull_sample (GST_APP_SINK (e));
456 9 : buffer = gst_sample_get_buffer (sample);
457 :
458 9 : cb_sink_event (e, buffer, user_data);
459 :
460 9 : gst_sample_unref (sample);
461 9 : return GST_FLOW_OK;
462 : }
463 :
464 : /**
465 : * @brief Callback for bus message.
466 : */
467 : static void
468 5400 : cb_bus_sync_message (GstBus * bus, GstMessage * message, gpointer user_data)
469 : {
470 : ml_pipeline *pipe_h;
471 :
472 5400 : pipe_h = (ml_pipeline *) user_data;
473 :
474 5400 : if (pipe_h == NULL)
475 0 : return;
476 :
477 5400 : switch (GST_MESSAGE_TYPE (message)) {
478 7 : case GST_MESSAGE_EOS:
479 7 : pipe_h->isEOS = TRUE;
480 7 : break;
481 4391 : case GST_MESSAGE_STATE_CHANGED:
482 4391 : if (GST_MESSAGE_SRC (message) == GST_OBJECT_CAST (pipe_h->element)) {
483 : GstState old_state, new_state;
484 :
485 507 : gst_message_parse_state_changed (message, &old_state, &new_state, NULL);
486 507 : pipe_h->pipe_state = (ml_pipeline_state_e) new_state;
487 :
488 507 : _ml_logd (_ml_detail ("The pipeline state changed from %s to %s.",
489 : gst_element_state_get_name (old_state),
490 : gst_element_state_get_name (new_state)));
491 :
492 507 : if (pipe_h->state_cb.cb) {
493 11 : pipe_h->state_cb.cb (pipe_h->pipe_state, pipe_h->state_cb.user_data);
494 : }
495 : }
496 4391 : break;
497 1002 : default:
498 1002 : break;
499 : }
500 : }
501 :
502 : /**
503 : * @brief Clean up each element of the pipeline.
504 : */
505 : static void
506 151 : free_element_handle (gpointer data)
507 : {
508 151 : ml_pipeline_common_elem *item = (ml_pipeline_common_elem *) data;
509 : ml_pipeline_element *elem;
510 :
511 151 : if (!(item && item->callback_info)) {
512 109 : g_free (item);
513 109 : return;
514 : }
515 :
516 : /* clear callbacks */
517 42 : item->callback_info->sink_cb = NULL;
518 42 : elem = item->element;
519 42 : if (elem->type == ML_PIPELINE_ELEMENT_APP_SRC) {
520 2 : GstAppSrcCallbacks appsrc_cb = { 0, };
521 2 : gst_app_src_set_callbacks (GST_APP_SRC (elem->element), &appsrc_cb,
522 : NULL, NULL);
523 : }
524 :
525 42 : g_free (item->callback_info);
526 42 : item->callback_info = NULL;
527 42 : g_free (item);
528 : }
529 :
530 : /**
531 : * @brief Private function for ml_pipeline_destroy, cleaning up nodes in namednodes
532 : */
533 : static void
534 394 : cleanup_node (gpointer data)
535 : {
536 394 : ml_pipeline_element *e = data;
537 :
538 394 : g_mutex_lock (&e->lock);
539 : /** @todo CRITICAL. Stop the handle callbacks if they are running/ready */
540 394 : if (e->handle_id > 0) {
541 27 : g_signal_handler_disconnect (e->element, e->handle_id);
542 27 : e->handle_id = 0;
543 : }
544 :
545 : /* clear all handles first */
546 394 : if (e->handles)
547 63 : g_list_free_full (e->handles, free_element_handle);
548 394 : e->handles = NULL;
549 :
550 394 : if (e->type == ML_PIPELINE_ELEMENT_APP_SRC && !e->pipe->isEOS) {
551 55 : int eos_check_cnt = 0;
552 :
553 : /** to push EOS event, the pipeline should be in PAUSED state */
554 55 : gst_element_set_state (e->pipe->element, GST_STATE_PAUSED);
555 :
556 55 : if (gst_app_src_end_of_stream (GST_APP_SRC (e->element)) != GST_FLOW_OK) {
557 0 : _ml_logw (_ml_detail
558 : ("Cleaning up a pipeline has failed to set End-Of-Stream for the pipeline element of %s",
559 : e->name));
560 : }
561 55 : g_mutex_unlock (&e->lock);
562 5500 : while (!e->pipe->isEOS) {
563 5500 : eos_check_cnt++;
564 : /** check EOS every 1ms */
565 5500 : g_usleep (1000);
566 5500 : if (eos_check_cnt >= EOS_MESSAGE_TIME_LIMIT) {
567 55 : _ml_loge (_ml_detail
568 : ("Cleaning up a pipeline has requested to set End-Of-Stream. However, the pipeline has not become EOS after the timeout. It has failed to become EOS with the element of %s.",
569 : e->name));
570 55 : break;
571 : }
572 : }
573 55 : g_mutex_lock (&e->lock);
574 : }
575 :
576 394 : if (e->custom_destroy) {
577 6 : e->custom_destroy (e->custom_data, e);
578 : }
579 :
580 394 : g_free (e->name);
581 394 : if (e->src)
582 41 : gst_object_unref (e->src);
583 394 : if (e->sink)
584 28 : gst_object_unref (e->sink);
585 :
586 394 : gst_object_unref (e->element);
587 :
588 394 : gst_tensors_info_free (&e->tensors_info);
589 :
590 394 : g_mutex_unlock (&e->lock);
591 394 : g_mutex_clear (&e->lock);
592 :
593 394 : g_free (e);
594 394 : }
595 :
596 : /**
597 : * @brief Private function to release the pipeline resources
598 : */
599 : static void
600 0 : cleanup_resource (gpointer data)
601 : {
602 0 : pipeline_resource_s *res = data;
603 :
604 : /* check resource type and free data */
605 0 : if (g_str_has_prefix (res->type, "tizen")) {
606 0 : release_tizen_resource (res->handle, res->type);
607 : }
608 :
609 0 : g_free (res->type);
610 0 : g_free (res);
611 0 : }
612 :
613 : /**
614 : * @brief Converts predefined element in pipeline description.
615 : */
616 : static int
617 148 : convert_description (ml_pipeline_h pipe, const gchar * description,
618 : gchar ** result, gboolean is_internal)
619 : {
620 : gchar *converted;
621 148 : int status = ML_ERROR_NONE;
622 :
623 296 : g_return_val_if_fail (pipe, ML_ERROR_INVALID_PARAMETER);
624 148 : g_return_val_if_fail (description && result, ML_ERROR_INVALID_PARAMETER);
625 :
626 : /* init null */
627 148 : *result = NULL;
628 :
629 148 : converted = _ml_convert_predefined_entity (description);
630 :
631 : /* convert pre-defined element for Tizen */
632 148 : status = convert_tizen_element (pipe, &converted, is_internal);
633 :
634 148 : if (status == ML_ERROR_NONE) {
635 148 : _ml_logd (_ml_detail
636 : ("Pipeline element converted with aliases for gstreamer (Tizen element aliases): %s",
637 : converted));
638 148 : *result = converted;
639 : } else {
640 0 : g_free (converted);
641 0 : _ml_error_report_continue
642 : ("Failed to convert element: convert_tizen_element() returned %d",
643 : status);
644 : }
645 :
646 148 : return status;
647 : }
648 :
649 : /**
650 : * @brief Handle tensor-filter options.
651 : */
652 : static void
653 39 : process_tensor_filter_option (ml_pipeline_element * e)
654 : {
655 39 : gchar *fw = NULL;
656 39 : gchar *model = NULL;
657 : pipe_custom_data_s *custom_data;
658 :
659 39 : g_object_get (G_OBJECT (e->element), "framework", &fw, "model", &model, NULL);
660 :
661 39 : if (fw && g_ascii_strcasecmp (fw, "custom-easy") == 0) {
662 : /* ref to tensor-filter custom-easy handle. */
663 3 : custom_data = pipe_custom_find_data (PIPE_CUSTOM_TYPE_FILTER, model);
664 3 : if (custom_data) {
665 3 : ml_pipeline_custom_filter_ref (custom_data->handle);
666 :
667 3 : e->custom_destroy = pipe_custom_destroy_cb;
668 3 : e->custom_data = custom_data;
669 : }
670 : }
671 :
672 39 : g_free (fw);
673 39 : g_free (model);
674 39 : }
675 :
676 : /**
677 : * @brief Handle tensor-if options.
678 : */
679 : static void
680 3 : process_tensor_if_option (ml_pipeline_element * e)
681 : {
682 3 : gint cv = 0;
683 3 : gchar *cv_option = NULL;
684 : pipe_custom_data_s *custom_data;
685 :
686 3 : g_object_get (G_OBJECT (e->element), "compared-value", &cv,
687 : "compared-value-option", &cv_option, NULL);
688 :
689 3 : if (cv == 5) {
690 : /* cv is TIFCV_CUSTOM, ref to tensor-if custom handle. */
691 3 : custom_data = pipe_custom_find_data (PIPE_CUSTOM_TYPE_IF, cv_option);
692 3 : if (custom_data) {
693 3 : ml_pipeline_if_custom_ref (custom_data->handle);
694 :
695 3 : e->custom_destroy = pipe_custom_destroy_cb;
696 3 : e->custom_data = custom_data;
697 : }
698 : }
699 :
700 3 : g_free (cv_option);
701 3 : }
702 :
703 : /**
704 : * @brief Initializes the GStreamer library. This is internal function.
705 : */
706 : int
707 233 : _ml_initialize_gstreamer (void)
708 : {
709 233 : GError *err = NULL;
710 :
711 233 : if (!gst_init_check (NULL, NULL, &err)) {
712 0 : if (err) {
713 0 : _ml_error_report
714 : ("Initrializing ML-API failed: GStreamer has the following error from gst_init_check(): %s",
715 : err->message);
716 0 : g_clear_error (&err);
717 : } else {
718 0 : _ml_error_report ("Cannot initialize GStreamer. Unknown reason.");
719 : }
720 :
721 233 : return ML_ERROR_STREAMS_PIPE;
722 : }
723 :
724 233 : return ML_ERROR_NONE;
725 : }
726 :
727 : /**
728 : * @brief Checks the element is registered and available on the pipeline.
729 : */
730 : int
731 79 : ml_check_element_availability (const char *element_name, bool *available)
732 : {
733 : GstElementFactory *factory;
734 : int status;
735 :
736 79 : check_feature_state (ML_FEATURE_INFERENCE);
737 :
738 79 : if (!element_name)
739 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
740 : "The parameter, element_name, is NULL. It should be a name (string) to be queried if it exists as a GStreamer/NNStreamer element.");
741 :
742 78 : if (!available)
743 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
744 : "The parameter, available, is NULL. It should be a valid pointer to a bool entry so that the API (ml_check_element_availability) may return the queried result via \"available\" parameter. E.g., bool available; ml_check_element_availability (\"tensor_converter\", &available);");
745 :
746 77 : _ml_error_report_return_continue_iferr (_ml_initialize_gstreamer (),
747 : "Internal error of _ml_initialize_gstreamer(). Check the availability of gstreamer libraries in your system.");
748 :
749 : /* init false */
750 77 : *available = false;
751 :
752 77 : factory = gst_element_factory_find (element_name);
753 77 : if (factory) {
754 76 : GstPluginFeature *feature = GST_PLUGIN_FEATURE (factory);
755 76 : const gchar *plugin_name = gst_plugin_feature_get_plugin_name (feature);
756 :
757 : /* check restricted element */
758 76 : status = _ml_check_plugin_availability (plugin_name, element_name);
759 76 : if (status == ML_ERROR_NONE)
760 63 : *available = true;
761 :
762 76 : gst_object_unref (factory);
763 : }
764 :
765 77 : return ML_ERROR_NONE;
766 : }
767 :
768 : /**
769 : * @brief Checks the availability of the plugin.
770 : */
771 : int
772 927 : _ml_check_plugin_availability (const char *plugin_name,
773 : const char *element_name)
774 : {
775 : static gboolean list_loaded = FALSE;
776 : static gchar **allowed_elements = NULL;
777 :
778 927 : if (!plugin_name)
779 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
780 : "The parameter, plugin_name, is NULL. It should be a valid string.");
781 :
782 926 : if (!element_name)
783 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
784 : "The parameter, element_name, is NULL. It should be a valid string.");
785 :
786 925 : if (!list_loaded) {
787 : gboolean restricted;
788 :
789 : restricted =
790 7 : nnsconf_get_custom_value_bool ("element-restriction",
791 : "enable_element_restriction", FALSE);
792 7 : if (restricted) {
793 : gchar *elements;
794 :
795 : /* check white-list of available plugins */
796 : elements =
797 7 : nnsconf_get_custom_value_string ("element-restriction",
798 : "allowed_elements");
799 7 : if (elements) {
800 7 : allowed_elements = g_strsplit_set (elements, " ,;", -1);
801 7 : g_free (elements);
802 : }
803 : }
804 :
805 7 : list_loaded = TRUE;
806 : }
807 :
808 : /* nnstreamer elements */
809 925 : if (g_str_has_prefix (plugin_name, "nnstreamer") &&
810 272 : g_str_has_prefix (element_name, "tensor_")) {
811 272 : return ML_ERROR_NONE;
812 : }
813 :
814 653 : if (allowed_elements &&
815 653 : find_key_strv ((const gchar **) allowed_elements, element_name) < 0) {
816 13 : _ml_error_report_return (ML_ERROR_NOT_SUPPORTED,
817 : "The element %s is restricted.", element_name);
818 : }
819 :
820 640 : return ML_ERROR_NONE;
821 : }
822 :
823 : /**
824 : * @brief Get the ml_pipeline_element_e type from its element name
825 : */
826 : static ml_pipeline_element_e
827 860 : get_elem_type_from_name (GHashTable * table, const gchar * name)
828 : {
829 860 : gpointer value = g_hash_table_lookup (table, name);
830 860 : if (!value)
831 504 : return ML_PIPELINE_ELEMENT_UNKNOWN;
832 :
833 356 : return GPOINTER_TO_INT (value);
834 : }
835 :
836 : /**
837 : * @brief Iterate elements and prepare element handle.
838 : */
839 : static int
840 145 : iterate_element (ml_pipeline * pipe_h, GstElement * pipeline,
841 : gboolean is_internal)
842 : {
843 145 : GstIterator *it = NULL;
844 145 : int status = ML_ERROR_NONE;
845 :
846 145 : g_return_val_if_fail (pipe_h && pipeline, ML_ERROR_INVALID_PARAMETER);
847 :
848 145 : g_mutex_lock (&pipe_h->lock);
849 :
850 145 : it = gst_bin_iterate_elements (GST_BIN (pipeline));
851 145 : if (it != NULL) {
852 145 : gboolean done = FALSE;
853 145 : GValue item = G_VALUE_INIT;
854 : GObject *obj;
855 : gchar *name;
856 :
857 : /* Fill in the hashtable, "namednodes" with named Elements */
858 1295 : while (!done) {
859 1005 : switch (gst_iterator_next (it, &item)) {
860 860 : case GST_ITERATOR_OK:
861 860 : obj = g_value_get_object (&item);
862 :
863 860 : if (GST_IS_ELEMENT (obj)) {
864 860 : GstElement *elem = GST_ELEMENT (obj);
865 : GstPluginFeature *feature =
866 860 : GST_PLUGIN_FEATURE (gst_element_get_factory (elem));
867 : const gchar *plugin_name =
868 860 : gst_plugin_feature_get_plugin_name (feature);
869 860 : const gchar *element_name = gst_plugin_feature_get_name (feature);
870 :
871 : /* validate the availability of the plugin */
872 860 : if (!is_internal && _ml_check_plugin_availability (plugin_name,
873 : element_name) != ML_ERROR_NONE) {
874 0 : _ml_error_report_continue
875 : ("There is a pipeline element (filter) that is not allowed for applications via ML-API (privilege not granted) or now available: '%s'/'%s'.",
876 : plugin_name, element_name);
877 0 : status = ML_ERROR_NOT_SUPPORTED;
878 0 : done = TRUE;
879 0 : break;
880 : }
881 :
882 860 : name = gst_element_get_name (elem);
883 860 : if (name != NULL) {
884 : ml_pipeline_element_e element_type =
885 860 : get_elem_type_from_name (pipe_h->pipe_elm_type, element_name);
886 :
887 : /* check 'sync' property in sink element */
888 860 : if (element_type == ML_PIPELINE_ELEMENT_SINK ||
889 : element_type == ML_PIPELINE_ELEMENT_APP_SINK) {
890 137 : gboolean sync = FALSE;
891 :
892 137 : g_object_get (G_OBJECT (elem), "sync", &sync, NULL);
893 137 : if (sync) {
894 1 : _ml_logw (_ml_detail
895 : ("It is recommended to apply 'sync=false' property to a sink element in most AI applications. Otherwise, inference results of large neural networks will be frequently dropped by the synchronization mechanism at the sink element."));
896 : }
897 : }
898 :
899 860 : if (element_type != ML_PIPELINE_ELEMENT_UNKNOWN) {
900 : ml_pipeline_element *e;
901 :
902 356 : e = construct_element (gst_object_ref (elem), pipe_h, name,
903 : element_type);
904 356 : if (e != NULL) {
905 356 : if (g_str_equal (element_name, "tensor_if"))
906 3 : process_tensor_if_option (e);
907 353 : else if (g_str_equal (element_name, "tensor_filter"))
908 39 : process_tensor_filter_option (e);
909 :
910 356 : g_hash_table_insert (pipe_h->namednodes, g_strdup (name), e);
911 : } else {
912 : /* allocation failure */
913 0 : gst_object_unref (elem);
914 0 : _ml_error_report_continue
915 : ("Cannot allocate memory with construct_element().");
916 0 : status = ML_ERROR_OUT_OF_MEMORY;
917 0 : done = TRUE;
918 : }
919 : }
920 :
921 860 : g_free (name);
922 : }
923 : }
924 :
925 860 : g_value_reset (&item);
926 860 : break;
927 0 : case GST_ITERATOR_RESYNC:
928 : case GST_ITERATOR_ERROR:
929 0 : _ml_logw (_ml_detail
930 : ("There is an error or a resync-event while inspecting a pipeline. However, we can still execute the pipeline."));
931 : /* fallthrough */
932 145 : case GST_ITERATOR_DONE:
933 145 : done = TRUE;
934 : }
935 : }
936 :
937 145 : g_value_unset (&item);
938 : /** @todo CRITICAL check the validity of elem=item registered in e */
939 145 : gst_iterator_free (it);
940 : }
941 :
942 145 : g_mutex_unlock (&pipe_h->lock);
943 145 : return status;
944 : }
945 :
946 : /**
947 : * @brief Internal function to create the hash table for managing internal resources
948 : */
949 : static void
950 148 : create_internal_hash (ml_pipeline * pipe_h)
951 : {
952 148 : pipe_h->namednodes =
953 148 : g_hash_table_new_full (g_str_hash, g_str_equal, g_free, cleanup_node);
954 148 : pipe_h->resources =
955 148 : g_hash_table_new_full (g_str_hash, g_str_equal, g_free, cleanup_resource);
956 :
957 148 : pipe_h->pipe_elm_type =
958 148 : g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
959 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("tensor_sink"),
960 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_SINK));
961 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("appsrc"),
962 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_APP_SRC));
963 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("appsink"),
964 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_APP_SINK));
965 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("valve"),
966 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_VALVE));
967 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("input-selector"),
968 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_SWITCH_INPUT));
969 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("output-selector"),
970 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_SWITCH_OUTPUT));
971 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("tensor_if"),
972 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_COMMON));
973 148 : g_hash_table_insert (pipe_h->pipe_elm_type, g_strdup ("tensor_filter"),
974 : GINT_TO_POINTER (ML_PIPELINE_ELEMENT_COMMON));
975 148 : }
976 :
977 : /**
978 : * @brief Internal function to construct the pipeline.
979 : * If is_internal is true, this will ignore the permission in Tizen.
980 : */
981 : static int
982 150 : construct_pipeline_internal (const char *pipeline_description,
983 : ml_pipeline_state_cb cb, void *user_data, ml_pipeline_h * pipe,
984 : gboolean is_internal)
985 : {
986 150 : GError *err = NULL;
987 : GstElement *pipeline;
988 150 : gchar *description = NULL;
989 150 : int status = ML_ERROR_NONE;
990 :
991 : ml_pipeline *pipe_h;
992 :
993 300 : check_feature_state (ML_FEATURE_INFERENCE);
994 :
995 150 : if (!pipe)
996 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
997 : "ml_pipeline_construct error: parameter pipe is NULL. It should be a valid ml_pipeline_h pointer. E.g., ml_pipeline_h pipe; ml_pipeline_construct (..., &pip);");
998 :
999 149 : if (!pipeline_description)
1000 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1001 : "ml_pipeline_construct error: parameter pipeline_description is NULL. It should be a valid string of Gstreamer/NNStreamer pipeline description.");
1002 :
1003 : /* init null */
1004 148 : *pipe = NULL;
1005 :
1006 148 : _ml_error_report_return_continue_iferr (_ml_initialize_gstreamer (),
1007 : "ml_pipeline_construct error: it has failed to initialize gstreamer(). Please check if you have a valid GStreamer library installed in your system.");
1008 :
1009 : /* prepare pipeline handle */
1010 148 : pipe_h = g_new0 (ml_pipeline, 1);
1011 148 : if (pipe_h == NULL)
1012 0 : _ml_error_report_return (ML_ERROR_OUT_OF_MEMORY,
1013 : "ml_pipeline_construct error: failed to allocate memory for pipeline handle. Out of memory?");
1014 :
1015 148 : g_mutex_init (&pipe_h->lock);
1016 :
1017 148 : pipe_h->isEOS = FALSE;
1018 148 : pipe_h->pipe_state = ML_PIPELINE_STATE_UNKNOWN;
1019 :
1020 148 : create_internal_hash (pipe_h);
1021 :
1022 : /* convert predefined element and launch the pipeline */
1023 148 : status = convert_description ((ml_pipeline_h) pipe_h, pipeline_description,
1024 : &description, is_internal);
1025 148 : if (status != ML_ERROR_NONE) {
1026 0 : _ml_error_report_continue
1027 : ("ml_pipeline_construct error: failed while converting pipeline description for GStreamer w/ convert_description() function, which has returned %d",
1028 : status);
1029 0 : goto failed;
1030 : }
1031 :
1032 148 : pipeline = gst_parse_launch (description, &err);
1033 148 : g_free (description);
1034 :
1035 148 : if (pipeline == NULL || err) {
1036 3 : _ml_error_report
1037 : ("ml_pipeline_construct error: gst_parse_launch cannot parse and launch the given pipeline = [%s]. The error message from gst_parse_launch is '%s'.",
1038 : pipeline_description, (err) ? err->message : "unknown reason");
1039 3 : g_clear_error (&err);
1040 :
1041 3 : if (pipeline)
1042 3 : gst_object_unref (pipeline);
1043 :
1044 3 : status = ML_ERROR_STREAMS_PIPE;
1045 3 : goto failed;
1046 : }
1047 :
1048 145 : g_assert (GST_IS_PIPELINE (pipeline));
1049 145 : pipe_h->element = pipeline;
1050 :
1051 : /* bus and message callback */
1052 145 : pipe_h->bus = gst_element_get_bus (pipeline);
1053 145 : g_assert (pipe_h->bus);
1054 :
1055 145 : gst_bus_enable_sync_message_emission (pipe_h->bus);
1056 145 : pipe_h->signal_msg = g_signal_connect (pipe_h->bus, "sync-message",
1057 : G_CALLBACK (cb_bus_sync_message), pipe_h);
1058 :
1059 : /* state change callback */
1060 145 : pipe_h->state_cb.cb = cb;
1061 145 : pipe_h->state_cb.user_data = user_data;
1062 :
1063 : /* iterate elements and prepare element handle */
1064 145 : status = iterate_element (pipe_h, pipeline, is_internal);
1065 145 : if (status != ML_ERROR_NONE) {
1066 0 : _ml_error_report_continue ("ml_pipeline_construct error: ...");
1067 0 : goto failed;
1068 : }
1069 :
1070 : /* finally set pipeline state to PAUSED */
1071 145 : status = ml_pipeline_stop ((ml_pipeline_h) pipe_h);
1072 :
1073 145 : if (status == ML_ERROR_NONE) {
1074 : /**
1075 : * Let's wait until the pipeline state is changed to paused.
1076 : * Otherwise, the following APIs like 'set_property' may incur
1077 : * unintended behaviors. But, don't need to return any error
1078 : * even if this state change is not finished within the timeout,
1079 : * just replying on the caller.
1080 : */
1081 144 : gst_element_get_state (pipeline, NULL, NULL, 10 * GST_MSECOND);
1082 : } else {
1083 1 : _ml_error_report_continue
1084 : ("ml_pipeline_construct error: ml_pipeline_stop has failed with %d return. The pipeline should be able to be stopped when it is constructed.",
1085 : status);
1086 : }
1087 :
1088 148 : failed:
1089 148 : if (status != ML_ERROR_NONE) {
1090 : /* failed to construct the pipeline */
1091 4 : ml_pipeline_destroy ((ml_pipeline_h) pipe_h);
1092 : } else {
1093 144 : *pipe = pipe_h;
1094 : }
1095 :
1096 148 : return status;
1097 : }
1098 :
1099 : /**
1100 : * @brief Construct the pipeline (more info in nnstreamer.h)
1101 : */
1102 : int
1103 148 : ml_pipeline_construct (const char *pipeline_description,
1104 : ml_pipeline_state_cb cb, void *user_data, ml_pipeline_h * pipe)
1105 : {
1106 : /* not an internal pipeline construction */
1107 148 : return construct_pipeline_internal (pipeline_description, cb, user_data, pipe,
1108 : FALSE);
1109 : }
1110 :
1111 : #if defined (__TIZEN__)
1112 : /**
1113 : * @brief Construct the pipeline (Tizen internal, see nnstreamer-tizen-internal.h)
1114 : */
1115 : int
1116 2 : ml_pipeline_construct_internal (const char *pipeline_description,
1117 : ml_pipeline_state_cb cb, void *user_data, ml_pipeline_h * pipe)
1118 : {
1119 : /* Tizen internal pipeline construction */
1120 2 : return construct_pipeline_internal (pipeline_description, cb, user_data, pipe,
1121 : TRUE);
1122 : }
1123 : #endif /* __TIZEN__ */
1124 :
1125 : /**
1126 : * @brief Destroy the pipeline (more info in nnstreamer.h)
1127 : */
1128 : int
1129 143 : ml_pipeline_destroy (ml_pipeline_h pipe)
1130 : {
1131 143 : ml_pipeline *p = pipe;
1132 : GstStateChangeReturn scret;
1133 : GstState state;
1134 143 : guint check_paused_cnt = 0;
1135 :
1136 286 : check_feature_state (ML_FEATURE_INFERENCE);
1137 :
1138 143 : if (p == NULL)
1139 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1140 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h handle instance, usually created by ml_pipeline_construct().");
1141 :
1142 143 : g_mutex_lock (&p->lock);
1143 :
1144 : /* Before changing the state, remove all callbacks. */
1145 143 : p->state_cb.cb = NULL;
1146 :
1147 : /* Destroy registered callback handles and resources */
1148 143 : g_hash_table_destroy (p->namednodes);
1149 143 : g_hash_table_destroy (p->resources);
1150 143 : g_hash_table_destroy (p->pipe_elm_type);
1151 143 : p->namednodes = p->resources = p->pipe_elm_type = NULL;
1152 :
1153 143 : if (p->element) {
1154 : /* Pause the pipeline if it's playing */
1155 140 : scret = gst_element_get_state (p->element, &state, NULL, 10 * GST_MSECOND); /* 10ms */
1156 140 : if (scret != GST_STATE_CHANGE_FAILURE && state == GST_STATE_PLAYING) {
1157 3 : scret = gst_element_set_state (p->element, GST_STATE_PAUSED);
1158 3 : if (scret == GST_STATE_CHANGE_FAILURE) {
1159 0 : g_mutex_unlock (&p->lock);
1160 0 : _ml_error_report_return (ML_ERROR_STREAMS_PIPE,
1161 : "gst_element_get_state() has failed to wait until state changed from PLAYING to PAUSED and returned GST_STATE_CHANGE_FAILURE. For the detail, please check the GStreamer log messages (or dlog messages in Tizen). It is possible that there is a filter or neural network that is taking too much time to finish.");
1162 : }
1163 : }
1164 :
1165 140 : g_mutex_unlock (&p->lock);
1166 239 : while (p->pipe_state == ML_PIPELINE_STATE_PLAYING) {
1167 100 : check_paused_cnt++;
1168 : /** check PAUSED every 1ms */
1169 100 : g_usleep (1000);
1170 100 : if (check_paused_cnt >= WAIT_PAUSED_TIME_LIMIT) {
1171 1 : _ml_error_report
1172 : ("Timeout while waiting for a state change to 'PAUSED' from a 'sync-message' signal from the pipeline. It is possible that there is a filter or neural network that is taking too much time to finish.");
1173 1 : break;
1174 : }
1175 : }
1176 140 : g_mutex_lock (&p->lock);
1177 :
1178 : /* Stop (NULL State) the pipeline */
1179 140 : scret = gst_element_set_state (p->element, GST_STATE_NULL);
1180 140 : if (scret != GST_STATE_CHANGE_SUCCESS) {
1181 0 : g_mutex_unlock (&p->lock);
1182 0 : _ml_error_report_return (ML_ERROR_STREAMS_PIPE,
1183 : "gst_element_set_state to stop the pipeline has failed after trying to stop the pipeline with PAUSE and waiting for stopping. For the detail, please check the GStreamer log messages. It is possible that there is a filter of neural network that is taking too much time to finish.");
1184 : }
1185 :
1186 140 : if (p->bus) {
1187 140 : g_signal_handler_disconnect (p->bus, p->signal_msg);
1188 140 : gst_object_unref (p->bus);
1189 : }
1190 :
1191 140 : gst_object_unref (p->element);
1192 140 : p->element = NULL;
1193 : }
1194 :
1195 143 : g_mutex_unlock (&p->lock);
1196 143 : g_mutex_clear (&p->lock);
1197 :
1198 143 : g_free (p);
1199 143 : return ML_ERROR_NONE;
1200 : }
1201 :
1202 : /**
1203 : * @brief Get the pipeline state (more info in nnstreamer.h)
1204 : */
1205 : int
1206 43 : ml_pipeline_get_state (ml_pipeline_h pipe, ml_pipeline_state_e * state)
1207 : {
1208 43 : ml_pipeline *p = pipe;
1209 : GstState _state;
1210 : GstStateChangeReturn scret;
1211 :
1212 86 : check_feature_state (ML_FEATURE_INFERENCE);
1213 :
1214 43 : if (p == NULL)
1215 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1216 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h handle, which is usually created by ml_pipeline_construct ().");
1217 43 : if (state == NULL)
1218 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1219 : "The parameter, state, is NULL. It should be a valid pointer of ml_pipeline_state_e. E.g., ml_pipeline_state_e state; ml_pipeline_get_state(pipe, &state);");
1220 :
1221 43 : *state = ML_PIPELINE_STATE_UNKNOWN;
1222 :
1223 43 : g_mutex_lock (&p->lock);
1224 43 : scret = gst_element_get_state (p->element, &_state, NULL, GST_MSECOND); /* Do it within 1ms! */
1225 43 : g_mutex_unlock (&p->lock);
1226 :
1227 43 : if (scret == GST_STATE_CHANGE_FAILURE)
1228 0 : _ml_error_report_return (ML_ERROR_STREAMS_PIPE,
1229 : "Failed to get the state of the pipeline. For the detail, please check the GStreamer log messages.");
1230 :
1231 43 : *state = (ml_pipeline_state_e) _state;
1232 43 : return ML_ERROR_NONE;
1233 : }
1234 :
1235 : /****************************************************
1236 : ** NNStreamer Pipeline Start/Stop Control **
1237 : ****************************************************/
1238 : /**
1239 : * @brief Start/Resume the pipeline! (more info in nnstreamer.h)
1240 : */
1241 : int
1242 53 : ml_pipeline_start (ml_pipeline_h pipe)
1243 : {
1244 53 : ml_pipeline *p = pipe;
1245 : GstStateChangeReturn scret;
1246 53 : int status = ML_ERROR_NONE;
1247 :
1248 53 : check_feature_state (ML_FEATURE_INFERENCE);
1249 :
1250 53 : if (p == NULL)
1251 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1252 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h handle, which is usually created by ml_pipeline_construct ().");
1253 :
1254 53 : g_mutex_lock (&p->lock);
1255 :
1256 : /* check the resources when starting the pipeline */
1257 53 : if (g_hash_table_size (p->resources)) {
1258 : GHashTableIter iter;
1259 : gpointer key, value;
1260 :
1261 : /* iterate all handle and acquire res if released */
1262 0 : g_hash_table_iter_init (&iter, p->resources);
1263 0 : while (g_hash_table_iter_next (&iter, &key, &value)) {
1264 0 : if (g_str_has_prefix (key, "tizen")) {
1265 0 : status = get_tizen_resource (pipe, key);
1266 0 : if (status != ML_ERROR_NONE) {
1267 0 : _ml_error_report_continue
1268 : ("Internal API _ml_tizen_get_resource () has failed: Tizen mm resource manager has failed to acquire the resource of '%s'",
1269 : (gchar *) key);
1270 0 : goto done;
1271 : }
1272 : }
1273 : }
1274 : }
1275 :
1276 53 : scret = gst_element_set_state (p->element, GST_STATE_PLAYING);
1277 53 : if (scret == GST_STATE_CHANGE_FAILURE) {
1278 0 : _ml_error_report
1279 : ("Failed to set the state of the pipeline to PLAYING. For the detail, please check the GStreamer log messages.");
1280 0 : status = ML_ERROR_STREAMS_PIPE;
1281 : }
1282 :
1283 53 : done:
1284 53 : g_mutex_unlock (&p->lock);
1285 53 : return status;
1286 : }
1287 :
1288 : /**
1289 : * @brief Pause the pipeline! (more info in nnstreamer.h)
1290 : */
1291 : int
1292 193 : ml_pipeline_stop (ml_pipeline_h pipe)
1293 : {
1294 193 : ml_pipeline *p = pipe;
1295 : GstStateChangeReturn scret;
1296 :
1297 193 : check_feature_state (ML_FEATURE_INFERENCE);
1298 :
1299 193 : if (p == NULL)
1300 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1301 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
1302 :
1303 193 : g_mutex_lock (&p->lock);
1304 193 : scret = gst_element_set_state (p->element, GST_STATE_PAUSED);
1305 193 : g_mutex_unlock (&p->lock);
1306 :
1307 193 : if (scret == GST_STATE_CHANGE_FAILURE)
1308 1 : _ml_error_report_return (ML_ERROR_STREAMS_PIPE,
1309 : "Failed to set the state of the pipeline to PAUSED. For the detail, please check the GStreamer log messages.");
1310 :
1311 192 : return ML_ERROR_NONE;
1312 : }
1313 :
1314 : /**
1315 : * @brief Clears all data and resets the running-time of the pipeline (more info in nnstreamer.h)
1316 : */
1317 : int
1318 2 : ml_pipeline_flush (ml_pipeline_h pipe, bool start)
1319 : {
1320 2 : ml_pipeline *p = pipe;
1321 2 : int status = ML_ERROR_NONE;
1322 :
1323 2 : check_feature_state (ML_FEATURE_INFERENCE);
1324 :
1325 2 : if (p == NULL)
1326 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1327 : "The parameter, pipe, is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
1328 :
1329 1 : _ml_error_report_return_continue_iferr (ml_pipeline_stop (pipe),
1330 : "Failed to stop the pipeline with ml_pipeline_stop (). It has returned %d.",
1331 : _ERRNO);
1332 :
1333 1 : _ml_logi ("The pipeline is stopped, clear all data from the pipeline.");
1334 :
1335 : /* send flush event to pipeline */
1336 1 : g_mutex_lock (&p->lock);
1337 1 : if (!gst_element_send_event (p->element, gst_event_new_flush_start ())) {
1338 0 : _ml_logw ("Error occurs while sending flush_start event.");
1339 : }
1340 :
1341 1 : if (!gst_element_send_event (p->element, gst_event_new_flush_stop (TRUE))) {
1342 0 : _ml_logw ("Error occurs while sending flush_stop event.");
1343 : }
1344 1 : g_mutex_unlock (&p->lock);
1345 :
1346 1 : if (start && status == ML_ERROR_NONE)
1347 1 : status = ml_pipeline_start (pipe);
1348 :
1349 1 : return status;
1350 : }
1351 :
1352 : /****************************************************
1353 : ** NNStreamer Pipeline Sink/Src Control **
1354 : ****************************************************/
1355 : /**
1356 : * @brief Register a callback for sink (more info in nnstreamer.h)
1357 : */
1358 : int
1359 46 : ml_pipeline_sink_register (ml_pipeline_h pipe, const char *sink_name,
1360 : ml_pipeline_sink_cb cb, void *user_data, ml_pipeline_sink_h * h)
1361 : {
1362 : ml_pipeline_element *elem;
1363 46 : ml_pipeline *p = pipe;
1364 : ml_pipeline_common_elem *sink;
1365 46 : int ret = ML_ERROR_NONE;
1366 :
1367 46 : check_feature_state (ML_FEATURE_INFERENCE);
1368 :
1369 46 : if (h == NULL)
1370 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1371 : "The argument, h (ml_pipeline_sink_h), is NULL. It should be a valid pointer to a new ml_pipeline_sink_h instance. E.g., ml_pipeline_sink_h h; ml_pipeline_sink_register (...., &h);");
1372 :
1373 : /* init null */
1374 45 : *h = NULL;
1375 :
1376 45 : if (pipe == NULL)
1377 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1378 : "The argument, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h instance, usually created by ml_pipeline_construct.");
1379 :
1380 44 : if (sink_name == NULL)
1381 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1382 : "The argument, sink_name (const char *), is NULL. It should be a valid string naming the sink handle (h).");
1383 :
1384 43 : if (cb == NULL)
1385 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1386 : "The argument, cb (ml_pipeline_sink_cb), is NULL. It should be a call-back function called for data-sink events.");
1387 :
1388 42 : g_mutex_lock (&p->lock);
1389 42 : elem = g_hash_table_lookup (p->namednodes, sink_name);
1390 :
1391 42 : if (elem == NULL) {
1392 1 : _ml_error_report
1393 : ("There is no element named [%s](sink_name) in the pipeline. Please check your pipeline description.",
1394 : sink_name);
1395 1 : ret = ML_ERROR_INVALID_PARAMETER;
1396 1 : goto unlock_return;
1397 : }
1398 :
1399 41 : if (elem->type != ML_PIPELINE_ELEMENT_SINK &&
1400 5 : elem->type != ML_PIPELINE_ELEMENT_APP_SINK) {
1401 1 : _ml_error_report
1402 : ("The element [%s](sink_name) in the pipeline is not a sink element. Please supply the name of tensor_sink or appsink.",
1403 : sink_name);
1404 1 : ret = ML_ERROR_INVALID_PARAMETER;
1405 1 : goto unlock_return;
1406 : }
1407 :
1408 40 : if (elem->handle_id > 0) {
1409 : /* no need to connect signal to sink element */
1410 1 : _ml_logw ("Sink callback is already registered.");
1411 : } else {
1412 : /* set callback for new data */
1413 39 : if (elem->type == ML_PIPELINE_ELEMENT_SINK) {
1414 : /* tensor_sink */
1415 36 : g_object_set (G_OBJECT (elem->element), "emit-signal", (gboolean) TRUE,
1416 : NULL);
1417 36 : elem->handle_id =
1418 36 : g_signal_connect (elem->element, "new-data",
1419 : G_CALLBACK (cb_sink_event), elem);
1420 : } else {
1421 : /* appsink */
1422 3 : g_object_set (G_OBJECT (elem->element), "emit-signals", (gboolean) TRUE,
1423 : NULL);
1424 3 : elem->handle_id =
1425 3 : g_signal_connect (elem->element, "new-sample",
1426 : G_CALLBACK (cb_appsink_new_sample), elem);
1427 : }
1428 :
1429 39 : if (elem->handle_id == 0) {
1430 0 : _ml_error_report
1431 : ("Failed to connect a signal to the element [%s](sink_name). g_signal_connect has returned NULL.",
1432 : sink_name);
1433 0 : ret = ML_ERROR_STREAMS_PIPE;
1434 0 : goto unlock_return;
1435 : }
1436 : }
1437 :
1438 40 : sink = g_new0 (ml_pipeline_common_elem, 1);
1439 40 : if (sink == NULL) {
1440 0 : _ml_error_report
1441 : ("Failed to allocate memory for the sink handle of %s. Out of memory?",
1442 : sink_name);
1443 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1444 0 : goto unlock_return;
1445 : }
1446 :
1447 40 : sink->callback_info = g_new0 (callback_info_s, 1);
1448 40 : if (sink->callback_info == NULL) {
1449 0 : g_free (sink);
1450 0 : _ml_error_report
1451 : ("Failed to allocate memory for the sink handle of %s. Out of memory?",
1452 : sink_name);
1453 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1454 0 : goto unlock_return;
1455 : }
1456 :
1457 40 : sink->pipe = p;
1458 40 : sink->element = elem;
1459 40 : sink->callback_info->sink_cb = cb;
1460 40 : sink->callback_info->sink_pdata = user_data;
1461 40 : *h = sink;
1462 :
1463 40 : g_mutex_lock (&elem->lock);
1464 :
1465 40 : elem->maxid++;
1466 40 : sink->id = elem->maxid;
1467 40 : elem->handles = g_list_append (elem->handles, sink);
1468 :
1469 40 : g_mutex_unlock (&elem->lock);
1470 :
1471 42 : unlock_return:
1472 42 : g_mutex_unlock (&p->lock);
1473 42 : return ret;
1474 : }
1475 :
1476 : /**
1477 : * @brief Unregister a callback for sink (more info in nnstreamer.h)
1478 : */
1479 : int
1480 13 : ml_pipeline_sink_unregister (ml_pipeline_sink_h h)
1481 : {
1482 13 : handle_init (sink, h);
1483 :
1484 13 : if (elem->handle_id > 0) {
1485 12 : g_signal_handler_disconnect (elem->element, elem->handle_id);
1486 12 : elem->handle_id = 0;
1487 : }
1488 :
1489 13 : elem->handles = g_list_remove (elem->handles, sink);
1490 13 : free_element_handle (sink);
1491 :
1492 13 : handle_exit (h);
1493 : }
1494 :
1495 : /**
1496 : * @brief Parse tensors info of src element.
1497 : */
1498 : static int
1499 12438 : ml_pipeline_src_parse_tensors_info (ml_pipeline_element * elem)
1500 : {
1501 12438 : GstCaps *caps = NULL;
1502 12438 : gboolean found = FALSE, flexible = FALSE;
1503 :
1504 12438 : if (elem->src == NULL) {
1505 41 : elem->src = gst_element_get_static_pad (elem->element, "src");
1506 : }
1507 :
1508 12438 : if (elem->src == NULL) {
1509 0 : _ml_error_report
1510 : ("Failed to get the src pad of the element[%s]. The designated source element does not have available src pad? For the detail, please check the GStreamer log messages.",
1511 : elem->name);
1512 12438 : return ML_ERROR_STREAMS_PIPE;
1513 : }
1514 :
1515 : /* If caps is given, use it. e.g. Use cap "image/png" when the pipeline is */
1516 : /* given as "appsrc caps=image/png ! pngdec ! ... " */
1517 12438 : caps = gst_pad_get_current_caps (elem->src);
1518 12438 : if (!caps)
1519 12374 : caps = gst_pad_get_allowed_caps (elem->src);
1520 :
1521 12438 : if (!caps) {
1522 0 : _ml_logw
1523 : ("Cannot find caps. The pipeline is not yet negotiated for src element [%s].",
1524 : elem->name);
1525 0 : gst_object_unref (elem->src);
1526 0 : elem->src = NULL;
1527 0 : return ML_ERROR_TRY_AGAIN;
1528 : }
1529 :
1530 12438 : found = get_tensors_info_from_caps (caps, &elem->tensors_info, &flexible);
1531 :
1532 12438 : if (found) {
1533 12432 : elem->is_flexible_tensor = flexible;
1534 : } else {
1535 6 : if (gst_caps_is_fixed (caps)) {
1536 5 : GstStructure *st = gst_caps_get_structure (caps, 0);
1537 5 : elem->is_media_stream = !gst_structure_is_tensor_stream (st);
1538 : }
1539 : }
1540 :
1541 12438 : gst_caps_unref (caps);
1542 12438 : return ML_ERROR_NONE;
1543 : }
1544 :
1545 : /**
1546 : * @brief Get a handle to operate a src (more info in nnstreamer.h)
1547 : */
1548 : int
1549 48 : ml_pipeline_src_get_handle (ml_pipeline_h pipe, const char *src_name,
1550 : ml_pipeline_src_h * h)
1551 : {
1552 48 : ml_pipeline *p = pipe;
1553 : ml_pipeline_element *elem;
1554 : ml_pipeline_common_elem *src;
1555 48 : int ret = ML_ERROR_NONE;
1556 :
1557 48 : check_feature_state (ML_FEATURE_INFERENCE);
1558 :
1559 48 : if (h == NULL)
1560 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1561 : "The parameter, h (ml_pipeline_src_h), is NULL. It should be a valid pointer to a new (or to be cleared) instance. E.g., ml_pipeline_src_h h; ml_pipeline_src_get_handle (..., &h);");
1562 :
1563 : /* init null */
1564 47 : *h = NULL;
1565 :
1566 47 : if (pipe == NULL)
1567 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1568 : "The parameter, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
1569 :
1570 46 : if (src_name == NULL)
1571 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1572 : "The parameter, src_name (const char *), is NULL. This string is the name of source element (appsrc) you want to push data stream from your application threads.");
1573 :
1574 45 : g_mutex_lock (&p->lock);
1575 :
1576 45 : elem = g_hash_table_lookup (p->namednodes, src_name);
1577 :
1578 45 : if (elem == NULL) {
1579 1 : _ml_error_report
1580 : ("Cannot find the name, '%s': there is no element named [%s] in the given pipeline.",
1581 : src_name, src_name);
1582 1 : ret = ML_ERROR_INVALID_PARAMETER;
1583 1 : goto unlock_return;
1584 : }
1585 :
1586 44 : if (elem->type != ML_PIPELINE_ELEMENT_APP_SRC) {
1587 1 : _ml_error_report
1588 : ("The element designated by '%s' is not a source element (appsrc). Please provide a name of source element for ml_pipeline_src_get_handle API.",
1589 : src_name);
1590 1 : ret = ML_ERROR_INVALID_PARAMETER;
1591 1 : goto unlock_return;
1592 : }
1593 :
1594 43 : src = *h = g_new0 (ml_pipeline_common_elem, 1);
1595 43 : if (src == NULL) {
1596 0 : _ml_error_report
1597 : ("Failed to allocate the src handle for %s. Out of memory?", src_name);
1598 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1599 0 : goto unlock_return;
1600 : }
1601 :
1602 43 : src->pipe = p;
1603 43 : src->element = elem;
1604 :
1605 43 : g_mutex_lock (&elem->lock);
1606 :
1607 43 : elem->maxid++;
1608 43 : src->id = elem->maxid;
1609 43 : elem->handles = g_list_append (elem->handles, src);
1610 :
1611 43 : ml_pipeline_src_parse_tensors_info (elem);
1612 43 : g_mutex_unlock (&elem->lock);
1613 :
1614 45 : unlock_return:
1615 45 : g_mutex_unlock (&p->lock);
1616 :
1617 45 : return ret;
1618 : }
1619 :
1620 : /**
1621 : * @brief Close a src node (more info in nnstreamer.h)
1622 : */
1623 : int
1624 7 : ml_pipeline_src_release_handle (ml_pipeline_src_h h)
1625 : {
1626 7 : handle_init (src, h);
1627 :
1628 7 : elem->handles = g_list_remove (elem->handles, src);
1629 7 : free_element_handle (src);
1630 :
1631 7 : handle_exit (h);
1632 : }
1633 :
1634 : /**
1635 : * @brief Push a data frame to a src (more info in nnstreamer.h)
1636 : */
1637 : int
1638 6240 : ml_pipeline_src_input_data (ml_pipeline_src_h h, ml_tensors_data_h data,
1639 : ml_pipeline_buf_policy_e policy)
1640 : {
1641 : GstBuffer *buffer;
1642 : GstMemory *mem, *tmp;
1643 : gpointer mem_data;
1644 : gsize mem_size;
1645 : GstFlowReturn gret;
1646 : GstTensorsInfo gst_info;
1647 : ml_tensors_data_s *_data;
1648 : unsigned int i;
1649 :
1650 12480 : handle_init (src, h);
1651 :
1652 6240 : _data = (ml_tensors_data_s *) data;
1653 6240 : if (!_data) {
1654 1 : _ml_error_report
1655 : ("The given parameter, data (ml_tensors_data_h), is NULL. It should be a valid ml_tensor_data_h instance, which is usually created by ml_tensors_data_create().");
1656 1 : ret = ML_ERROR_INVALID_PARAMETER;
1657 1 : goto unlock_return;
1658 : }
1659 6239 : G_LOCK_UNLESS_NOLOCK (*_data);
1660 :
1661 6239 : if (_data->num_tensors < 1 || _data->num_tensors > ML_TENSOR_SIZE_LIMIT) {
1662 0 : _ml_error_report
1663 : ("The number of tensors of the given data (ml_tensors_data_h) is invalid. The number of tensors of data is %u. It should be between 1 and %u.",
1664 : _data->num_tensors, ML_TENSOR_SIZE_LIMIT);
1665 0 : ret = ML_ERROR_INVALID_PARAMETER;
1666 0 : goto dont_destroy_data;
1667 : }
1668 :
1669 6239 : ret = ml_pipeline_src_parse_tensors_info (elem);
1670 :
1671 6239 : if (ret != ML_ERROR_NONE) {
1672 0 : if (ret == ML_ERROR_TRY_AGAIN)
1673 0 : _ml_error_report_continue
1674 : ("The pipeline is not ready to accept input streams. The input is ignored.");
1675 : else
1676 0 : _ml_error_report_continue
1677 : ("The pipeline is either not ready to accept input streams, yet, or does not have appropriate source elements to accept input streams.");
1678 0 : goto dont_destroy_data;
1679 : }
1680 :
1681 6239 : if (!elem->is_media_stream && !elem->is_flexible_tensor) {
1682 6232 : if (elem->tensors_info.num_tensors != _data->num_tensors) {
1683 0 : _ml_error_report
1684 : ("The src push of [%s] cannot be handled because the number of tensors in a frame mismatches. %u != %u",
1685 : elem->name, elem->tensors_info.num_tensors, _data->num_tensors);
1686 :
1687 0 : ret = ML_ERROR_INVALID_PARAMETER;
1688 0 : goto dont_destroy_data;
1689 : }
1690 :
1691 12617 : for (i = 0; i < _data->num_tensors; i++) {
1692 6387 : size_t sz = gst_tensors_info_get_size (&elem->tensors_info, i);
1693 :
1694 6387 : if (sz != _data->tensors[i].size) {
1695 2 : _ml_error_report
1696 : ("The given input tensor size (%d'th, %zu bytes) mismatches the source pad (%zu bytes)",
1697 : i, _data->tensors[i].size, sz);
1698 :
1699 2 : ret = ML_ERROR_INVALID_PARAMETER;
1700 2 : goto dont_destroy_data;
1701 : }
1702 : }
1703 : }
1704 :
1705 : /* Create buffer to be pushed from buf[] */
1706 6237 : buffer = gst_buffer_new ();
1707 6237 : _ml_tensors_info_copy_from_ml (&gst_info, _data->info);
1708 :
1709 12635 : for (i = 0; i < _data->num_tensors; i++) {
1710 : GstTensorInfo *_gst_tensor_info =
1711 6398 : gst_tensors_info_get_nth_info (&gst_info, i);
1712 6398 : mem_data = _data->tensors[i].data;
1713 6398 : mem_size = _data->tensors[i].size;
1714 :
1715 6398 : mem = tmp = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY,
1716 : mem_data, mem_size, 0, mem_size, mem_data,
1717 : (policy == ML_PIPELINE_BUF_POLICY_AUTO_FREE) ? g_free : NULL);
1718 :
1719 : /* flex tensor, append header. */
1720 6398 : if (elem->is_flexible_tensor) {
1721 : GstTensorMetaInfo meta;
1722 :
1723 9 : gst_tensor_info_convert_to_meta (_gst_tensor_info, &meta);
1724 :
1725 9 : mem = gst_tensor_meta_info_append_header (&meta, tmp);
1726 9 : gst_memory_unref (tmp);
1727 : }
1728 :
1729 6398 : gst_tensor_buffer_append_memory (buffer, mem, _gst_tensor_info);
1730 : /** @todo Verify that gst_buffer_append lists tensors/gstmem in the correct order */
1731 : }
1732 :
1733 6237 : gst_tensors_info_free (&gst_info);
1734 :
1735 : /* Unlock if it's not auto-free. We do not know when it'll be freed. */
1736 6237 : if (policy != ML_PIPELINE_BUF_POLICY_AUTO_FREE)
1737 55 : G_UNLOCK_UNLESS_NOLOCK (*_data);
1738 :
1739 : /* Push the data! */
1740 6237 : gret = gst_app_src_push_buffer (GST_APP_SRC (elem->element), buffer);
1741 :
1742 : /* Free data ptr if buffer policy is auto-free */
1743 6237 : if (policy == ML_PIPELINE_BUF_POLICY_AUTO_FREE) {
1744 6182 : G_UNLOCK_UNLESS_NOLOCK (*_data);
1745 6182 : _ml_tensors_data_destroy_internal (_data, FALSE);
1746 6182 : _data = NULL;
1747 : }
1748 :
1749 6237 : if (gret == GST_FLOW_FLUSHING) {
1750 0 : _ml_logw
1751 : ("The pipeline is not in PAUSED/PLAYING. The input may be ignored.");
1752 0 : ret = ML_ERROR_TRY_AGAIN;
1753 6237 : } else if (gret == GST_FLOW_EOS) {
1754 0 : _ml_logw ("THe pipeline is in EOS state. The input is ignored.");
1755 0 : ret = ML_ERROR_STREAMS_PIPE;
1756 : }
1757 :
1758 6237 : goto unlock_return;
1759 :
1760 2 : dont_destroy_data:
1761 2 : G_UNLOCK_UNLESS_NOLOCK (*_data);
1762 :
1763 6240 : handle_exit (h);
1764 : }
1765 :
1766 : /**
1767 : * @brief Internal function to fetch ml_pipeline_src_callbacks_s pointer
1768 : */
1769 : static ml_pipeline_src_callbacks_s *
1770 6152 : get_app_src_callback (ml_pipeline_common_elem * src_h, void **data)
1771 : {
1772 6152 : ml_pipeline_src_callbacks_s *src_cb = NULL;
1773 :
1774 6152 : if (src_h->callback_info) {
1775 6152 : src_cb = &src_h->callback_info->src_cb;
1776 6152 : *data = src_h->callback_info->src_pdata;
1777 : }
1778 :
1779 6152 : return src_cb;
1780 : }
1781 :
1782 : /**
1783 : * @brief Internal function for appsrc callback - need_data.
1784 : */
1785 : static void
1786 6152 : _pipe_src_cb_need_data (GstAppSrc * src, guint length, gpointer user_data)
1787 : {
1788 : ml_pipeline_common_elem *src_h;
1789 6152 : ml_pipeline_src_callbacks_s *src_cb = NULL;
1790 6152 : void *pdata = NULL;
1791 :
1792 6152 : src_h = (ml_pipeline_common_elem *) user_data;
1793 6152 : if (!src_h)
1794 0 : return;
1795 :
1796 6152 : src_cb = get_app_src_callback (src_h, &pdata);
1797 6152 : if (src_cb && src_cb->need_data)
1798 6152 : src_cb->need_data (src_h, length, pdata);
1799 : }
1800 :
1801 : /**
1802 : * @brief Internal function for appsrc callback - enough_data.
1803 : */
1804 : static void
1805 0 : _pipe_src_cb_enough_data (GstAppSrc * src, gpointer user_data)
1806 : {
1807 : ml_pipeline_common_elem *src_h;
1808 0 : ml_pipeline_src_callbacks_s *src_cb = NULL;
1809 0 : void *pdata = NULL;
1810 :
1811 0 : src_h = (ml_pipeline_common_elem *) user_data;
1812 0 : if (!src_h)
1813 0 : return;
1814 :
1815 0 : src_cb = get_app_src_callback (src_h, &pdata);
1816 0 : if (src_cb && src_cb->enough_data)
1817 0 : src_cb->enough_data (src_h, pdata);
1818 : }
1819 :
1820 : /**
1821 : * @brief Internal function for appsrc callback - seek_data.
1822 : */
1823 : static gboolean
1824 0 : _pipe_src_cb_seek_data (GstAppSrc * src, guint64 offset, gpointer user_data)
1825 : {
1826 : ml_pipeline_common_elem *src_h;
1827 0 : ml_pipeline_src_callbacks_s *src_cb = NULL;
1828 0 : void *pdata = NULL;
1829 :
1830 0 : src_h = (ml_pipeline_common_elem *) user_data;
1831 0 : if (!src_h)
1832 0 : return TRUE;
1833 :
1834 0 : src_cb = get_app_src_callback (src_h, &pdata);
1835 0 : if (src_cb && src_cb->seek_data)
1836 0 : src_cb->seek_data (src_h, offset, pdata);
1837 :
1838 0 : return TRUE;
1839 : }
1840 :
1841 : /**
1842 : * @brief Register callbacks for src events (more info in nnstreamer.h)
1843 : */
1844 : int
1845 4 : ml_pipeline_src_set_event_cb (ml_pipeline_src_h src_handle,
1846 : ml_pipeline_src_callbacks_s * cb, void *user_data)
1847 : {
1848 4 : GstAppSrcCallbacks appsrc_cb = { 0, };
1849 :
1850 7 : handle_init (src, src_handle);
1851 :
1852 3 : if (cb == NULL) {
1853 1 : ret = ML_ERROR_INVALID_PARAMETER;
1854 1 : goto unlock_return;
1855 : }
1856 :
1857 2 : if (src->callback_info == NULL)
1858 2 : src->callback_info = g_new0 (callback_info_s, 1);
1859 2 : if (src->callback_info == NULL) {
1860 0 : _ml_error_report
1861 : ("Failed to allocate memory of the callback info for %s. Out of memory?",
1862 : elem->name);
1863 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1864 0 : goto unlock_return;
1865 : }
1866 :
1867 2 : src->callback_info->src_cb = *cb;
1868 2 : src->callback_info->src_pdata = user_data;
1869 :
1870 2 : appsrc_cb.need_data = _pipe_src_cb_need_data;
1871 2 : appsrc_cb.enough_data = _pipe_src_cb_enough_data;
1872 2 : appsrc_cb.seek_data = _pipe_src_cb_seek_data;
1873 :
1874 2 : gst_app_src_set_callbacks (GST_APP_SRC (elem->element), &appsrc_cb,
1875 : src_handle, NULL);
1876 :
1877 3 : handle_exit (src_handle);
1878 : }
1879 :
1880 : /**
1881 : * @brief Gets a handle for the tensors metadata of given src node.
1882 : */
1883 : int
1884 6156 : ml_pipeline_src_get_tensors_info (ml_pipeline_src_h h, ml_tensors_info_h * info)
1885 : {
1886 6156 : handle_init (src, h);
1887 :
1888 6156 : if (info == NULL) {
1889 0 : _ml_error_report
1890 : ("The parameter, info (ml_tensors_info_h *), is NULL. It should be a valid pointer to a ml_tensors_info_h instance, which is usually created by ml_tensors_info_create().");
1891 0 : ret = ML_ERROR_INVALID_PARAMETER;
1892 0 : goto unlock_return;
1893 : }
1894 :
1895 6156 : ret = ml_pipeline_src_parse_tensors_info (elem);
1896 :
1897 6156 : if (ret == ML_ERROR_NONE) {
1898 6156 : ret = _ml_tensors_info_create_from_gst (info, &elem->tensors_info);
1899 : } else {
1900 0 : _ml_error_report_continue
1901 : ("ml_pipeline_src_parse_tensors_info () has returned error; it cannot fetch input tensor info (metadata of input stream) for the given ml_pipeline_src_h handle (h). ml_pipeline_src_get_tensors_info () cannot continue.");
1902 : }
1903 :
1904 6156 : handle_exit (h);
1905 : }
1906 :
1907 : /****************************************************
1908 : ** NNStreamer Pipeline Switch/Valve Control **
1909 : ****************************************************/
1910 :
1911 : /**
1912 : * @brief Get a handle to operate a selector (more info in nnstreamer.h)
1913 : */
1914 : int
1915 10 : ml_pipeline_switch_get_handle (ml_pipeline_h pipe, const char *switch_name,
1916 : ml_pipeline_switch_e * type, ml_pipeline_switch_h * h)
1917 : {
1918 : ml_pipeline_element *elem;
1919 10 : ml_pipeline *p = pipe;
1920 : ml_pipeline_common_elem *swtc;
1921 10 : int ret = ML_ERROR_NONE;
1922 :
1923 10 : check_feature_state (ML_FEATURE_INFERENCE);
1924 :
1925 10 : if (h == NULL)
1926 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1927 : "The parameter, h (ml_pipeline_switch_h *), is NULL. It should be a new or to-be-cleared instance of ml_pipeline_switch_h. E.g., ml_pipeline_switch_h h; ml_pipeline_switch_get_handle (..., &h);");
1928 :
1929 : /* init null */
1930 9 : *h = NULL;
1931 :
1932 9 : if (pipe == NULL)
1933 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1934 : "The parameter, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h pipeline instance, which is usually created by ml_pipeline_construct().");
1935 :
1936 8 : if (switch_name == NULL)
1937 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
1938 : "The parameter, switch_name, is NULL. It should be a valid string of the corresponding name of a switch element.");
1939 :
1940 7 : g_mutex_lock (&p->lock);
1941 7 : elem = g_hash_table_lookup (p->namednodes, switch_name);
1942 :
1943 7 : if (elem == NULL) {
1944 1 : _ml_error_report
1945 : ("The parameter, switch_name (%s), is invalid. An element with the name, '%s', cannot be found in the supplied pipeline (pipe)",
1946 : switch_name, switch_name);
1947 1 : ret = ML_ERROR_INVALID_PARAMETER;
1948 1 : goto unlock_return;
1949 : }
1950 :
1951 6 : if (elem->type == ML_PIPELINE_ELEMENT_SWITCH_INPUT) {
1952 4 : if (type)
1953 1 : *type = ML_PIPELINE_SWITCH_INPUT_SELECTOR;
1954 2 : } else if (elem->type == ML_PIPELINE_ELEMENT_SWITCH_OUTPUT) {
1955 1 : if (type)
1956 1 : *type = ML_PIPELINE_SWITCH_OUTPUT_SELECTOR;
1957 : } else {
1958 1 : _ml_error_report
1959 : ("An element with the given name, '%s', is found; however, it is not a 'switch' element. A switch-handle cannot be fetched from a non-switch element. It should be either input-selector or output-selector.",
1960 : switch_name);
1961 1 : ret = ML_ERROR_INVALID_PARAMETER;
1962 1 : goto unlock_return;
1963 : }
1964 :
1965 5 : swtc = *h = g_new0 (ml_pipeline_common_elem, 1);
1966 5 : if (swtc == NULL) {
1967 0 : _ml_error_report
1968 : ("Failed to allocate memory of the switch handle, %s. Out of memory?",
1969 : switch_name);
1970 0 : ret = ML_ERROR_OUT_OF_MEMORY;
1971 0 : goto unlock_return;
1972 : }
1973 :
1974 5 : swtc->pipe = p;
1975 5 : swtc->element = elem;
1976 :
1977 5 : g_mutex_lock (&elem->lock);
1978 :
1979 5 : elem->maxid++;
1980 5 : swtc->id = elem->maxid;
1981 5 : elem->handles = g_list_append (elem->handles, swtc);
1982 :
1983 5 : g_mutex_unlock (&elem->lock);
1984 :
1985 7 : unlock_return:
1986 7 : g_mutex_unlock (&p->lock);
1987 7 : return ret;
1988 : }
1989 :
1990 : /**
1991 : * @brief Close the given switch handle (more info in nnstreamer.h)
1992 : */
1993 : int
1994 5 : ml_pipeline_switch_release_handle (ml_pipeline_switch_h h)
1995 : {
1996 5 : handle_init (swtc, h);
1997 :
1998 5 : elem->handles = g_list_remove (elem->handles, swtc);
1999 5 : free_element_handle (swtc);
2000 :
2001 5 : handle_exit (h);
2002 : }
2003 :
2004 : /**
2005 : * @brief Control the switch (more info in nnstreamer.h)
2006 : */
2007 : int
2008 5 : ml_pipeline_switch_select (ml_pipeline_switch_h h, const char *pad_name)
2009 : {
2010 : GstPad *active_pad, *new_pad;
2011 : gchar *active_name;
2012 :
2013 9 : handle_init (swtc, h);
2014 :
2015 4 : if (pad_name == NULL) {
2016 1 : _ml_error_report
2017 : ("The parameter, pad_name (const char *), is NULL. It should be a valid name of a pad (GSTPAD) in the given switch, h.");
2018 1 : ret = ML_ERROR_INVALID_PARAMETER;
2019 1 : goto unlock_return;
2020 : }
2021 :
2022 3 : g_object_get (G_OBJECT (elem->element), "active-pad", &active_pad, NULL);
2023 3 : active_name = gst_pad_get_name (active_pad);
2024 :
2025 3 : if (g_strcmp0 (pad_name, active_name) == 0) {
2026 0 : _ml_logi ("Switch is called, but there is no effective changes: %s->%s.",
2027 : active_name, pad_name);
2028 0 : g_free (active_name);
2029 0 : gst_object_unref (active_pad);
2030 :
2031 0 : goto unlock_return;
2032 : }
2033 :
2034 3 : g_free (active_name);
2035 3 : gst_object_unref (active_pad);
2036 :
2037 3 : new_pad = gst_element_get_static_pad (elem->element, pad_name);
2038 3 : if (new_pad == NULL) {
2039 : /* Not Found! */
2040 1 : _ml_error_report
2041 : ("Cannot find the pad, [%s], from the switch, [%s]. Please check the pad name. You may use ml_pipeline_switch_pad_list() to fetch the valid pad names.",
2042 : pad_name, elem->name);
2043 1 : ret = ML_ERROR_INVALID_PARAMETER;
2044 1 : goto unlock_return;
2045 : }
2046 :
2047 2 : g_object_set (G_OBJECT (elem->element), "active-pad", new_pad, NULL);
2048 2 : gst_object_unref (new_pad);
2049 :
2050 2 : _ml_logi ("Switched to [%s] successfully at switch [%s].", pad_name,
2051 : elem->name);
2052 :
2053 4 : handle_exit (h);
2054 : }
2055 :
2056 : /**
2057 : * @brief Gets the pad names of a switch.
2058 : */
2059 : int
2060 2 : ml_pipeline_switch_get_pad_list (ml_pipeline_switch_h h, char ***list)
2061 : {
2062 : GstIterator *it;
2063 2 : GValue item = G_VALUE_INIT;
2064 2 : gboolean done = FALSE;
2065 2 : GList *dllist = NULL;
2066 : GstPad *pad;
2067 2 : int counter = 0;
2068 :
2069 4 : handle_init (swtc, h);
2070 :
2071 2 : if (list == NULL) {
2072 0 : _ml_error_report
2073 : ("The parameter, list (char ***), is NULL. It should be a valid pointer to store a list of strings. E.g., char **list; ml_pipeline_switch_get_pad_list (h, &list);");
2074 0 : ret = ML_ERROR_INVALID_PARAMETER;
2075 0 : goto unlock_return;
2076 : }
2077 :
2078 : /* init null */
2079 2 : *list = NULL;
2080 :
2081 2 : if (elem->type == ML_PIPELINE_ELEMENT_SWITCH_INPUT)
2082 1 : it = gst_element_iterate_sink_pads (elem->element);
2083 1 : else if (elem->type == ML_PIPELINE_ELEMENT_SWITCH_OUTPUT)
2084 1 : it = gst_element_iterate_src_pads (elem->element);
2085 : else {
2086 0 : _ml_error_report
2087 : ("The element, [%s], is supposed to be input/output switch, but it is not. Internal data structure is broken.",
2088 : elem->name);
2089 0 : ret = ML_ERROR_STREAMS_PIPE;
2090 0 : goto unlock_return;
2091 : }
2092 :
2093 8 : while (!done) {
2094 6 : switch (gst_iterator_next (it, &item)) {
2095 4 : case GST_ITERATOR_OK:
2096 4 : pad = GST_PAD (g_value_get_object (&item));
2097 4 : dllist = g_list_append (dllist, gst_pad_get_name (pad));
2098 4 : counter++;
2099 4 : g_value_reset (&item);
2100 4 : break;
2101 0 : case GST_ITERATOR_RESYNC:
2102 0 : g_list_free_full (dllist, g_free); /* This frees all strings as well */
2103 0 : dllist = NULL;
2104 0 : counter = 0;
2105 0 : gst_iterator_resync (it);
2106 0 : break;
2107 0 : case GST_ITERATOR_ERROR:
2108 0 : _ml_error_report
2109 : ("Cannot access the list of pad properly of a switch, [%s]. Internal data structure is broken?",
2110 : elem->name);
2111 0 : ret = ML_ERROR_STREAMS_PIPE;
2112 0 : break;
2113 2 : case GST_ITERATOR_DONE:
2114 2 : done = TRUE;
2115 2 : break;
2116 : }
2117 : }
2118 :
2119 2 : gst_iterator_free (it);
2120 :
2121 : /* There has been no error with that "while" loop. */
2122 2 : if (ret == ML_ERROR_NONE) {
2123 2 : int i = 0;
2124 : GList *l;
2125 :
2126 2 : *list = g_malloc0 (sizeof (char *) * (counter + 1));
2127 2 : if (*list == NULL) {
2128 0 : _ml_error_report
2129 : ("Failed to allocate memory for pad list (parameter list). Out of memory?");
2130 0 : ret = ML_ERROR_OUT_OF_MEMORY;
2131 0 : g_list_free_full (dllist, g_free);
2132 0 : goto unlock_return;
2133 : }
2134 :
2135 6 : for (l = dllist; l != NULL; l = l->next) {
2136 4 : (*list)[i] = l->data; /* Allocated by gst_pad_get_name(). Caller has to free it */
2137 4 : i++;
2138 :
2139 4 : if (i > counter) {
2140 0 : g_list_free_full (dllist, g_free); /* This frees all strings as well */
2141 0 : g_free (*list);
2142 0 : *list = NULL;
2143 :
2144 0 : _ml_error_report
2145 : ("Internal data inconsistency. This could be a bug in nnstreamer. Switch [%s].",
2146 : elem->name);
2147 0 : ret = ML_ERROR_STREAMS_PIPE;
2148 0 : goto unlock_return;
2149 : }
2150 : }
2151 : }
2152 2 : g_list_free (dllist); /* This does not free the strings.. fortunately. */
2153 :
2154 2 : handle_exit (h);
2155 : }
2156 :
2157 : /**
2158 : * @brief Get a handle to operate a Valve (more info in nnstreamer.h)
2159 : */
2160 : int
2161 6 : ml_pipeline_valve_get_handle (ml_pipeline_h pipe, const char *valve_name,
2162 : ml_pipeline_valve_h * h)
2163 : {
2164 : ml_pipeline_element *elem;
2165 6 : ml_pipeline *p = pipe;
2166 : ml_pipeline_common_elem *valve;
2167 6 : int ret = ML_ERROR_NONE;
2168 :
2169 6 : check_feature_state (ML_FEATURE_INFERENCE);
2170 :
2171 6 : if (h == NULL)
2172 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2173 : "The parameter, h (ml_pipeline_valve_h), is NULL. It should be a valid pointer of ml_pipeline_valve_h. E.g., ml_pipeline_valve_h h; ml_pipeline_valve_get_handle (..., &h);");
2174 :
2175 : /* init null */
2176 5 : *h = NULL;
2177 :
2178 5 : if (pipe == NULL)
2179 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2180 : "The parameter, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
2181 :
2182 4 : if (valve_name == NULL)
2183 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2184 : "The parameter, valve_name (const char *), is NULL. It should be a valid string of the valve name.");
2185 :
2186 3 : g_mutex_lock (&p->lock);
2187 3 : elem = g_hash_table_lookup (p->namednodes, valve_name);
2188 :
2189 3 : if (elem == NULL) {
2190 1 : _ml_error_report
2191 : ("Cannot find the valve with the given name, '%s', in the pipeline. There is no element in the pipeline with such a name. Please check if you have a value with the appropriate name.",
2192 : valve_name);
2193 1 : ret = ML_ERROR_INVALID_PARAMETER;
2194 1 : goto unlock_return;
2195 : }
2196 :
2197 2 : if (elem->type != ML_PIPELINE_ELEMENT_VALVE) {
2198 1 : _ml_error_report
2199 : ("Cannot find the value with the given name, '%s', in the pipeline. There is an element with such a name; however, the element is not a valve. Please correct the names of element in the pipeline.",
2200 : valve_name);
2201 1 : ret = ML_ERROR_INVALID_PARAMETER;
2202 1 : goto unlock_return;
2203 : }
2204 :
2205 1 : valve = *h = g_new0 (ml_pipeline_common_elem, 1);
2206 1 : if (valve == NULL) {
2207 0 : _ml_error_report
2208 : ("Cannot allocate memory for valve handle of %s. Out of memory?",
2209 : valve_name);
2210 0 : ret = ML_ERROR_OUT_OF_MEMORY;
2211 0 : goto unlock_return;
2212 : }
2213 :
2214 1 : valve->pipe = p;
2215 1 : valve->element = elem;
2216 :
2217 1 : g_mutex_lock (&elem->lock);
2218 :
2219 1 : elem->maxid++;
2220 1 : valve->id = elem->maxid;
2221 1 : elem->handles = g_list_append (elem->handles, valve);
2222 :
2223 1 : g_mutex_unlock (&elem->lock);
2224 :
2225 3 : unlock_return:
2226 3 : g_mutex_unlock (&p->lock);
2227 3 : return ret;
2228 : }
2229 :
2230 : /**
2231 : * @brief Close the given valve handle (more info in nnstreamer.h)
2232 : */
2233 : int
2234 1 : ml_pipeline_valve_release_handle (ml_pipeline_valve_h h)
2235 : {
2236 1 : handle_init (valve, h);
2237 :
2238 1 : elem->handles = g_list_remove (elem->handles, valve);
2239 1 : free_element_handle (valve);
2240 :
2241 1 : handle_exit (h);
2242 : }
2243 :
2244 : /**
2245 : * @brief Control the valve with the given handle (more info in nnstreamer.h)
2246 : */
2247 : int
2248 2 : ml_pipeline_valve_set_open (ml_pipeline_valve_h h, bool open)
2249 : {
2250 2 : gboolean drop = FALSE;
2251 4 : handle_init (valve, h);
2252 :
2253 2 : g_object_get (G_OBJECT (elem->element), "drop", &drop, NULL);
2254 :
2255 2 : if ((open != false) != (drop != FALSE)) {
2256 : /* Nothing to do */
2257 0 : _ml_logi ("Valve is called, but there is no effective changes");
2258 0 : goto unlock_return;
2259 : }
2260 :
2261 2 : drop = (open) ? FALSE : TRUE;
2262 2 : g_object_set (G_OBJECT (elem->element), "drop", drop, NULL);
2263 :
2264 2 : handle_exit (h);
2265 : }
2266 :
2267 : /********************************************************
2268 : ** NNStreamer Element Property Control in Pipeline **
2269 : ********************************************************/
2270 : /**
2271 : * @brief Gets an element handle in NNStreamer pipelines to control its properties.
2272 : */
2273 : int
2274 64 : ml_pipeline_element_get_handle (ml_pipeline_h pipe, const char *element_name,
2275 : ml_pipeline_element_h * elem_h)
2276 : {
2277 64 : int ret = ML_ERROR_NONE;
2278 : ml_pipeline_element *elem;
2279 : ml_pipeline_common_elem *common_elem;
2280 64 : ml_pipeline *p = pipe;
2281 :
2282 : /* Check input parameter */
2283 64 : if (pipe == NULL)
2284 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2285 : "The parameter, pipe (ml_pipeline_h), is NULL. It should be a valid ml_pipeline_h instance, which is usually created by ml_pipeline_construct().");
2286 :
2287 64 : if (element_name == NULL)
2288 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2289 : "The parameter, element_name (const char *), is NULL. It should be a valid string of the element name to be searched.");
2290 :
2291 63 : if (elem_h == NULL)
2292 0 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2293 : "The parameter, elem_h (ml_pipeline_element_h), is NULL. It should be a valid pointer of ml_pipeline_element_h. E.g., ml_pipeline_element_h eh; ml_pipeline_element_get_handle (..., &eh);");
2294 63 : *elem_h = NULL;
2295 :
2296 63 : g_mutex_lock (&p->lock);
2297 :
2298 : /* 1. Search element in lookup table first */
2299 63 : elem = g_hash_table_lookup (p->namednodes, element_name);
2300 63 : if (elem == NULL) {
2301 : /* 2. Search element in pipeline itself */
2302 : GstElement *gst_elem;
2303 :
2304 49 : gst_elem = gst_bin_get_by_name (GST_BIN (p->element), element_name);
2305 49 : if (gst_elem == NULL) {
2306 1 : _ml_error_report
2307 : ("Cannot find the element with the given name, '%s', in the pipeline. There is no element in the pipeline with such a name. Please check if you have an element with the appropriate name.",
2308 : element_name);
2309 1 : ret = ML_ERROR_INVALID_PARAMETER;
2310 1 : goto unlock_return;
2311 : }
2312 :
2313 : /* Caching for next search */
2314 48 : elem = construct_element (gst_elem, pipe, element_name,
2315 : ML_PIPELINE_ELEMENT_COMMON);
2316 48 : if (elem == NULL) {
2317 0 : _ml_error_report
2318 : ("Cannot allocate memory for element handle of %s. Out of memory?",
2319 : element_name);
2320 0 : ret = ML_ERROR_OUT_OF_MEMORY;
2321 0 : goto unlock_return;
2322 : }
2323 48 : g_hash_table_insert (p->namednodes, g_strdup (element_name), elem);
2324 : }
2325 :
2326 : /* Type checking */
2327 62 : if (elem->type == ML_PIPELINE_ELEMENT_UNKNOWN) {
2328 0 : _ml_error_report
2329 : ("There is an element named [%s] in the pipeline, but its type is unknown. It is possible that the app thread has touched ML-API's internal data structure.",
2330 : element_name);
2331 0 : ret = ML_ERROR_INVALID_PARAMETER;
2332 0 : goto unlock_return;
2333 : }
2334 :
2335 62 : common_elem = *elem_h = g_new0 (ml_pipeline_common_elem, 1);
2336 62 : if (common_elem == NULL) {
2337 0 : _ml_error_report
2338 : ("Failed to allocate the internal handler for %s. Out of memory?",
2339 : element_name);
2340 0 : ret = ML_ERROR_OUT_OF_MEMORY;
2341 0 : goto unlock_return;
2342 : }
2343 :
2344 62 : common_elem->pipe = p;
2345 62 : common_elem->element = elem;
2346 :
2347 62 : g_mutex_lock (&elem->lock);
2348 62 : elem->maxid++;
2349 62 : common_elem->id = elem->maxid;
2350 62 : elem->handles = g_list_append (elem->handles, common_elem);
2351 62 : g_mutex_unlock (&elem->lock);
2352 :
2353 63 : unlock_return:
2354 63 : g_mutex_unlock (&p->lock);
2355 63 : return ret;
2356 : }
2357 :
2358 : /**
2359 : * @brief Releases the given element handle.
2360 : */
2361 : int
2362 62 : ml_pipeline_element_release_handle (ml_pipeline_element_h elem_h)
2363 : {
2364 62 : handle_init (common_elem, elem_h);
2365 :
2366 61 : elem->handles = g_list_remove (elem->handles, common_elem);
2367 61 : free_element_handle (common_elem);
2368 :
2369 61 : handle_exit (elem_h);
2370 : }
2371 :
2372 : /**
2373 : * @brief Check property existence and its type.
2374 : */
2375 : static bool
2376 109 : ml_pipeline_element_check_property (GObjectClass * klass,
2377 : const char *property_name, const GType type)
2378 : {
2379 109 : GParamSpec *pspec = NULL;
2380 :
2381 : /* Check property existence */
2382 109 : pspec = g_object_class_find_property (klass, property_name);
2383 109 : if (pspec == NULL) {
2384 16 : _ml_logw ("The property name [%s] does not exist.", property_name);
2385 16 : return FALSE;
2386 : }
2387 :
2388 : /* Compare property's type with given type */
2389 109 : if (!((pspec->value_type == type) ||
2390 33 : (type == G_TYPE_ENUM && G_TYPE_IS_ENUM (pspec->value_type)) ||
2391 22 : (type == G_TYPE_INT64 && pspec->value_type == G_TYPE_LONG) ||
2392 22 : (type == G_TYPE_UINT64 && pspec->value_type == G_TYPE_ULONG) ||
2393 22 : (type == G_TYPE_INT && G_TYPE_IS_ENUM (pspec->value_type)) ||
2394 5 : (type == G_TYPE_UINT && G_TYPE_IS_ENUM (pspec->value_type)) ||
2395 2 : (type == G_TYPE_DOUBLE && pspec->value_type == G_TYPE_FLOAT))) {
2396 16 : _ml_logw ("The type of property name [%s] is '%s'", property_name,
2397 : g_type_name (pspec->value_type));
2398 16 : return FALSE;
2399 : }
2400 77 : return TRUE;
2401 : }
2402 :
2403 : /**
2404 : * @brief Sets the value of given element's property in NNStreamer pipelines.
2405 : */
2406 : static int
2407 83 : ml_pipeline_element_set_property (ml_pipeline_element_h elem_h,
2408 : const char *property_name, gpointer value, GType type)
2409 : {
2410 83 : handle_init (common_elem, elem_h);
2411 :
2412 : /* Check the input parameter */
2413 75 : if (property_name == NULL) {
2414 0 : _ml_error_report
2415 : ("The parameter, property_name (const char *), is NULL. It should be a valid string of property name.");
2416 0 : ret = ML_ERROR_INVALID_PARAMETER;
2417 0 : goto unlock_return;
2418 : }
2419 :
2420 : /* Check property existence & its type */
2421 75 : if (!ml_pipeline_element_check_property (G_OBJECT_GET_CLASS (elem->element),
2422 : property_name, type)) {
2423 16 : _ml_error_report
2424 : ("The property ('%s') of the element, '%s', cannot be checked. It looks like this property does not exist in this element.",
2425 : property_name, elem->name);
2426 16 : ret = ML_ERROR_INVALID_PARAMETER;
2427 16 : goto unlock_return;
2428 : }
2429 :
2430 : /* Set property */
2431 59 : if (type == G_TYPE_DOUBLE || type == G_TYPE_FLOAT) {
2432 7 : g_object_set (G_OBJECT (elem->element), property_name,
2433 : *(double *) value, NULL);
2434 52 : } else if (type == G_TYPE_INT64) {
2435 7 : g_object_set (G_OBJECT (elem->element), property_name,
2436 : *(int64_t *) value, NULL);
2437 45 : } else if (type == G_TYPE_UINT64) {
2438 7 : g_object_set (G_OBJECT (elem->element), property_name,
2439 : *(uint64_t *) value, NULL);
2440 : } else {
2441 38 : g_object_set (G_OBJECT (elem->element), property_name, value, NULL);
2442 : }
2443 :
2444 75 : handle_exit (elem_h);
2445 : }
2446 :
2447 : /**
2448 : * @brief Gets the value of given element's property in NNStreamer pipelines.
2449 : */
2450 : static int
2451 50 : ml_pipeline_element_get_property (ml_pipeline_element_h elem_h,
2452 : const char *property_name, GType type, gpointer pvalue)
2453 : {
2454 50 : handle_init (common_elem, elem_h);
2455 :
2456 : /* Check the input parameter */
2457 42 : if (property_name == NULL) {
2458 0 : _ml_error_report
2459 : ("The parameter, property_name (const char *), is NULL. It should be a valid string of the property name of an element.");
2460 0 : ret = ML_ERROR_INVALID_PARAMETER;
2461 0 : goto unlock_return;
2462 : }
2463 :
2464 42 : if (pvalue == NULL) {
2465 8 : _ml_error_report
2466 : ("The parameter, pvalue (gpointer / a pointer of a value), is NULL. It should be a valid gpointer (a pointer of a value). E.g., char *str; ... ml_pipeline_get_property_string (... &str); ... int32_t val; ... ml_pipeline_get_property_int32 (..., &val);");
2467 8 : ret = ML_ERROR_INVALID_PARAMETER;
2468 8 : goto unlock_return;
2469 : }
2470 :
2471 : /* Check property existence & its type */
2472 34 : if (!ml_pipeline_element_check_property (G_OBJECT_GET_CLASS (elem->element),
2473 : property_name, type)) {
2474 16 : _ml_error_report
2475 : ("Cannot check the property ('%s') or the element ('%s'). Please check if you have the corresponding element in the pipeline.",
2476 : property_name, elem->name);
2477 16 : ret = ML_ERROR_INVALID_PARAMETER;
2478 16 : goto unlock_return;
2479 : }
2480 :
2481 : /* Get property */
2482 18 : g_object_get (G_OBJECT (elem->element), property_name, pvalue, NULL);
2483 :
2484 42 : handle_exit (elem_h);
2485 : }
2486 :
2487 : /**
2488 : * @brief Sets the boolean value of element's property in NNStreamer pipelines.
2489 : */
2490 : int
2491 11 : ml_pipeline_element_set_property_bool (ml_pipeline_element_h elem_h,
2492 : const char *property_name, const int32_t value)
2493 : {
2494 11 : return ml_pipeline_element_set_property (elem_h, property_name,
2495 11 : GINT_TO_POINTER (value), G_TYPE_BOOLEAN);
2496 : }
2497 :
2498 : /**
2499 : * @brief Sets the string value of element's property in NNStreamer pipelines.
2500 : */
2501 : int
2502 6 : ml_pipeline_element_set_property_string (ml_pipeline_element_h elem_h,
2503 : const char *property_name, const char *value)
2504 : {
2505 6 : return ml_pipeline_element_set_property (elem_h, property_name,
2506 : (gpointer) value, G_TYPE_STRING);
2507 : }
2508 :
2509 : /**
2510 : * @brief Sets the integer value of element's property in NNStreamer pipelines.
2511 : */
2512 : int
2513 12 : ml_pipeline_element_set_property_int32 (ml_pipeline_element_h elem_h,
2514 : const char *property_name, const int32_t value)
2515 : {
2516 12 : return ml_pipeline_element_set_property (elem_h, property_name,
2517 12 : GINT_TO_POINTER (value), G_TYPE_INT);
2518 : }
2519 :
2520 : /**
2521 : * @brief Sets the integer 64bit value of element's property in NNStreamer pipelines.
2522 : */
2523 : int
2524 10 : ml_pipeline_element_set_property_int64 (ml_pipeline_element_h elem_h,
2525 : const char *property_name, const int64_t value)
2526 : {
2527 10 : return ml_pipeline_element_set_property (elem_h, property_name,
2528 : (gpointer) (&value), G_TYPE_INT64);
2529 : }
2530 :
2531 : /**
2532 : * @brief Sets the unsigned integer value of element's property in NNStreamer pipelines.
2533 : */
2534 : int
2535 12 : ml_pipeline_element_set_property_uint32 (ml_pipeline_element_h elem_h,
2536 : const char *property_name, const uint32_t value)
2537 : {
2538 12 : return ml_pipeline_element_set_property (elem_h, property_name,
2539 12 : GUINT_TO_POINTER (value), G_TYPE_UINT);
2540 : }
2541 :
2542 : /**
2543 : * @brief Sets the unsigned integer 64bit value of element's property in NNStreamer pipelines.
2544 : */
2545 : int
2546 10 : ml_pipeline_element_set_property_uint64 (ml_pipeline_element_h elem_h,
2547 : const char *property_name, const uint64_t value)
2548 : {
2549 10 : return ml_pipeline_element_set_property (elem_h, property_name,
2550 : (gpointer) (&value), G_TYPE_UINT64);
2551 : }
2552 :
2553 : /**
2554 : * @brief Sets the floating point value of element's property in NNStreamer pipelines.
2555 : */
2556 : int
2557 10 : ml_pipeline_element_set_property_double (ml_pipeline_element_h elem_h,
2558 : const char *property_name, const double value)
2559 : {
2560 10 : return ml_pipeline_element_set_property (elem_h, property_name,
2561 : (gpointer) (&value), G_TYPE_DOUBLE);
2562 : }
2563 :
2564 : /**
2565 : * @brief Sets the enumeration value of element's property in NNStreamer pipelines.
2566 : */
2567 : int
2568 12 : ml_pipeline_element_set_property_enum (ml_pipeline_element_h elem_h,
2569 : const char *property_name, const uint32_t value)
2570 : {
2571 12 : return ml_pipeline_element_set_property (elem_h, property_name,
2572 12 : GUINT_TO_POINTER (value), G_TYPE_ENUM);
2573 : }
2574 :
2575 : /**
2576 : * @brief Gets the boolean value of element's property in NNStreamer pipelines.
2577 : */
2578 : int
2579 6 : ml_pipeline_element_get_property_bool (ml_pipeline_element_h elem_h,
2580 : const char *property_name, int32_t * value)
2581 : {
2582 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2583 : G_TYPE_BOOLEAN, (gpointer) value);
2584 : }
2585 :
2586 : /**
2587 : * @brief Gets the string value of element's property in NNStreamer pipelines.
2588 : */
2589 : int
2590 6 : ml_pipeline_element_get_property_string (ml_pipeline_element_h elem_h,
2591 : const char *property_name, char **value)
2592 : {
2593 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2594 : G_TYPE_STRING, (gpointer) value);
2595 : }
2596 :
2597 : /**
2598 : * @brief Gets the integer value of element's property in NNStreamer pipelines.
2599 : */
2600 : int
2601 7 : ml_pipeline_element_get_property_int32 (ml_pipeline_element_h elem_h,
2602 : const char *property_name, int32_t * value)
2603 : {
2604 7 : return ml_pipeline_element_get_property (elem_h, property_name,
2605 : G_TYPE_INT, (gpointer) value);
2606 : }
2607 :
2608 : /**
2609 : * @brief Gets the integer 64bit value of element's property in NNStreamer pipelines.
2610 : */
2611 : int
2612 6 : ml_pipeline_element_get_property_int64 (ml_pipeline_element_h elem_h,
2613 : const char *property_name, int64_t * value)
2614 : {
2615 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2616 : G_TYPE_INT64, (gpointer) value);
2617 : }
2618 :
2619 : /**
2620 : * @brief Gets the unsigned integer value of element's property in NNStreamer pipelines.
2621 : */
2622 : int
2623 7 : ml_pipeline_element_get_property_uint32 (ml_pipeline_element_h elem_h,
2624 : const char *property_name, uint32_t * value)
2625 : {
2626 7 : return ml_pipeline_element_get_property (elem_h, property_name,
2627 : G_TYPE_UINT, (gpointer) value);
2628 : }
2629 :
2630 : /**
2631 : * @brief Gets the unsigned integer 64bit value of element's property in NNStreamer pipelines.
2632 : */
2633 : int
2634 6 : ml_pipeline_element_get_property_uint64 (ml_pipeline_element_h elem_h,
2635 : const char *property_name, uint64_t * value)
2636 : {
2637 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2638 : G_TYPE_UINT64, (gpointer) value);
2639 : }
2640 :
2641 : /**
2642 : * @brief Gets the floating point value of element's property in NNStreamer pipelines.
2643 : */
2644 : int
2645 6 : ml_pipeline_element_get_property_double (ml_pipeline_element_h elem_h,
2646 : const char *property_name, double *value)
2647 : {
2648 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2649 : G_TYPE_DOUBLE, (gpointer) value);
2650 : }
2651 :
2652 : /**
2653 : * @brief Gets the enumeration value of element's property in NNStreamer pipelines.
2654 : */
2655 : int
2656 6 : ml_pipeline_element_get_property_enum (ml_pipeline_element_h elem_h,
2657 : const char *property_name, uint32_t * value)
2658 : {
2659 6 : return ml_pipeline_element_get_property (elem_h, property_name,
2660 : G_TYPE_ENUM, (gpointer) value);
2661 : }
2662 :
2663 : /**
2664 : * @brief Gets the element of pipeline itself (GstElement).
2665 : */
2666 : GstElement *
2667 0 : _ml_pipeline_get_gst_pipeline (ml_pipeline_h pipe)
2668 : {
2669 0 : ml_pipeline *p = (ml_pipeline *) pipe;
2670 0 : GstElement *element = NULL;
2671 :
2672 0 : if (p) {
2673 0 : g_mutex_lock (&p->lock);
2674 :
2675 0 : element = p->element;
2676 0 : if (element)
2677 0 : gst_object_ref (element);
2678 :
2679 0 : g_mutex_unlock (&p->lock);
2680 : }
2681 :
2682 0 : return element;
2683 : }
2684 :
2685 : /**
2686 : * @brief Gets the element in pipeline (GstElement).
2687 : */
2688 : GstElement *
2689 0 : _ml_pipeline_get_gst_element (ml_pipeline_element_h handle)
2690 : {
2691 0 : ml_pipeline_common_elem *e = (ml_pipeline_common_elem *) handle;
2692 0 : GstElement *element = NULL;
2693 :
2694 0 : if (e && e->element) {
2695 0 : ml_pipeline_element *elem = e->element;
2696 :
2697 0 : g_mutex_lock (&elem->lock);
2698 :
2699 0 : element = elem->element;
2700 0 : if (element)
2701 0 : gst_object_ref (element);
2702 :
2703 0 : g_mutex_unlock (&elem->lock);
2704 : }
2705 :
2706 0 : return element;
2707 : }
2708 :
2709 : /**
2710 : * @brief Increases ref count of custom-easy filter.
2711 : */
2712 : static void
2713 3 : ml_pipeline_custom_filter_ref (ml_custom_easy_filter_h custom)
2714 : {
2715 3 : ml_custom_filter_s *c = (ml_custom_filter_s *) custom;
2716 :
2717 3 : if (c) {
2718 3 : g_mutex_lock (&c->lock);
2719 3 : c->ref_count++;
2720 3 : g_mutex_unlock (&c->lock);
2721 : }
2722 3 : }
2723 :
2724 : /**
2725 : * @brief Decreases ref count of custom-easy filter.
2726 : */
2727 : static void
2728 3 : ml_pipeline_custom_filter_unref (ml_custom_easy_filter_h custom)
2729 : {
2730 3 : ml_custom_filter_s *c = (ml_custom_filter_s *) custom;
2731 :
2732 3 : if (!c)
2733 0 : return;
2734 :
2735 3 : g_mutex_lock (&c->lock);
2736 3 : if (c->ref_count > 0)
2737 3 : c->ref_count--;
2738 3 : g_mutex_unlock (&c->lock);
2739 : }
2740 :
2741 : /**
2742 : * @brief Releases custom filter handle.
2743 : */
2744 : static void
2745 4 : ml_pipeline_custom_free_handle (ml_custom_filter_s * custom)
2746 : {
2747 4 : if (custom) {
2748 4 : g_mutex_lock (&custom->lock);
2749 :
2750 4 : g_free (custom->name);
2751 4 : ml_tensors_info_destroy (custom->in_info);
2752 4 : ml_tensors_info_destroy (custom->out_info);
2753 :
2754 4 : g_mutex_unlock (&custom->lock);
2755 4 : g_mutex_clear (&custom->lock);
2756 :
2757 4 : g_free (custom);
2758 : }
2759 4 : }
2760 :
2761 : /**
2762 : * @brief Invoke callback for custom-easy filter.
2763 : */
2764 : static int
2765 5 : ml_pipeline_custom_invoke (void *data, const GstTensorFilterProperties * prop,
2766 : const GstTensorMemory * in, GstTensorMemory * out)
2767 : {
2768 : int status;
2769 : ml_custom_filter_s *c;
2770 : ml_tensors_data_h in_data, out_data;
2771 : ml_tensors_data_s *_data;
2772 : guint i;
2773 :
2774 5 : in_data = out_data = NULL;
2775 5 : c = (ml_custom_filter_s *) data;
2776 :
2777 : /* internal error? */
2778 5 : if (!c || !c->cb)
2779 5 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2780 : "Internal error of callback function, ml_pipeline_custom_invoke. Its internal data structure is broken.");
2781 :
2782 5 : g_mutex_lock (&c->lock);
2783 :
2784 : /* prepare invoke */
2785 5 : status = _ml_tensors_data_create_no_alloc (c->in_info, &in_data);
2786 5 : if (status != ML_ERROR_NONE) {
2787 0 : _ml_error_report_continue ("_ml_tensors_data_create_no_alloc has failed.");
2788 0 : goto done;
2789 : }
2790 :
2791 5 : _data = (ml_tensors_data_s *) in_data;
2792 10 : for (i = 0; i < _data->num_tensors; i++)
2793 5 : _data->tensors[i].data = in[i].data;
2794 :
2795 5 : status = _ml_tensors_data_create_no_alloc (c->out_info, &out_data);
2796 5 : if (status != ML_ERROR_NONE) {
2797 0 : _ml_error_report_continue ("_ml_tensors_data_create_no_alloc has failed.");
2798 0 : goto done;
2799 : }
2800 :
2801 5 : _data = (ml_tensors_data_s *) out_data;
2802 10 : for (i = 0; i < _data->num_tensors; i++)
2803 5 : _data->tensors[i].data = out[i].data;
2804 :
2805 : /* call invoke callback */
2806 5 : status = c->cb (in_data, out_data, c->pdata);
2807 :
2808 5 : done:
2809 5 : g_mutex_unlock (&c->lock);
2810 : /* NOTE: DO NOT free tensor data */
2811 5 : _ml_tensors_data_destroy_internal (in_data, FALSE);
2812 5 : _ml_tensors_data_destroy_internal (out_data, FALSE);
2813 :
2814 5 : return status;
2815 : }
2816 :
2817 : /**
2818 : * @brief Registers a custom filter.
2819 : */
2820 : int
2821 11 : ml_pipeline_custom_easy_filter_register (const char *name,
2822 : const ml_tensors_info_h in, const ml_tensors_info_h out,
2823 : ml_custom_easy_invoke_cb cb, void *user_data,
2824 : ml_custom_easy_filter_h * custom)
2825 : {
2826 11 : int status = ML_ERROR_NONE;
2827 : ml_custom_filter_s *c;
2828 : GstTensorsInfo in_info, out_info;
2829 :
2830 22 : check_feature_state (ML_FEATURE_INFERENCE);
2831 :
2832 11 : if (!name)
2833 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2834 : "The parameter, name (const char *), is NULL. It should be a valid string of the filter name.");
2835 10 : if (!cb)
2836 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2837 : "The parameter, cb (ml_custom_easy_invoke_cb), is NULL. It should be a valid call-back struct containing function pointer and its related data.");
2838 9 : if (!custom)
2839 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2840 : "The parameter, custom (ml_custom_easy_filter_h *), is NULL. It should be a valid pointer of ml_custom_easy_filter. E.g., ml_custom_easy_filter_h custom; ml_pipeline_custom_easy_filter_register (..., &custom);");
2841 :
2842 : /* init null */
2843 8 : *custom = NULL;
2844 :
2845 8 : if (!ml_tensors_info_is_valid (in))
2846 2 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2847 : "The parameter, in (const ml_tensors_info_h), is not valid. ml_tensors_info_is_valid(in) has returned FALSE. Please check if its cloned/fetched from a valid object or if you have configured it properly.");
2848 6 : if (!ml_tensors_info_is_valid (out))
2849 2 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2850 : "The parameter, out (const ml_tensors_info_h), is not valid. ml_tensors_info_is_valid(in) has returned FALSE. Please check if its cloned/fetched from a valid object or if you have configured it properly.");
2851 :
2852 : /* create and init custom handle */
2853 4 : if ((c = g_new0 (ml_custom_filter_s, 1)) == NULL)
2854 0 : _ml_error_report_return (ML_ERROR_OUT_OF_MEMORY,
2855 : "Cannot allocate memory. Out of memory?");
2856 :
2857 4 : g_mutex_init (&c->lock);
2858 :
2859 : /** no need to acquire c->lock as its created locally */
2860 4 : c->name = g_strdup (name);
2861 4 : c->ref_count = 0;
2862 4 : c->cb = cb;
2863 4 : c->pdata = user_data;
2864 4 : ml_tensors_info_create_extended (&c->in_info);
2865 4 : ml_tensors_info_create_extended (&c->out_info);
2866 :
2867 4 : status = ml_tensors_info_clone (c->in_info, in);
2868 4 : if (status != ML_ERROR_NONE) {
2869 0 : _ml_error_report_continue
2870 : ("ml_tensors_info_clone has failed with %d. Cannot fetch input tensor-info (metadata).",
2871 : status);
2872 0 : goto exit;
2873 : }
2874 :
2875 4 : status = ml_tensors_info_clone (c->out_info, out);
2876 4 : if (status != ML_ERROR_NONE) {
2877 0 : _ml_error_report_continue
2878 : ("ml_tensors_info_clone has filed with %d. Cannot fetch output tensor-info (metadata).",
2879 : status);
2880 0 : goto exit;
2881 : }
2882 :
2883 : /* register custom filter */
2884 4 : _ml_tensors_info_copy_from_ml (&in_info, c->in_info);
2885 4 : _ml_tensors_info_copy_from_ml (&out_info, c->out_info);
2886 :
2887 4 : status = NNS_custom_easy_register (name, ml_pipeline_custom_invoke, c,
2888 : &in_info, &out_info);
2889 4 : if (status != 0) {
2890 1 : char buf[255] = { 0 };
2891 1 : if (status == -EINVAL) {
2892 1 : status = ML_ERROR_INVALID_PARAMETER;
2893 1 : strncpy (buf, "invalid parameters are given.", 254);
2894 0 : } else if (status == -ENOMEM) {
2895 0 : status = ML_ERROR_OUT_OF_MEMORY;
2896 0 : strncpy (buf, "out of memory. cannot allocate.", 254);
2897 : } else {
2898 0 : status = ML_ERROR_UNKNOWN;
2899 0 : strncpy (buf, "unknown error.", 254);
2900 : }
2901 1 : _ml_error_report
2902 : ("Failed to register custom filter %s with NNStreamer API, NNS_custom_easy_register(). It has returned %d, which means '%s'.",
2903 : name, status, buf);
2904 : }
2905 :
2906 3 : exit:
2907 4 : if (status == ML_ERROR_NONE) {
2908 3 : pipe_custom_add_data (PIPE_CUSTOM_TYPE_FILTER, name, c);
2909 3 : *custom = c;
2910 : } else {
2911 1 : ml_pipeline_custom_free_handle (c);
2912 : }
2913 :
2914 4 : return status;
2915 : }
2916 :
2917 : /**
2918 : * @brief Unregisters the custom filter.
2919 : */
2920 : int
2921 6 : ml_pipeline_custom_easy_filter_unregister (ml_custom_easy_filter_h custom)
2922 : {
2923 : ml_custom_filter_s *c;
2924 6 : int status = ML_ERROR_NONE;
2925 :
2926 6 : check_feature_state (ML_FEATURE_INFERENCE);
2927 :
2928 6 : if (!custom)
2929 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
2930 : "The parameter, custom (ml_custom_easy_filter_h), is NULL. It should be a valid ml_custom_easy_filter_h instance, usually created by ml_pipeline_custom_easy_filter_register().");
2931 :
2932 5 : c = (ml_custom_filter_s *) custom;
2933 5 : g_mutex_lock (&c->lock);
2934 :
2935 5 : if (c->ref_count > 0) {
2936 2 : _ml_error_report
2937 : ("Failed to unregister custom filter %s, it is used in the pipeline. Its reference counter value is %u.",
2938 : c->name, c->ref_count);
2939 2 : status = ML_ERROR_INVALID_PARAMETER;
2940 2 : goto done;
2941 : }
2942 :
2943 3 : status = NNS_custom_easy_unregister (c->name);
2944 3 : if (status != 0) {
2945 0 : _ml_error_report
2946 : ("Failed to unregister custom filter %s. It is possible that this is already unregistered or not registered.",
2947 : c->name);
2948 0 : status = ML_ERROR_INVALID_PARAMETER;
2949 0 : goto done;
2950 : }
2951 :
2952 3 : done:
2953 5 : g_mutex_unlock (&c->lock);
2954 :
2955 5 : if (status == ML_ERROR_NONE) {
2956 3 : pipe_custom_remove_data (PIPE_CUSTOM_TYPE_FILTER, c->name);
2957 3 : ml_pipeline_custom_free_handle (c);
2958 : }
2959 :
2960 5 : return status;
2961 : }
2962 :
2963 : /**
2964 : * @brief Increases ref count of tensor_if custom condition.
2965 : */
2966 : static void
2967 3 : ml_pipeline_if_custom_ref (ml_pipeline_if_h custom)
2968 : {
2969 3 : ml_if_custom_s *c = (ml_if_custom_s *) custom;
2970 :
2971 3 : if (c) {
2972 3 : g_mutex_lock (&c->lock);
2973 3 : c->ref_count++;
2974 3 : g_mutex_unlock (&c->lock);
2975 : }
2976 3 : }
2977 :
2978 : /**
2979 : * @brief Decreases ref count of tensor_if custom condition.
2980 : */
2981 : static void
2982 3 : ml_pipeline_if_custom_unref (ml_pipeline_if_h custom)
2983 : {
2984 3 : ml_if_custom_s *c = (ml_if_custom_s *) custom;
2985 :
2986 3 : if (c) {
2987 3 : g_mutex_lock (&c->lock);
2988 3 : if (c->ref_count > 0)
2989 3 : c->ref_count--;
2990 3 : g_mutex_unlock (&c->lock);
2991 : }
2992 3 : }
2993 :
2994 : /**
2995 : * @brief Callback for tensor_if custom condition.
2996 : */
2997 : static gboolean
2998 10 : ml_pipeline_if_custom (const GstTensorsInfo * info,
2999 : const GstTensorMemory * input, void *data, gboolean * result)
3000 : {
3001 10 : int status = 0;
3002 : guint i;
3003 : ml_if_custom_s *c;
3004 10 : ml_tensors_data_h in_data = NULL;
3005 : ml_tensors_data_s *_data;
3006 10 : ml_tensors_info_h ml_info = NULL;
3007 10 : gboolean ret = FALSE;
3008 :
3009 10 : c = (ml_if_custom_s *) data;
3010 :
3011 : /* internal error? */
3012 10 : if (!c || !c->cb)
3013 10 : _ml_error_report_return (FALSE,
3014 : "Internal error: the parameter, data, is not valid. App thread might have touched internal data structure.");
3015 :
3016 10 : status = _ml_tensors_info_create_from_gst (&ml_info, info);
3017 10 : if (status != ML_ERROR_NONE)
3018 0 : _ml_error_report_return_continue (FALSE,
3019 : "Cannot create tensors-info from the parameter, info (const GstTensorsInfo). _ml_tensors_info_create_from_gst has returned %d.",
3020 : status);
3021 10 : status = _ml_tensors_data_create_no_alloc (ml_info, &in_data);
3022 10 : if (status != ML_ERROR_NONE) {
3023 0 : _ml_error_report_continue
3024 : ("Cannot create data entry from the given metadata, info (const GstTensorMemory, although we could create tensor-info from info. _ml_tensors_data_create_no_alloc() has returned %d.",
3025 : status);
3026 0 : goto done;
3027 : }
3028 :
3029 10 : _data = (ml_tensors_data_s *) in_data;
3030 20 : for (i = 0; i < _data->num_tensors; i++)
3031 10 : _data->tensors[i].data = input[i].data;
3032 :
3033 : /* call invoke callback */
3034 10 : g_mutex_lock (&c->lock);
3035 10 : status = c->cb (in_data, ml_info, result, c->pdata);
3036 10 : g_mutex_unlock (&c->lock);
3037 :
3038 10 : ret = (status == ML_ERROR_NONE);
3039 10 : if (!ret)
3040 0 : _ml_error_report
3041 : ("The callback function of if-statement has returned error: %d.",
3042 : status);
3043 :
3044 10 : done:
3045 10 : ml_tensors_info_destroy (ml_info);
3046 10 : _ml_tensors_data_destroy_internal (in_data, FALSE);
3047 :
3048 10 : return ret;
3049 : }
3050 :
3051 : /**
3052 : * @brief Releases tensor_if custom condition.
3053 : */
3054 : static void
3055 4 : ml_pipeline_if_custom_free (ml_if_custom_s * custom)
3056 : {
3057 4 : if (custom) {
3058 4 : g_mutex_lock (&custom->lock);
3059 :
3060 4 : g_free (custom->name);
3061 :
3062 4 : g_mutex_unlock (&custom->lock);
3063 4 : g_mutex_clear (&custom->lock);
3064 :
3065 4 : g_free (custom);
3066 : }
3067 4 : }
3068 :
3069 : /**
3070 : * @brief Registers the tensor_if custom callback.
3071 : */
3072 : int
3073 7 : ml_pipeline_tensor_if_custom_register (const char *name,
3074 : ml_pipeline_if_custom_cb cb, void *user_data, ml_pipeline_if_h * if_custom)
3075 : {
3076 7 : int status = ML_ERROR_NONE;
3077 : ml_if_custom_s *c;
3078 :
3079 7 : check_feature_state (ML_FEATURE_INFERENCE);
3080 :
3081 7 : if (!name)
3082 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
3083 : "The parameter, name (const char *), is NULL. It should be a valid string of the tensor_if element in your pipeline.");
3084 6 : if (!cb)
3085 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
3086 : "The parameter, cb (ml_pipeline_if_custom_cb callback function pointer), is NULL. It should be a valid function pointer that determines if the 'if' statement is TRUE or FALSE from the given tensor data frame.");
3087 5 : if (!if_custom)
3088 1 : _ml_error_report_return (ML_ERROR_INVALID_PARAMETER,
3089 : "The parameter, if_custom (ml_pipeline_if_h *), is NULL. It should be a valid pointer to the pipeline-if handle instance. E.g., ml_pipeline_if_h h; ml_pipeline_tensor_if_custom_register (..., &h);");
3090 :
3091 : /* init null */
3092 4 : *if_custom = NULL;
3093 :
3094 : /* create and init custom handle */
3095 4 : if ((c = g_try_new0 (ml_if_custom_s, 1)) == NULL)
3096 0 : _ml_error_report_return (ML_ERROR_OUT_OF_MEMORY,
3097 : "Cannot allocate memory. Out of memory?");
3098 :
3099 4 : g_mutex_init (&c->lock);
3100 :
3101 4 : g_mutex_lock (&c->lock);
3102 4 : c->name = g_strdup (name);
3103 4 : c->ref_count = 0;
3104 4 : c->cb = cb;
3105 4 : c->pdata = user_data;
3106 :
3107 4 : status = nnstreamer_if_custom_register (name, ml_pipeline_if_custom, c);
3108 4 : if (status != 0) {
3109 1 : if (status == -ENOMEM) {
3110 0 : _ml_error_report
3111 : ("Failed to register tensor_if custom condition %s because nnstreamer_if_custom_register has failed to allocate memory. Out of memory?",
3112 : name);
3113 0 : status = ML_ERROR_OUT_OF_MEMORY;
3114 1 : } else if (status == -EINVAL) {
3115 1 : _ml_error_report
3116 : ("Failed to register tensor_if custom condition %s because nnstreamer_if_custom_register has reported that an invalid parameter is given to the API call. Please check if the given name is 0-length or duplicated (already registered), memory is full, or the name is not allowed ('any', 'auto' are not allowed).",
3117 : name);
3118 1 : status = ML_ERROR_INVALID_PARAMETER;
3119 : } else {
3120 0 : _ml_error_report
3121 : ("Failed to register tensor_if custom condition %s because nnstreamer_if_custom_register has returned unknown error.",
3122 : name);
3123 0 : status = ML_ERROR_UNKNOWN;
3124 : }
3125 : }
3126 4 : g_mutex_unlock (&c->lock);
3127 :
3128 4 : if (status == ML_ERROR_NONE) {
3129 3 : pipe_custom_add_data (PIPE_CUSTOM_TYPE_IF, name, c);
3130 3 : *if_custom = c;
3131 : } else {
3132 1 : ml_pipeline_if_custom_free (c);
3133 : }
3134 :
3135 4 : return status;
3136 : }
3137 :
3138 : /**
3139 : * @brief Unregisters the tensor_if custom callback.
3140 : */
3141 : int
3142 6 : ml_pipeline_tensor_if_custom_unregister (ml_pipeline_if_h if_custom)
3143 : {
3144 : ml_if_custom_s *c;
3145 6 : int status = ML_ERROR_NONE;
3146 :
3147 6 : check_feature_state (ML_FEATURE_INFERENCE);
3148 :
3149 6 : if (!if_custom)
3150 1 : return ML_ERROR_INVALID_PARAMETER;
3151 :
3152 5 : c = (ml_if_custom_s *) if_custom;
3153 5 : g_mutex_lock (&c->lock);
3154 :
3155 5 : if (c->ref_count > 0) {
3156 2 : _ml_error_report
3157 : ("Failed to unregister custom condition %s, it is used in the pipeline.",
3158 : c->name);
3159 2 : status = ML_ERROR_INVALID_PARAMETER;
3160 2 : goto done;
3161 : }
3162 :
3163 3 : status = nnstreamer_if_custom_unregister (c->name);
3164 3 : if (status != 0) {
3165 0 : if (status == -EINVAL)
3166 0 : _ml_error_report
3167 : ("Failed to unregister tensor_if custom condition %s. It appears that it is already unregistered or not yet registered.",
3168 : c->name);
3169 : else
3170 0 : _ml_error_report
3171 : ("Failed to unregister tensor_if custom condition %s with unknown reason. Internal error?",
3172 : c->name);
3173 0 : status = ML_ERROR_STREAMS_PIPE;
3174 0 : goto done;
3175 : }
3176 :
3177 3 : done:
3178 5 : g_mutex_unlock (&c->lock);
3179 :
3180 5 : if (status == ML_ERROR_NONE) {
3181 3 : pipe_custom_remove_data (PIPE_CUSTOM_TYPE_IF, c->name);
3182 3 : ml_pipeline_if_custom_free (c);
3183 : }
3184 :
3185 5 : return status;
3186 : }
|
