Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer / NNStreamer tensor_converter subplugin, "python3"
4 : * Copyright (C) 2021 Gichan Jang <gichan2.jang@samsung.com>
5 : */
6 : /**
7 : * @file tensor_converter_python3.c
8 : * @date May 03 2021
9 : * @brief NNStreamer tensor-converter subplugin, "python3",
10 : * which converts to tensors using python.
11 : * @see https://github.com/nnstreamer/nnstreamer
12 : * @author Gichan Jang <gichan2.jang@samsung.com>
13 : * @bug python converter with Python3.9.10 is stuck during Py_Finalize().
14 : */
15 :
16 : #include <nnstreamer_plugin_api.h>
17 : #include <nnstreamer_plugin_api_converter.h>
18 : #include <nnstreamer_util.h>
19 : #include "nnstreamer_python3_helper.h"
20 :
21 : #ifdef __cplusplus
22 : extern "C" {
23 : #endif /* __cplusplus */
24 : void init_converter_py (void) __attribute__ ((constructor));
25 : void fini_converter_py (void) __attribute__ ((destructor));
26 : #ifdef __cplusplus
27 : }
28 : #endif /* __cplusplus */
29 :
30 : /**
31 : * @brief Python embedding core structure
32 : */
33 : class PYConverterCore
34 : {
35 : public:
36 : /**
37 : * member functions.
38 : */
39 : PYConverterCore (const char *_script_path);
40 : ~PYConverterCore ();
41 :
42 : int init ();
43 : const char *getScriptPath ();
44 : GstBuffer *convert (GstBuffer *in_buf, GstTensorsConfig *config);
45 :
46 : /** @brief Lock python-related actions */
47 26 : void Py_LOCK ()
48 : {
49 26 : g_mutex_lock (&py_mutex);
50 26 : }
51 : /** @brief Unlock python-related actions */
52 26 : void Py_UNLOCK ()
53 : {
54 26 : g_mutex_unlock (&py_mutex);
55 26 : }
56 :
57 : private:
58 : std::string module_name;
59 : const std::string script_path;
60 : PyObject *shape_cls;
61 : PyObject *core_obj;
62 : void *handle; /**< returned handle by dlopen() */
63 : GMutex py_mutex;
64 : };
65 :
66 : /**
67 : * @brief PYConverterCore creator
68 : * @param _script_path : the logical path to '{script_name}.py' file
69 : * @note the script of _script_path will be loaded simultaneously
70 : * @return Nothing
71 : */
72 23 : PYConverterCore::PYConverterCore (const char *_script_path)
73 46 : : script_path (_script_path)
74 : {
75 23 : if (openPythonLib (&handle))
76 0 : throw std::runtime_error (dlerror ());
77 :
78 23 : _import_array (); /** for numpy */
79 :
80 : /**
81 : * Parse script path to get module name
82 : * The module name should drop its extension (i.e., .py)
83 : */
84 23 : module_name = script_path;
85 23 : const size_t last_idx = module_name.find_last_of ("/");
86 :
87 23 : if (last_idx != std::string::npos)
88 23 : module_name.erase (0, last_idx + 1);
89 :
90 23 : const size_t ext_idx = module_name.rfind ('.');
91 23 : if (ext_idx != std::string::npos)
92 23 : module_name.erase (ext_idx);
93 :
94 23 : addToSysPath (script_path.substr (0, last_idx).c_str ());
95 :
96 23 : core_obj = NULL;
97 23 : shape_cls = NULL;
98 :
99 23 : g_mutex_init (&py_mutex);
100 23 : }
101 :
102 : /**
103 : * @brief PYConverterCore Destructor
104 : * @return Nothing
105 : */
106 46 : PYConverterCore::~PYConverterCore ()
107 : {
108 23 : Py_SAFEDECREF (core_obj);
109 23 : Py_SAFEDECREF (shape_cls);
110 23 : PyErr_Clear ();
111 :
112 23 : dlclose (handle);
113 23 : g_mutex_clear (&py_mutex);
114 23 : }
115 :
116 : /**
117 : * @brief convert any media stream to tensor
118 : */
119 : GstBuffer *
120 26 : PYConverterCore::convert (GstBuffer *in_buf, GstTensorsConfig *config)
121 : {
122 : GstMemory *in_mem[NNS_TENSOR_SIZE_LIMIT], *out_mem;
123 : GstMapInfo in_info[NNS_TENSOR_SIZE_LIMIT];
124 26 : GstBuffer *out_buf = NULL;
125 : PyObject *tensors_info, *output, *pyValue, *param;
126 : gint rate_n, rate_d;
127 : guint i, num;
128 : gsize mem_size;
129 : gpointer mem_data;
130 :
131 26 : if (nullptr == in_buf)
132 0 : throw std::invalid_argument ("Null pointers are given to PYConverterCore::convert().\n");
133 26 : num = gst_tensor_buffer_get_count (in_buf);
134 26 : tensors_info = output = pyValue = param = nullptr;
135 :
136 26 : Py_LOCK ();
137 26 : param = PyList_New (num);
138 :
139 52 : for (i = 0; i < num; i++) {
140 26 : in_mem[i] = gst_tensor_buffer_get_nth_memory (in_buf, i);
141 :
142 26 : if (!gst_memory_map (in_mem[i], &in_info[i], GST_MAP_READ)) {
143 0 : Py_ERRMSG ("Cannot map input memory / tensor_converter::custom-script");
144 0 : num = i;
145 0 : goto done;
146 : }
147 :
148 26 : npy_intp input_dims[] = { (npy_intp) (in_info[i].size) };
149 : PyObject *input_array
150 26 : = PyArray_SimpleNewFromData (1, input_dims, NPY_UINT8, in_info[i].data);
151 :
152 26 : PyList_SetItem (param, i, input_array);
153 : }
154 :
155 26 : if (!PyObject_HasAttrString (core_obj, (char *) "convert")) {
156 0 : Py_ERRMSG ("Cannot find 'convert'");
157 0 : goto done;
158 : }
159 :
160 26 : pyValue = PyObject_CallMethod (core_obj, "convert", "(O)", param);
161 :
162 26 : if (!PyArg_ParseTuple (pyValue, "OOii", &tensors_info, &output, &rate_n, &rate_d)) {
163 0 : Py_ERRMSG ("Failed to parse converting result");
164 0 : goto done;
165 : }
166 :
167 26 : if (parseTensorsInfo (tensors_info, &config->info) != 0) {
168 0 : Py_ERRMSG ("Failed to parse tensors info");
169 0 : goto done;
170 : }
171 26 : config->rate_n = rate_n;
172 26 : config->rate_d = rate_d;
173 :
174 26 : if (output) {
175 : GstTensorInfo *_info;
176 26 : Py_ssize_t num_tensors = PyList_Size (output);
177 :
178 26 : if (num_tensors < 0 || num_tensors > NNS_TENSOR_SIZE_LIMIT) {
179 0 : Py_ERRMSG ("Fail to get output from 'convert', invalid output size.");
180 0 : goto done;
181 : }
182 :
183 26 : out_buf = gst_buffer_new ();
184 74 : for (i = 0; i < (guint) num_tensors; i++) {
185 : PyArrayObject *output_array
186 48 : = (PyArrayObject *) PyList_GetItem (output, (Py_ssize_t) i);
187 :
188 48 : _info = gst_tensors_info_get_nth_info (&config->info, i);
189 48 : mem_size = PyArray_SIZE (output_array);
190 48 : mem_data = _g_memdup ((guint8 *) PyArray_DATA (output_array), mem_size);
191 :
192 48 : out_mem = gst_memory_new_wrapped (
193 : (GstMemoryFlags) 0, mem_data, mem_size, 0, mem_size, mem_data, g_free);
194 :
195 48 : gst_tensor_buffer_append_memory (out_buf, out_mem, _info);
196 : }
197 : } else {
198 0 : Py_ERRMSG ("Fail to get output from 'convert'");
199 : }
200 :
201 26 : done:
202 52 : for (i = 0; i < num; i++) {
203 26 : gst_memory_unmap (in_mem[i], &in_info[i]);
204 26 : gst_memory_unref (in_mem[i]);
205 : }
206 :
207 26 : Py_SAFEDECREF (param);
208 26 : Py_SAFEDECREF (pyValue);
209 :
210 26 : Py_UNLOCK ();
211 26 : return out_buf;
212 : }
213 :
214 : /**
215 : * @brief initialize the object with python script
216 : * @return 0 if OK. non-zero if error.
217 : */
218 : int
219 23 : PYConverterCore::init ()
220 : {
221 : /** Find nnstreamer_api module */
222 23 : PyObject *api_module = PyImport_ImportModule ("nnstreamer_python");
223 23 : if (api_module == NULL)
224 0 : return -EINVAL;
225 :
226 23 : shape_cls = PyObject_GetAttrString (api_module, "TensorShape");
227 23 : Py_SAFEDECREF (api_module);
228 :
229 23 : if (shape_cls == NULL)
230 0 : return -EINVAL;
231 :
232 23 : return loadScript (&core_obj, module_name.c_str (), "CustomConverter");
233 : }
234 :
235 : /**
236 : * @brief get the script path
237 : * @return the script path.
238 : */
239 : const char *
240 2 : PYConverterCore::getScriptPath ()
241 : {
242 2 : return script_path.c_str ();
243 : }
244 :
245 : /**
246 : * @brief Free privateData and move on.
247 : */
248 : static void
249 16 : py_close (void **private_data)
250 : {
251 16 : PYConverterCore *core = static_cast<PYConverterCore *> (*private_data);
252 :
253 16 : g_return_if_fail (core != NULL);
254 :
255 16 : PyGILState_STATE gstate = PyGILState_Ensure ();
256 16 : delete core;
257 16 : PyGILState_Release (gstate);
258 :
259 16 : *private_data = NULL;
260 : }
261 :
262 : /**
263 : * @brief The open callback for GstTensorConverterFramework. Called before anything else
264 : * @param path: python script path
265 : * @param private_data: python plugin's private data
266 : */
267 : static int
268 24 : py_open (const gchar *path, void **priv_data)
269 : {
270 24 : int ret = 0;
271 : PYConverterCore *core;
272 : PyGILState_STATE gstate;
273 :
274 24 : if (!Py_IsInitialized ())
275 0 : throw std::runtime_error ("Python is not initialize.");
276 :
277 : /** Load python script file */
278 24 : core = static_cast<PYConverterCore *> (*priv_data);
279 :
280 24 : if (core != NULL) {
281 2 : if (g_strcmp0 (path, core->getScriptPath ()) == 0)
282 1 : return 1; /* skipped */
283 :
284 1 : py_close (priv_data);
285 : }
286 :
287 : /* init null */
288 23 : *priv_data = NULL;
289 :
290 23 : gstate = PyGILState_Ensure ();
291 23 : core = new PYConverterCore (path);
292 23 : if (core == NULL) {
293 0 : Py_ERRMSG ("Failed to allocate memory for converter subplugin or path invalid: Python\n");
294 0 : ret = -ENOMEM;
295 0 : goto done;
296 : }
297 :
298 23 : if (core->init () != 0) {
299 7 : delete core;
300 7 : Py_ERRMSG ("failed to initialize the object or the python script is invalid: Python\n");
301 7 : ret = -EINVAL;
302 7 : goto done;
303 : }
304 :
305 16 : *priv_data = core;
306 23 : done:
307 23 : PyGILState_Release (gstate);
308 23 : return ret;
309 : }
310 :
311 : /** @brief tensor converter plugin's NNStreamerExternalConverter callback */
312 : static GstCaps *
313 458 : python_query_caps (const GstTensorsConfig *config)
314 : {
315 : UNUSED (config);
316 458 : return gst_caps_from_string ("application/octet-stream");
317 : }
318 :
319 : /**
320 : * @brief tensor converter plugin's NNStreamerExternalConverter callback
321 : */
322 : static gboolean
323 20 : python_get_out_config (const GstCaps *in_cap, GstTensorsConfig *config)
324 : {
325 : GstStructure *structure;
326 20 : g_return_val_if_fail (config != NULL, FALSE);
327 19 : gst_tensors_config_init (config);
328 19 : g_return_val_if_fail (in_cap != NULL, FALSE);
329 :
330 18 : structure = gst_caps_get_structure (in_cap, 0);
331 18 : g_return_val_if_fail (structure != NULL, FALSE);
332 :
333 : /* All tensor info should be updated later in chain function. */
334 18 : config->info.info[0].type = _NNS_UINT8;
335 18 : config->info.num_tensors = 1;
336 18 : if (gst_tensor_parse_dimension ("1:1:1:1", config->info.info[0].dimension) == 0) {
337 0 : Py_ERRMSG ("Failed to set initial dimension for subplugin");
338 0 : return FALSE;
339 : }
340 :
341 18 : if (gst_structure_has_field (structure, "framerate")) {
342 10 : gst_structure_get_fraction (structure, "framerate", &config->rate_n, &config->rate_d);
343 : } else {
344 : /* cannot get the framerate */
345 8 : config->rate_n = 0;
346 8 : config->rate_d = 1;
347 : }
348 18 : return TRUE;
349 : }
350 :
351 : /**
352 : * @brief tensor converter plugin's NNStreamerExternalConverter callback
353 : */
354 : static GstBuffer *
355 28 : python_convert (GstBuffer *in_buf, GstTensorsConfig *config, void *priv_data)
356 : {
357 : GstBuffer *ret;
358 28 : PYConverterCore *core = static_cast<PYConverterCore *> (priv_data);
359 : PyGILState_STATE gstate;
360 28 : g_return_val_if_fail (in_buf, NULL);
361 27 : g_return_val_if_fail (config, NULL);
362 :
363 26 : gstate = PyGILState_Ensure ();
364 26 : ret = core->convert (in_buf, config);
365 26 : PyGILState_Release (gstate);
366 26 : return ret;
367 : }
368 :
369 : static const gchar converter_subplugin_python[] = "python3";
370 :
371 : /** @brief flatbuffer tensor converter sub-plugin NNStreamerExternalConverter instance */
372 : static NNStreamerExternalConverter Python = {
373 : .name = converter_subplugin_python,
374 : .convert = python_convert,
375 : .get_out_config = python_get_out_config,
376 : .query_caps = python_query_caps,
377 : .open = py_open,
378 : .close = py_close,
379 : };
380 :
381 : #ifdef __cplusplus
382 : extern "C" {
383 : #endif /* __cplusplus */
384 : /** @brief Initialize this object for tensor converter sub-plugin */
385 : void
386 431 : init_converter_py (void)
387 : {
388 : /** Python should be initialized and finalized only once */
389 431 : nnstreamer_python_init_refcnt ();
390 431 : registerExternalConverter (&Python);
391 431 : }
392 :
393 : /** @brief Destruct this object for tensor converter sub-plugin */
394 : void
395 431 : fini_converter_py (void)
396 : {
397 431 : nnstreamer_python_status_check ();
398 431 : nnstreamer_python_fini_refcnt ();
399 431 : unregisterExternalConverter (Python.name);
400 : /**
401 : * @todo Remove below lines after this issue is addressed.
402 : * Tizen issues: After python version has been upgraded from 3.9.1 to 3.9.10,
403 : * python converter is stopped at Py_Finalize. Since Py_Initialize is not called
404 : * twice from this object, Py_Finalize is temporarily removed.
405 : */
406 : #if 0
407 : /** Python should be initialized and finalized only once */
408 : if (Py_IsInitialized())
409 : Py_Finalize ();
410 : #endif
411 431 : }
412 : #ifdef __cplusplus
413 : }
414 : #endif /* __cplusplus */
|
