Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer / NNStreamer tensor_decoder subplugin, "python3"
4 : * Copyright (C) 2021 Gichan Jang <gichan2.jang@samsung.com>
5 : */
6 : /**
7 : * @file tensordec-python3.cc
8 : * @date May 06 2021
9 : * @brief NNStreamer tensor-decoder subplugin, "python3",
10 : * which decodes tensor or tensors to any media type.
11 : * @see https://github.com/nnstreamer/nnstreamer
12 : * @author Gichan Jang <gichan2.jang@samsung.com>
13 : * @bug No known bugs except for NYI items
14 : */
15 :
16 : #include <nnstreamer_plugin_api_decoder.h>
17 : #include <nnstreamer_util.h>
18 : #include "nnstreamer_python3_helper.h"
19 : #include "tensordecutil.h"
20 :
21 : #ifdef __cplusplus
22 : extern "C" {
23 : #endif /* __cplusplus */
24 : void init_decoder_py (void) __attribute__ ((constructor));
25 : void fini_decoder_py (void) __attribute__ ((destructor));
26 : #ifdef __cplusplus
27 : }
28 : #endif /* __cplusplus */
29 :
30 : /**
31 : * @brief Python embedding core structure
32 : */
33 : class PYDecoderCore
34 : {
35 : public:
36 : /**
37 : * member functions.
38 : */
39 : PYDecoderCore (const char *_script_path);
40 : ~PYDecoderCore ();
41 :
42 : int init ();
43 : const char *getScriptPath ();
44 : GstFlowReturn decode (const GstTensorsConfig *config,
45 : const GstTensorMemory *input, GstBuffer *outbuf);
46 : GstCaps *getOutCaps (const GstTensorsConfig *config);
47 :
48 : /** @brief Lock python-related actions */
49 68 : void Py_LOCK ()
50 : {
51 68 : g_mutex_lock (&py_mutex);
52 68 : }
53 : /** @brief Unlock python-related actions */
54 68 : void Py_UNLOCK ()
55 : {
56 68 : g_mutex_unlock (&py_mutex);
57 68 : }
58 :
59 : private:
60 : std::string module_name;
61 : const std::string script_path;
62 : PyObject *shape_cls;
63 : PyObject *core_obj;
64 : void *handle; /**< returned handle by dlopen() */
65 : GMutex py_mutex;
66 : };
67 :
68 : /**
69 : * @brief PYDecoderCore creator
70 : * @param _script_path : the logical path to '{script_name}.py' file
71 : * @note the script of _script_path will be loaded simultaneously
72 : * @return Nothing
73 : */
74 7 : PYDecoderCore::PYDecoderCore (const char *_script_path)
75 14 : : script_path (_script_path)
76 : {
77 7 : if (openPythonLib (&handle))
78 0 : throw std::runtime_error (dlerror ());
79 :
80 7 : _import_array (); /** for numpy */
81 :
82 : /**
83 : * Parse script path to get module name
84 : * The module name should drop its extension (i.e., .py)
85 : */
86 7 : module_name = script_path;
87 7 : const size_t last_idx = module_name.find_last_of ("/\\");
88 :
89 7 : if (last_idx != std::string::npos)
90 7 : module_name.erase (0, last_idx + 1);
91 :
92 7 : const size_t ext_idx = module_name.rfind ('.');
93 7 : if (ext_idx != std::string::npos)
94 7 : module_name.erase (ext_idx);
95 :
96 7 : addToSysPath (script_path.substr (0, last_idx).c_str ());
97 :
98 7 : core_obj = NULL;
99 7 : shape_cls = NULL;
100 :
101 7 : g_mutex_init (&py_mutex);
102 7 : }
103 :
104 : /**
105 : * @brief PYDecoderCore Destructor
106 : * @return Nothing
107 : */
108 14 : PYDecoderCore::~PYDecoderCore ()
109 : {
110 7 : Py_SAFEDECREF (core_obj);
111 7 : Py_SAFEDECREF (shape_cls);
112 7 : PyErr_Clear ();
113 :
114 7 : dlclose (handle);
115 7 : g_mutex_clear (&py_mutex);
116 7 : }
117 :
118 : /**
119 : * @brief decode tensor(s) to any media stream
120 : */
121 : GstFlowReturn
122 19 : PYDecoderCore::decode (const GstTensorsConfig *config,
123 : const GstTensorMemory *input, GstBuffer *outbuf)
124 : {
125 : GstMapInfo out_info;
126 : GstMemory *out_mem;
127 : gboolean need_alloc;
128 : size_t mem_size;
129 19 : int rate_n = 0, rate_d = 1;
130 19 : PyObject *output = NULL;
131 : PyObject *raw_data, *in_info;
132 19 : GstFlowReturn ret = GST_FLOW_OK;
133 : GstTensorsInfo *info;
134 :
135 19 : Py_LOCK ();
136 19 : info = (GstTensorsInfo *) &config->info;
137 19 : raw_data = PyList_New (info->num_tensors);
138 19 : in_info = PyList_New (info->num_tensors);
139 19 : rate_n = config->rate_n;
140 19 : rate_d = config->rate_d;
141 :
142 59 : for (unsigned int i = 0; i < info->num_tensors; i++) {
143 40 : GstTensorInfo *_info = gst_tensors_info_get_nth_info (info, i);
144 40 : tensor_type nns_type = _info->type;
145 : npy_intp input_dims[]
146 40 : = { (npy_intp) (input[i].size / gst_tensor_get_element_size (nns_type)) };
147 40 : PyObject *input_array = PyArray_SimpleNewFromData (
148 : 1, input_dims, getNumpyType (nns_type), input[i].data);
149 40 : PyList_SetItem (raw_data, i, input_array);
150 :
151 40 : PyObject *shape = PyTensorShape_New (shape_cls, _info);
152 40 : PyList_SetItem (in_info, i, shape);
153 : }
154 :
155 19 : if (!PyObject_HasAttrString (core_obj, (char *) "decode")) {
156 0 : Py_ERRMSG ("Cannot find 'decode'");
157 0 : ret = GST_FLOW_ERROR;
158 0 : goto done;
159 : }
160 :
161 19 : output = PyObject_CallMethod (core_obj, "decode", "OOii", raw_data, in_info, rate_n, rate_d);
162 :
163 19 : if (output) {
164 19 : need_alloc = (gst_buffer_get_size (outbuf) == 0);
165 19 : mem_size = PyBytes_Size (output);
166 :
167 19 : if (need_alloc) {
168 19 : out_mem = gst_allocator_alloc (NULL, mem_size, NULL);
169 : } else {
170 0 : if (gst_buffer_get_size (outbuf) < mem_size) {
171 0 : gst_buffer_set_size (outbuf, mem_size);
172 : }
173 0 : out_mem = gst_buffer_get_all_memory (outbuf);
174 : }
175 :
176 19 : if (!gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) {
177 0 : gst_memory_unref (out_mem);
178 0 : nns_loge ("Cannot map gst memory (tensor decoder flexbuf)\n");
179 0 : ret = GST_FLOW_ERROR;
180 0 : goto done;
181 : }
182 :
183 19 : memcpy (out_info.data, PyBytes_AsString (output), mem_size);
184 :
185 19 : gst_memory_unmap (out_mem, &out_info);
186 :
187 19 : if (need_alloc)
188 19 : gst_buffer_append_memory (outbuf, out_mem);
189 : else
190 0 : gst_buffer_replace_all_memory (outbuf, out_mem);
191 :
192 19 : Py_SAFEDECREF (output);
193 : } else {
194 0 : Py_ERRMSG ("Fail to get output from 'convert'");
195 0 : ret = GST_FLOW_ERROR;
196 : }
197 :
198 19 : done:
199 19 : Py_UNLOCK ();
200 19 : return ret;
201 : }
202 :
203 : /**
204 : * @brief get output caps
205 : */
206 : GstCaps *
207 49 : PYDecoderCore::getOutCaps (const GstTensorsConfig *config)
208 : {
209 49 : PyObject *result = NULL;
210 49 : GstCaps *caps = NULL;
211 : UNUSED (config);
212 :
213 49 : Py_LOCK ();
214 49 : if (!PyObject_HasAttrString (core_obj, (char *) "getOutCaps")) {
215 0 : ml_loge ("Cannot find 'getOutCaps'");
216 0 : ml_loge ("default caps is `application/octet-stream`");
217 0 : caps = gst_caps_from_string ("application/octet-stream");
218 0 : goto done;
219 : }
220 :
221 49 : result = PyObject_CallMethod (core_obj, (char *) "getOutCaps", NULL);
222 49 : if (result) {
223 49 : gchar *caps_str = PyBytes_AsString (result);
224 49 : caps = gst_caps_from_string (caps_str);
225 49 : Py_SAFEDECREF (result);
226 : } else {
227 0 : caps = gst_caps_from_string ("application/octet-stream");
228 : }
229 :
230 49 : done:
231 49 : Py_UNLOCK ();
232 49 : return caps;
233 : }
234 :
235 : /**
236 : * @brief initialize the object with python script
237 : */
238 : int
239 7 : PYDecoderCore::init ()
240 : {
241 : /** Find nnstreamer_api module */
242 7 : PyObject *api_module = PyImport_ImportModule ("nnstreamer_python");
243 7 : if (api_module == NULL) {
244 0 : return -EINVAL;
245 : }
246 :
247 7 : shape_cls = PyObject_GetAttrString (api_module, "TensorShape");
248 7 : Py_SAFEDECREF (api_module);
249 :
250 7 : if (shape_cls == NULL)
251 0 : return -EINVAL;
252 :
253 7 : return loadScript (&core_obj, module_name.c_str (), "CustomDecoder");
254 : }
255 :
256 : /**
257 : * @brief get the script path
258 : * @return the script path.
259 : */
260 : const char *
261 0 : PYDecoderCore::getScriptPath ()
262 : {
263 0 : return script_path.c_str ();
264 : }
265 :
266 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
267 : static int
268 7 : decoder_py_init (void **pdata)
269 : {
270 : UNUSED (pdata);
271 7 : return TRUE;
272 : }
273 :
274 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
275 : static void
276 7 : decoder_py_exit (void **pdata)
277 : {
278 7 : PYDecoderCore *core = static_cast<PYDecoderCore *> (*pdata);
279 :
280 7 : g_return_if_fail (core != NULL);
281 7 : PyGILState_STATE gstate = PyGILState_Ensure ();
282 7 : delete core;
283 7 : PyGILState_Release (gstate);
284 :
285 7 : *pdata = NULL;
286 : }
287 :
288 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
289 : static int
290 7 : decoder_py_setOption (void **pdata, int opNum, const char *param)
291 : {
292 7 : gchar *path = (gchar *) param;
293 7 : int ret = FALSE;
294 :
295 : /* opNum 1 = python script path */
296 7 : if (opNum == 0) {
297 : PYDecoderCore *core;
298 :
299 7 : if (!Py_IsInitialized ())
300 0 : throw std::runtime_error ("Python is not initialize.");
301 :
302 : /** Load python script file */
303 7 : core = static_cast<PYDecoderCore *> (*pdata);
304 :
305 7 : if (core != NULL) {
306 0 : if (g_strcmp0 (path, core->getScriptPath ()) == 0)
307 0 : return TRUE; /* skipped */
308 :
309 0 : decoder_py_exit (pdata);
310 : }
311 :
312 : /* init null */
313 7 : *pdata = NULL;
314 :
315 7 : PyGILState_STATE gstate = PyGILState_Ensure ();
316 :
317 : try {
318 7 : core = new PYDecoderCore (path);
319 0 : } catch (std::bad_alloc &exception) {
320 0 : ml_loge ("Failed to allocate memory for decoder subplugin: python3\n");
321 0 : ml_loge ("%s", exception.what ());
322 0 : goto done;
323 0 : }
324 :
325 7 : if (core->init () != 0) {
326 0 : delete core;
327 0 : ml_loge ("failed to initialize the object: Python3\n");
328 0 : goto done;
329 : }
330 :
331 7 : *pdata = core;
332 7 : ret = TRUE;
333 :
334 7 : done:
335 7 : PyGILState_Release (gstate);
336 7 : return ret;
337 : }
338 :
339 0 : GST_INFO ("Property mode-option-%d is ignored", opNum + 1);
340 0 : return TRUE;
341 : }
342 :
343 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
344 : static GstCaps *
345 49 : decoder_py_getOutCaps (void **pdata, const GstTensorsConfig *config)
346 : {
347 : GstCaps *caps;
348 49 : PYDecoderCore *core = static_cast<PYDecoderCore *> (*pdata);
349 :
350 49 : PyGILState_STATE gstate = PyGILState_Ensure ();
351 49 : caps = core->getOutCaps (config);
352 49 : PyGILState_Release (gstate);
353 49 : setFramerateFromConfig (caps, config);
354 49 : return caps;
355 : }
356 :
357 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
358 : static GstFlowReturn
359 19 : decoder_py_decode (void **pdata, const GstTensorsConfig *config,
360 : const GstTensorMemory *input, GstBuffer *outbuf)
361 : {
362 : GstFlowReturn ret;
363 19 : PYDecoderCore *core = static_cast<PYDecoderCore *> (*pdata);
364 19 : g_return_val_if_fail (config, GST_FLOW_ERROR);
365 19 : g_return_val_if_fail (input, GST_FLOW_ERROR);
366 19 : g_return_val_if_fail (outbuf, GST_FLOW_ERROR);
367 :
368 19 : PyGILState_STATE gstate = PyGILState_Ensure ();
369 19 : ret = core->decode (config, input, outbuf);
370 19 : PyGILState_Release (gstate);
371 19 : return ret;
372 : }
373 :
374 : static gchar decoder_subplugin_python3[] = "python3";
375 :
376 : /** @brief python3fer tensordec-plugin GstTensorDecoderDef instance */
377 : static GstTensorDecoderDef Python = { .modename = decoder_subplugin_python3,
378 : .init = decoder_py_init,
379 : .exit = decoder_py_exit,
380 : .setOption = decoder_py_setOption,
381 : .getOutCaps = decoder_py_getOutCaps,
382 : .decode = decoder_py_decode,
383 : .getTransformSize = NULL };
384 :
385 : #ifdef __cplusplus
386 : extern "C" {
387 : #endif /* __cplusplus */
388 : /** @brief Initialize this object for tensordec-plugin */
389 : void
390 38 : init_decoder_py (void)
391 : {
392 : /** Python should be initialized and finalized only once */
393 38 : nnstreamer_python_init_refcnt ();
394 38 : nnstreamer_decoder_probe (&Python);
395 38 : }
396 :
397 : /** @brief Destruct this object for tensordec-plugin */
398 : void
399 38 : fini_decoder_py (void)
400 : {
401 38 : nnstreamer_python_status_check ();
402 38 : nnstreamer_python_fini_refcnt ();
403 38 : nnstreamer_decoder_exit (Python.modename);
404 :
405 : /**
406 : * @todo Remove below lines after this issue is addressed.
407 : * Tizen issues: After python version has been upgraded from 3.9.1 to 3.9.10,
408 : * python converter is stopped at Py_Finalize. Since Py_Initialize is not called
409 : * twice from this object, Py_Finalize is temporarily removed.
410 : * We do not know if it's safe to call this at this point.
411 : * We can finalize when ALL python subplugins share the same ref counter.
412 : */
413 : #if 0
414 : /** Python should be initialized and finalized only once */
415 : if (Py_IsInitialized ())
416 : Py_Finalize ();
417 : #endif
418 38 : }
419 : #ifdef __cplusplus
420 : }
421 : #endif /* __cplusplus */
|
