Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer / NNStreamer tensor_converter subplugin, "Flatbuffer"
4 : * Copyright (C) 2020 Gichan Jang <gichan2.jang@samsung.com>
5 : */
6 : /**
7 :
8 : * @file tensor_converter_flatbuf.cc
9 : * @date 14 May 2020
10 : * @brief NNStreamer tensor-converter subplugin, "flatbuffer",
11 : * which converts flatbuufers byte stream to tensors.
12 : * @see https://github.com/nnstreamer/nnstreamer
13 : * @author Gichan Jang <gichan2.jang@samsung.com>
14 : * @bug No known bugs except for NYI items
15 : *
16 : */
17 :
18 : /**
19 : * Install flatbuffers
20 : * We assume that you use Ubuntu linux distribution.
21 : * You may simply download binary packages from PPA
22 : *
23 : * $ sudo apt-add-repository ppa:nnstreamer
24 : * $ sudo apt update
25 : * $ sudo apt install libflatbuffers libflatbuffers-dev flatbuffers-compiler
26 : */
27 :
28 : #include <fstream>
29 : #include <glib.h>
30 : #include <gst/gstinfo.h>
31 : #include <iostream>
32 : #include <nnstreamer_generated.h> /* Generated by `flatc`. */
33 : #include <nnstreamer_log.h>
34 : #include <nnstreamer_plugin_api.h>
35 : #include <nnstreamer_plugin_api_converter.h>
36 : #include <nnstreamer_util.h>
37 : #include <typeinfo>
38 : #include "../extra/nnstreamer_flatbuf.h"
39 : #include "tensor_converter_util.h"
40 :
41 : namespace nnstreamer
42 : {
43 : namespace flatbuf
44 : {
45 : #ifdef __cplusplus
46 : extern "C" {
47 : #endif /* __cplusplus */
48 : void init_fbc (void) __attribute__ ((constructor));
49 : void fini_fbc (void) __attribute__ ((destructor));
50 : #ifdef __cplusplus
51 : }
52 : #endif /* __cplusplus */
53 :
54 : /** @brief tensor converter plugin's NNStreamerExternalConverter callback */
55 : static GstCaps *
56 495 : fbc_query_caps (const GstTensorsConfig *config)
57 : {
58 : UNUSED (config);
59 495 : return gst_caps_from_string (GST_FLATBUF_TENSOR_CAP_DEFAULT);
60 : }
61 :
62 : /** @brief tensor converter plugin's NNStreamerExternalConverter callback
63 : * @todo : Consider multi frames, return Bufferlist and
64 : * remove frame size and the number of frames
65 : */
66 : static GstBuffer *
67 83 : fbc_convert (GstBuffer *in_buf, GstTensorsConfig *config, void *priv_data)
68 : {
69 : const Tensors *tensors;
70 : const flatbuffers::Vector<flatbuffers::Offset<Tensor>> *tensor;
71 : const flatbuffers::Vector<unsigned char> *tensor_data;
72 83 : frame_rate fr;
73 83 : GstBuffer *out_buf = NULL;
74 : GstMemory *in_mem, *out_mem;
75 : GstMapInfo in_info;
76 : gsize mem_size;
77 : GstTensorInfo *_info;
78 :
79 : UNUSED (priv_data);
80 :
81 83 : if (!in_buf || !config) {
82 2 : ml_loge ("NULL parameter is passed to tensor_converter::flatbuf");
83 2 : return NULL;
84 : }
85 :
86 81 : in_mem = gst_buffer_get_all_memory (in_buf);
87 81 : if (!gst_memory_map (in_mem, &in_info, GST_MAP_READ)) {
88 0 : nns_loge ("Cannot map input memory / tensor_converter::flatbuf");
89 0 : gst_memory_unref (in_mem);
90 0 : return NULL;
91 : }
92 :
93 81 : tensors = GetTensors (in_info.data);
94 81 : g_assert (tensors);
95 :
96 81 : config->info.num_tensors = tensors->num_tensor ();
97 81 : config->info.format = (tensor_format) tensors->format ();
98 :
99 81 : if (tensors->num_tensor () > NNS_TENSOR_SIZE_LIMIT) {
100 0 : nns_loge ("The number of tensors is limited to %d", NNS_TENSOR_SIZE_LIMIT);
101 0 : goto done;
102 : }
103 81 : config->rate_n = tensors->fr ()->rate_n ();
104 81 : config->rate_d = tensors->fr ()->rate_d ();
105 :
106 81 : tensor = tensors->tensor ();
107 81 : out_buf = gst_buffer_new ();
108 :
109 187 : for (guint i = 0; i < config->info.num_tensors; i++) {
110 : gsize offset;
111 106 : std::string _name = tensor->Get (i)->name ()->str ();
112 106 : const gchar *name = _name.c_str ();
113 :
114 106 : _info = gst_tensors_info_get_nth_info (&config->info, i);
115 :
116 106 : g_free (_info->name);
117 107 : _info->name = (name && strlen (name) > 0) ? g_strdup (name) : NULL;
118 106 : _info->type = (tensor_type) tensor->Get (i)->type ();
119 106 : tensor_data = tensor->Get (i)->data ();
120 :
121 1802 : for (guint j = 0; j < NNS_TENSOR_RANK_LIMIT; j++) {
122 1696 : _info->dimension[j] = tensor->Get (i)->dimension ()->Get (j);
123 : }
124 106 : mem_size = VectorLength (tensor_data);
125 :
126 106 : offset = tensor_data->data () - in_info.data;
127 :
128 106 : out_mem = gst_memory_share (in_mem, offset, mem_size);
129 :
130 106 : gst_tensor_buffer_append_memory (out_buf, out_mem, _info);
131 : }
132 :
133 : /** copy timestamps */
134 81 : gst_buffer_copy_into (
135 : out_buf, in_buf, (GstBufferCopyFlags) GST_BUFFER_COPY_METADATA, 0, -1);
136 81 : done:
137 81 : gst_memory_unmap (in_mem, &in_info);
138 81 : gst_memory_unref (in_mem);
139 :
140 81 : return out_buf;
141 : }
142 :
143 : static const gchar converter_subplugin_flatbuf[] = "flatbuf";
144 :
145 : /** @brief flatbuffer tensor converter sub-plugin NNStreamerExternalConverter instance */
146 : static NNStreamerExternalConverter flatBuf = { .name = converter_subplugin_flatbuf,
147 : .convert = fbc_convert,
148 : .get_out_config = tcu_get_out_config,
149 : .query_caps = fbc_query_caps,
150 : .open = NULL,
151 : .close = NULL };
152 :
153 : #ifdef __cplusplus
154 : extern "C" {
155 : #endif /* __cplusplus */
156 : /** @brief Initialize this object for tensor converter sub-plugin */
157 : void
158 431 : init_fbc (void)
159 : {
160 431 : registerExternalConverter (&flatBuf);
161 431 : }
162 :
163 : /** @brief Destruct this object for tensor converter sub-plugin */
164 : void
165 431 : fini_fbc (void)
166 : {
167 431 : unregisterExternalConverter (flatBuf.name);
168 431 : }
169 : #ifdef __cplusplus
170 : }
171 : #endif /* __cplusplus */
172 :
173 : }; /* Namespace flatbuf */
174 : }; /* Namespace nnstreamer */
|
