Line data Source code
1 : /* SPDX-License-Identifier: LGPL-2.1-only */
2 : /**
3 : * GStreamer / NNStreamer subplugin "protobuf" to support tensor converter and
4 : * decoder
5 : * Copyright (C) 2020 Gichan Jang <gichan2.jang@samsung.com>
6 : */
7 : /**
8 : * @file nnstreamer_protobuf.cc
9 : * @date 16 June 2020
10 : * @brief Protobuf util function for nnstreamer
11 : * @see https://github.com/nnstreamer/nnstreamer
12 : * @author Gichan Jang <gichan2.jang@samsung.com>
13 : * @bug No known bugs except for NYI items
14 : *
15 : */
16 : /**
17 : * Install protobuf
18 : * We assume that you use Ubuntu linux distribution.
19 : * You may simply download binary packages from PPA
20 : *
21 : * $ sudo apt-add-repository ppa:nnstreamer
22 : * $ sudo apt update
23 : * $ sudo apt install libprotobuf-dev libprotobuf-lite17 libprotobuf17
24 : * protobuf-compiler17
25 : */
26 :
27 : #include <nnstreamer_log.h>
28 : #include <nnstreamer_plugin_api.h>
29 : #include <nnstreamer_util.h>
30 : #include "nnstreamer.pb.h" /* Generated by `protoc` */
31 : #include "nnstreamer_protobuf.h"
32 :
33 : /** @brief tensordec-plugin's GstTensorDecoderDef callback */
34 : GstFlowReturn
35 84 : gst_tensor_decoder_protobuf (const GstTensorsConfig *config,
36 : const GstTensorMemory *input, GstBuffer *outbuf)
37 : {
38 : GstMapInfo out_info;
39 : GstMemory *out_mem;
40 : size_t size, outbuf_size;
41 84 : nnstreamer::protobuf::Tensors tensors;
42 84 : nnstreamer::protobuf::Tensors::frame_rate *fr = NULL;
43 : guint num_tensors;
44 : gboolean is_flexible;
45 : GstTensorMetaInfo meta;
46 : GstTensorInfo *_info;
47 :
48 84 : if (!config || !input || !outbuf) {
49 3 : ml_loge ("NULL parameter is passed to tensor_decoder::protobuf");
50 3 : return GST_FLOW_ERROR;
51 : }
52 :
53 81 : is_flexible = gst_tensors_config_is_flexible (config);
54 :
55 81 : num_tensors = config->info.num_tensors;
56 81 : if (num_tensors <= 0 || num_tensors > NNS_TENSOR_SIZE_LIMIT) {
57 0 : ml_loge ("The number of input tenosrs "
58 : "exceeds more than NNS_TENSOR_SIZE_LIMIT, %s",
59 : NNS_TENSOR_SIZE_LIMIT_STR);
60 0 : return GST_FLOW_ERROR;
61 : }
62 81 : tensors.set_num_tensor (num_tensors);
63 :
64 81 : fr = tensors.mutable_fr ();
65 81 : if (!fr) {
66 0 : nns_loge ("Failed to get pointer of tensors / tensordec-protobuf");
67 0 : return GST_FLOW_ERROR;
68 : }
69 :
70 81 : fr->set_rate_n (config->rate_n);
71 81 : fr->set_rate_d (config->rate_d);
72 :
73 81 : tensors.set_format (
74 81 : (nnstreamer::protobuf::Tensors::Tensor_format) config->info.format);
75 :
76 187 : for (unsigned int i = 0; i < num_tensors; ++i) {
77 106 : nnstreamer::protobuf::Tensor *tensor = tensors.add_tensor ();
78 :
79 106 : _info = gst_tensors_info_get_nth_info ((GstTensorsInfo *) &config->info, i);
80 :
81 106 : if (is_flexible) {
82 9 : gst_tensor_meta_info_parse_header (&meta, input[i].data);
83 9 : gst_tensor_meta_info_convert (&meta, _info);
84 : }
85 :
86 106 : tensor->set_name (_info->name ? _info->name : "");
87 106 : tensor->set_type ((nnstreamer::protobuf::Tensor::Tensor_type) _info->type);
88 :
89 1802 : for (int j = 0; j < NNS_TENSOR_RANK_LIMIT; ++j) {
90 1696 : tensor->add_dimension (_info->dimension[j]);
91 : }
92 :
93 106 : tensor->set_data (input[i].data, (int) input[i].size);
94 : }
95 :
96 81 : size = tensors.ByteSizeLong ();
97 81 : outbuf_size = gst_buffer_get_size (outbuf);
98 :
99 81 : if (outbuf_size == 0) {
100 81 : out_mem = gst_allocator_alloc (NULL, size, NULL);
101 : } else {
102 0 : if (outbuf_size < size) {
103 0 : gst_buffer_set_size (outbuf, size);
104 : }
105 0 : out_mem = gst_buffer_get_all_memory (outbuf);
106 : }
107 :
108 81 : if (!gst_memory_map (out_mem, &out_info, GST_MAP_WRITE)) {
109 0 : nns_loge ("Cannot map output memory / tensordec-protobuf");
110 0 : gst_memory_unref (out_mem);
111 0 : return GST_FLOW_ERROR;
112 : }
113 :
114 81 : tensors.SerializeToArray (out_info.data, size);
115 :
116 81 : gst_memory_unmap (out_mem, &out_info);
117 :
118 81 : if (outbuf_size == 0)
119 81 : gst_buffer_append_memory (outbuf, out_mem);
120 : else
121 0 : gst_buffer_replace_all_memory (outbuf, out_mem);
122 :
123 81 : return GST_FLOW_OK;
124 84 : }
125 :
126 : /** @brief tensor converter plugin's NNStreamerExternalConverter callback */
127 : GstBuffer *
128 83 : gst_tensor_converter_protobuf (GstBuffer *in_buf, GstTensorsConfig *config, void *priv_data)
129 : {
130 83 : nnstreamer::protobuf::Tensors tensors;
131 83 : nnstreamer::protobuf::Tensors::frame_rate *fr = NULL;
132 : GstTensorInfo *_info;
133 : GstMemory *in_mem, *out_mem;
134 : GstMapInfo in_info;
135 : GstBuffer *out_buf;
136 : gsize mem_size;
137 : gpointer mem_data;
138 : UNUSED (priv_data);
139 :
140 83 : if (!in_buf || !config) {
141 2 : ml_loge ("NULL parameter is passed to tensor_converter::protobuf");
142 2 : return NULL;
143 : }
144 :
145 81 : in_mem = gst_buffer_get_all_memory (in_buf);
146 81 : if (!gst_memory_map (in_mem, &in_info, GST_MAP_READ)) {
147 0 : nns_loge ("Cannot map input memory / tensor_converter_protobuf");
148 0 : gst_memory_unref (in_mem);
149 0 : return NULL;
150 : }
151 :
152 81 : tensors.ParseFromArray (in_info.data, in_info.size);
153 :
154 81 : config->info.num_tensors = tensors.num_tensor ();
155 81 : config->info.format = (tensor_format) tensors.format ();
156 81 : fr = tensors.mutable_fr ();
157 81 : config->rate_n = fr->rate_n ();
158 81 : config->rate_d = fr->rate_d ();
159 81 : out_buf = gst_buffer_new ();
160 :
161 187 : for (guint i = 0; i < config->info.num_tensors; i++) {
162 106 : const nnstreamer::protobuf::Tensor *tensor = &tensors.tensor (i);
163 106 : std::string _name = tensor->name ();
164 106 : const gchar *name = _name.c_str ();
165 :
166 106 : _info = gst_tensors_info_get_nth_info (&config->info, i);
167 :
168 107 : _info->name = (name && strlen (name) > 0) ? g_strdup (name) : NULL;
169 106 : _info->type = (tensor_type) tensor->type ();
170 1802 : for (guint j = 0; j < NNS_TENSOR_RANK_LIMIT; j++) {
171 1696 : _info->dimension[j] = tensor->dimension (j);
172 : }
173 106 : mem_size = tensor->data ().length ();
174 106 : mem_data = _g_memdup (tensor->data ().c_str (), mem_size);
175 :
176 106 : out_mem = gst_memory_new_wrapped (
177 : (GstMemoryFlags) 0, mem_data, mem_size, 0, mem_size, mem_data, g_free);
178 :
179 106 : gst_tensor_buffer_append_memory (out_buf, out_mem, _info);
180 : }
181 :
182 : /** copy timestamps */
183 81 : gst_buffer_copy_into (
184 : out_buf, in_buf, (GstBufferCopyFlags) GST_BUFFER_COPY_METADATA, 0, -1);
185 81 : gst_memory_unmap (in_mem, &in_info);
186 81 : gst_memory_unref (in_mem);
187 :
188 81 : return out_buf;
189 83 : }
|
