Script of Producer/Consumer

GStreamer: producer

$ ffmpeg -f x11grab -r 15 -s 1280x720 -i :0.0+0,0 -vcodec rawvideo -pix_fmt yuv420p -threads 0 -f v4l2 /dev/video0

$ gst-launch-1.0 videotestsrc ! v4l2sink device=/dev/video0

$ wget http://file-examples.com/wp-content/uploads/2017/04/file_example_MP4_640_3MG.mp4
$ gst-launch-1.0  filesrc location=./file_example_MP4_640_3MG.mp4 ! decodebin ! videoconvert ! v4l2sink device=/dev/video0

GStreamer: consumer

$ gst-launch-1.0 v4l2src device=/dev/video0 ! xvimagesink
(Tip: In case of remote connection such as VNC, run "gst-launch-1.0 v4l2src device=/dev/video0 ! videoconvert ! ximagesink")

$ ./nnstreamer_example_filter

Script of nnstreamer_example_filter using tensorflow lite model (e.g., Mobilenet)

• ltrace -f -tt -e gst_element_get_type -e memcpy -o mytracing1.log ...
• $ ./get-model.sh image-classification-tflite

gst-launch-1.0 -v -m --gst-debug=3 \
v4l2src ! videoconvert ! videoscale ! video/x-raw,width=640,height=480,format=RGB ! tee name=t_raw \
textoverlay name=overlay font-desc="Sans, 24" ! videoconvert ! ximagesink \
t_raw. ! queue ! overlay.video_sink \
t_raw. ! queue ! videoscale ! video/x-raw,width=224,height=224 ! tensor_converter ! \
tensor_filter framework=tensorflow-lite model=./tflite_model_img/mobilenet_v1_1.0_224_quant.tflite ! \
tensor_decoder mode=image_labeling option1=./tflite_model_img/labels.txt ! overlay.text_sink

Audio classification using yamnet tflite model

# Get the yamnet/classification tflite model file
wget https://tfhub.dev/google/lite-model/yamnet/classification/tflite/1?lite-format=tflite -O yamnet_classification.tflite

# Unzip the label file "yamnet_label_list.txt" from the model file
unzip yamnet_classification.tflite

# Run gstreamer pipeline.
# Simple explanation:
#   - get audio from some device (webcam or mic)
#   - its sample rate is 16000 hz
#   - make 15600-length tensor hopping 3900 samples
#   - 32768 ~ +32767 -> -1.0 ~ +1.0
#   - feed the input to the model
#   - get argmax of the output and get the corresponding label by image_labeling tensor_decoder
#   - overlay the result text on the videotestsrc
gst-launch-1.0 \
  alsasrc ! audioconvert ! audio/x-raw,format=S16LE,channels=1,rate=16000,layout=interleaved ! \
    tensor_converter frames-per-tensor=3900 ! \
    tensor_aggregator frames-in=3900 frames-out=15600 frames-flush=3900 frames-dim=1 ! \
    tensor_transform mode=arithmetic option=typecast:float32,add:0.5,div:32767.5 ! \
    tensor_transform mode=transpose option=1:0:2:3 ! \
    queue leaky=2 max-size-buffers=10 ! \
    tensor_filter framework=tensorflow2-lite model=yamnet_classification.tflite custom=Delegate:XNNPACK,NumThreads:4 ! \
    tensor_decoder mode=image_labeling option1=yamnet_label_list.txt ! overlay.text_sink \
  videotestsrc ! videoconvert ! videoscale ! video/x-raw,width=640,height=480,format=RGB ! queue ! overlay.video_sink \
  textoverlay name=overlay font-desc="Sans, 24" ! videoconvert ! autovideosink sync=false

Script of nnstreamer_example_filter using tensorflow model (e.g., Mobilenet)

• ltrace -f -tt -e gst_element_get_type -e memcpy -o mytracing1.log ...
• $ ./get-model.sh object-detection-tf

Object detection using tee

• The video is the same as the original camera output and the labels and bounding boxes are updated after processing in the tensor filter.
  (The video output rate is the same as the original video frame rate.)

gst-launch-1.0 -v -m --gst-debug=3 \
v4l2src name=cam_src ! videoscale ! videoconvert ! video/x-raw,width=640,height=480,format=RGB,framerate=30/1 ! tee name=t \
  t. ! queue leaky=2 max-size-buffers=2 ! videoscale ! tensor_converter ! \
    tensor_filter framework=tensorflow model=tf_model/ssdlite_mobilenet_v2.pb \
      input=3:640:480:1 inputname=image_tensor inputtype=uint8 \
      output=1:1:1:1,100:1:1:1,100:1:1:1,4:100:1:1 \
      outputname=num_detections,detection_classes,detection_scores,detection_boxes \
      outputtype=float32,float32,float32,float32 ! \
    tensor_decoder mode=bounding_boxes option1=mobilenet-ssd-postprocess option2=tf_model/coco_labels_list.txt option4=640:480 option5=640:480 ! \
    compositor name=mix sink_0::zorder=2 sink_1::zorder=1 ! videoconvert ! ximagesink \
  t. ! queue leaky=2 max-size-buffers=10 ! mix.

Object detection using output-combination option of the tensor filter

• Video, labels, and bounding boxes are updated after processing in the tensor filter.
  (The video output rate is the same as the processing rate of the tensor filter.)

gst-launch-1.0 -v -m --gst-debug=3 \
v4l2src name=cam_src ! videoscale ! videoconvert ! video/x-raw,width=640,height=480,format=RGB,framerate=30/1 ! \
  tensor_converter ! tensor_filter framework=tensorflow model=tf_model/ssdlite_mobilenet_v2.pb \
      input=3:640:480:1 inputname=image_tensor inputtype=uint8 \
      output=1:1:1:1,100:1:1:1,100:1:1:1,4:100:1:1 \
      outputname=num_detections,detection_classes,detection_scores,detection_boxes \
      outputtype=float32,float32,float32,float32 output-combination=i0,o0,o1,o2,o3 ! \
  tensor_demux name=demux tensorpick=0,1:2:3:4 demux.src_1 ! queue leaky=2 max-size-buffers=2 ! \
    tensor_decoder mode=bounding_boxes option1=mobilenet-ssd-postprocess option2=tf_model/coco_labels_list.txt option4=640:480 option5=640:480 ! \
    compositor name=mix sink_0::zorder=2 sink_1::zorder=1 ! videoconvert ! ximagesink \
  demux.src_0 ! queue leaky=2 max-size-buffers=2 ! tensor_decoder mode=direct_video ! videoconvert ! mix.

Others

gst-launch-1.0 -v \
videotestsrc pattern=1 ! video/x-raw,width=200,height=200,format=RGB \
    ! tee name=t \
videomixer name=mix \
      sink_0::xpos=0   sink_0::ypos=0    sink_0::zorder=0\
      sink_1::xpos=100 sink_1::ypos=0    sink_1::zorder=1\
      sink_2::xpos=200 sink_2::ypos=200  sink_2::zorder=2\
      sink_3::xpos=0   sink_3::ypos=200  sink_3::zorder=3\
    ! videoconvert ! autovideosink \
videotestsrc pattern="black" ! video/x-raw,width=200,height=200,format=RGB \
    ! mix.sink_0 \
t. ! queue ! mix.sink_1 \
t. ! queue ! mix.sink_2 \
t. ! queue ! mix.sink_3

gst-launch-1.0 \
v4l2src name=cam_src ! videoconvert ! videoscale ! \
video/x-raw,width=640,height=480,format=RGB,framerate=30/1 ! tee name=t_raw \
videomixer name=mix \
sink_0::xpos=0 sink_0::ypos=0 sink_0::zorder=0 sink_0::alpha=0.7 \
sink_1::xpos=50 sink_1::ypos=50 sink_1::zorder=1 sink_1::alpha=0.5 ! \
videoconvert ! ximagesink \
t_raw. ! queue ! tensor_converter ! tensor_decoder mode=direct_video ! videoconvert ! tee name=t_tensor \
t_raw. ! queue ! mix.sink_0 \
t_tensor. ! queue ! mix.sink_1 \
t_tensor. ! queue ! videoconvert ! ximagesink

gst-launch-1.0 \
v4l2src name=cam_src ! videoconvert ! videoscale ! \
video/x-raw,width=640,height=480,format=RGB,framerate=30/1 ! tee name=t_raw \
videomixer name=mix \
sink_0::xpos=0 sink_0::ypos=0 sink_0::zorder=0 sink_0::alpha=0.7 \
sink_1::xpos=50 sink_1::ypos=50 sink_1::zorder=1 sink_1::alpha=0.5 ! \
videoconvert ! ximagesink \
t_raw. ! queue ! tensor_converter ! tensor_decoder mode=direct_video ! videoconvert ! ximagesink \
t_raw. ! queue ! mix.sink_0 \
t_raw. ! queue ! tensor_converter ! tensor_decoder mode=direct_video ! videoconvert ! mix.sink_1

with CAM plugin

gst-launch-1.0 \
  v4l2src name=cam_src ! videoconvert ! videoscale ! \
  video/x-raw,width=640,height=480,format=RGB,framerate=\(fraction\)15/1 ! \
  videomixer name=mix sink_0::xpos=0 sink_0::ypos=0 sink_1::xpos=0 ! \
  videoconvert ! xvimagesink \
  videotestsrc ! \
  video/x-raw,width=320,height=240,format=RGB,framerate=\(fraction\)15/1 ! mix.

gst-launch-1.0 \
  v4l2src name=cam_src ! videoconvert ! videoscale ! \
  videoconvert ! video/x-raw,width=640,height=480,format=RGB,framerate=\(fraction\)30/1 ! \
  videobox border-alpha=0 top=0 left=0 ! \
  videomixer name=mix sink_0::alpha=0.7 sink_1::alpha=0.5 ! \
  videoconvert ! xvimagesink \
  videotestsrc ! \
  video/x-raw,format=RGB,framerate=\(fraction\)5/1,width=320,height=240 ! mix.

gst-launch-1.0 \
v4l2src name=cam_src ! \
videoconvert ! video/x-raw,width=640,height=480,format=RGB,framerate=\(fraction\)30/1 ! tee name=t ! \
queue ! videoconvert ! videomixer name=mix ! ximagesink \
t. ! queue ! tensor_converter ! tensor_decoder mode=direct_video ! videoconvert ! mix.

 gst-launch-1.0 -e videomixer name=mix ! videoconvert ! ximagesink \
   videotestsrc pattern=1 ! video/x-raw, framerate=5/1, width=320, height=180, format=RGB ! videobox border-alpha=0 top=0 left=0 ! mix. \
   videotestsrc pattern=15 ! video/x-raw, framerate=5/1, width=320, height=180, format=RGB ! videobox border-alpha=0 top=0 left=-320 ! mix. \
   videotestsrc pattern=13 ! video/x-raw, framerate=5/1, width=320, height=180, format=RGB ! videobox border-alpha=0 top=-180 left=0 ! mix. \
   videotestsrc pattern=0 ! video/x-raw, framerate=5/1, width=320, height=180, format=RGB ! videobox border-alpha=0 top=-180 left=-320 ! mix. \
   videotestsrc pattern=3 ! video/x-raw, framerate=5/1, width=640, height=360, format=RGB ! mix.

gst-launch-1.0 -v -m --gst-debug=3 \
    videomixer name=mix sink_0::xpos=0 sink_0::ypos=0 sink_1::xpos=0 sink_1::ypos=0 !  videoconvert ! glvideosink sync=false \
    filesrc location=bbb_sunflower_720p_24fps_equal.mp4 ! qtdemux name=demux \
    demux.video_0 ! h264parse ! omxh264dec ! tee name=t \
    t.src_0 ! queue ! mix.sink_0 \
    t.src_1 ! queue ! mix.sink_1