GStreamer

DeepStream是基于GStreamer开发的。它们主要都是做视频流处理的。现在我们来看一个GStreamer的HelloWorld。
在/opt/nvidia/deepstream/deepstream-5.0/sources/apps/sample_apps目录下创建一个新的文件夹basic-tutorial-1，进入该文件夹，创建一个basic-tutorial-1.c文件，内容如下

   #include <gst/gst.h>
main (int argc, char *argv[])
  GstElement *pipeline; //构建一个媒体管道的基本块,包含了各种element，如source,sink
  GstBus *bus;  //总线
  GstMessage *msg;  //总线消息
  /* Initialize GStreamer */
  gst_init (&argc, &argv); //初始化GStreamer
  /* Build the pipeline */
  /* playbin是一个element，它既是source，也是sink，能同时处理整个管道(pipeline)事务
     gst_parse_launch建立一个管道 */
  pipeline =
      gst_parse_launch
      ("playbin uri=https://www.freedesktop.org/software/gstreamer-sdk/data/media/sintel_trailer-480p.webm",
      NULL);
  /* Start playing */
  /* 每一个element都有一个状态，这里是播放这个element的流媒体
  gst_element_set_state (pipeline, GST_STATE_PLAYING);
  /* Wait until error or EOS */
  /* 获取通道element总线 */
  bus = gst_element_get_bus (pipeline);
  /* gst_bus_timed_pop_filtered会阻塞到遇到错误或者流媒体播放结束，并且得到一个消息 */
  msg =
      gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE,
      GST_MESSAGE_ERROR | GST_MESSAGE_EOS);
  /* Free resources */
  /* 释放资源 */
  if (msg != NULL)
    gst_message_unref (msg);
  gst_object_unref (bus);
  gst_element_set_state (pipeline, GST_STATE_NULL);
  gst_object_unref (pipeline);
  return 0;
   
    创建一个Makefile文件，内容如下(具体含义可以参考
    
     C++基础整理
    
    中的
    
     Linux 下的 C++ 环境
    
    )
   
   
    APP:= basic-tutorial-1
TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -)
NVDS_VERSION:=5.0
LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/lib/
APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/bin/
ifeq ($(TARGET_DEVICE),aarch64)
  CFLAGS:= -DPLATFORM_TEGRA
endif
SRCS:= $(wildcard *.c)
INCS:= $(wildcard *.h)
PKGS:= gstreamer-1.0
OBJS:= $(SRCS:.c=.o)
CFLAGS+= -I../../../includes
CFLAGS+= `pkg-config --cflags $(PKGS)`
LIBS:= `pkg-config --libs $(PKGS)`
LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta \
       -Wl,-rpath,$(LIB_INSTALL_DIR)
all: $(APP)
%.o: %.c $(INCS) Makefile
	$(CC) -c -o $@ $(CFLAGS) $<
$(APP): $(OBJS) Makefile
	$(CC) -o $(APP) $(OBJS) $(LIBS)
install: $(APP)
	cp -rv $(APP) $(APP_INSTALL_DIR)
clean:
	rm -rf $(OBJS) $(APP)
   
   
    编译后就有了可执行程序basic-tutorial-1。
   
   
    在GStreamer里面有这样的接口——pad(GstPad)。pad分为sink pad：数据从pad进入一个element；source pad：数据从pad流出element。source element仅包含source pad，sink element仅包含sink pad，filter element两种pad都包含。
   
   
    一个demuxer包含一个sink pad和多个source pad，数据从sink pad输入，然后每个流都有一个source pad。
   
   
    这是一个整体的通道流程图，有一个source，demuxer，两个filter，两个sink。
   
   
    在/opt/nvidia/deepstream/deepstream-5.0/sources/apps/sample_apps目录下创建一个新的文件夹basic-tutorial-2，进入该文件夹，创建一个basic-tutorial-2.c文件，内容如下(下面的代码跟上面的流程图没有关系)
   
   
    #include <gst/gst.h>
/* Structure to contain all our information, so we can pass it to callbacks */
/* 创建一个我们需要的所有element的结构体
typedef struct _CustomData {
  GstElement *pipeline;
  GstElement *source;
  GstElement *convert;
  GstElement *resample;
  GstElement *sink;
} CustomData;
/* Handler for the pad-added signal */
/* 定义回调函数 */
static void pad_added_handler (GstElement *src, GstPad *pad, CustomData *data);
int main(int argc, char *argv[]) {
  CustomData data;    //结构体数据
  GstBus *bus;        //总线
  GstMessage *msg;    //总线消息
  GstStateChangeReturn ret; //播放状态
  gboolean terminate = FALSE; //是否终止
  /* Initialize GStreamer */
  gst_init (&argc, &argv); //GStreamer初始化
  /* Create the elements */
  /* 创建elements */
  /* uridecodebin自己会在内部初始化必要的element,然后把一个URI变成一个原始的音视频流输出，差不多做了playbin2一半的工作，自带demuxer，source pad没有初始化 */
  data.source = gst_element_factory_make ("uridecodebin", "source");
  /* audioconvert在不同的音频格式转换时很有用，具有平台无关性 */
  data.convert = gst_element_factory_make ("audioconvert", "convert");
  /* audioresample音频重采样 */
  data.resample = gst_element_factory_make ("audioresample", "resample");
  /* autoaudiosink自动选择音频设备并输出 */
  data.sink = gst_element_factory_make ("autoaudiosink", "sink");
  /* Create the empty pipeline */
  /* 创建一个空的管道 */
  data.pipeline = gst_pipeline_new ("test-pipeline");
  if (!data.pipeline || !data.source || !data.convert || !data.resample || !data.sink) {
    g_printerr ("Not all elements could be created.\n");
    return -1;
  /* Build the pipeline. Note that we are NOT linking the source at this
   * point. We will do it later. */
  /* 一个pipeline就是一个特定类型的可以包含其他element的bin,可以用在bin上的方法也都可以用在pipeline上,
     gst_bin_add_many()方法在pipeline中加入element.增加单个element的方法是gst_bin_add() */
  gst_bin_add_many (GST_BIN (data.pipeline), data.source, data.convert, data.resample, data.sink, NULL);
  /* 这些刚增加的elements还没有互相连接起来,这里没有把source连接起来，因为source还没有source pad，
     只有在同一个bin里面的element才能连接起来，所以一定要把element在连接之前加入到pipeline中 */
  if (!gst_element_link_many (data.convert, data.resample, data.sink, NULL)) {
    g_printerr ("Elements could not be linked.\n");
    gst_object_unref (data.pipeline);
    return -1;
  /* Set the URI to play */
  /* 给source设置为通过属性播放 */
  g_object_set (data.source, "uri", "https://www.freedesktop.org/software/gstreamer-sdk/data/media/sintel_trailer-480p.webm", NULL);
  /* Connect to the pad-added signal */
  /* GSignal是GStreamer的一个重要部分。它会让你在你感兴趣的事情发生时收到通知。信号是通过名字来区分的，每个GObject都有它自己的信号。
     在这段代码里面，我们使用g_signal_connect()方法把“pad-added”信号和我们的源（uridecodebin）联系了起来，并且注册了一个回调函数。GStreamer把&data这个指针的内容传给回调函数，
     这样CustomData这个数据结构中的数据也就传递了过去。 
     当source element最后获得足够的数据时，它就自动生成source pad,并且触发"pad-added"信号，这样回调函数就会被调用 */
  g_signal_connect (data.source, "pad-added", G_CALLBACK (pad_added_handler), &data);
  /* Start playing */
  /* 开始播放 */
  ret = gst_element_set_state (data.pipeline, GST_STATE_PLAYING);
  if (ret == GST_STATE_CHANGE_FAILURE) {
    g_printerr ("Unable to set the pipeline to the playing state.\n");
    gst_object_unref (data.pipeline);
    return -1;
  /* Listen to the bus */
  /* 获取管道的总线 */
  bus = gst_element_get_bus (data.pipeline);
    /* 获取总线消息，阻塞状态 */
    msg = gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE,
        GST_MESSAGE_STATE_CHANGED | GST_MESSAGE_ERROR | GST_MESSAGE_EOS);
    /* Parse message */
    if (msg != NULL) {
      GError *err;
      gchar *debug_info;
      switch (GST_MESSAGE_TYPE (msg)) {
        case GST_MESSAGE_ERROR:
          gst_message_parse_error (msg, &err, &debug_info);
          g_printerr ("Error received from element %s: %s\n", GST_OBJECT_NAME (msg->src), err->message);
          g_printerr ("Debugging information: %s\n", debug_info ? debug_info : "none");
          g_clear_error (&err);
          g_free (debug_info);
          terminate = TRUE;
          break;
        case GST_MESSAGE_EOS:
          g_print ("End-Of-Stream reached.\n");
          terminate = TRUE;
          break;
        case GST_MESSAGE_STATE_CHANGED:
          /* We are only interested in state-changed messages from the pipeline */
          if (GST_MESSAGE_SRC (msg) == GST_OBJECT (data.pipeline)) {
            GstState old_state, new_state, pending_state;
            gst_message_parse_state_changed (msg, &old_state, &new_state, &pending_state);
            g_print ("Pipeline state changed from %s to %s:\n",
                gst_element_state_get_name (old_state), gst_element_state_get_name (new_state));
          break;
        default:
          /* We should not reach here */
          g_printerr ("Unexpected message received.\n");
          break;
      gst_message_unref (msg);
  } while (!terminate);
  /* Free resources */
  /* 释放资源 */
  gst_object_unref (bus);
  gst_element_set_state (data.pipeline, GST_STATE_NULL);
  gst_object_unref (data.pipeline);
  return 0;
/* This function will be called by the pad-added signal */
/* 回调函数
   src:触发信号的element,这里为uridecodebin
   new_pad:加到src上的pad
   data:随信号过来的参数
static void pad_added_handler (GstElement *src, GstPad *new_pad, CustomData *data) {
  /* 获取audioconvert的输入pad */
  GstPad *sink_pad = gst_element_get_static_pad (data->convert, "sink");
  GstPadLinkReturn ret;
  GstCaps *new_pad_caps = NULL;
  GstStructure *new_pad_struct = NULL;
  const gchar *new_pad_type = NULL;
  g_print ("Received new pad '%s' from '%s':\n", GST_PAD_NAME (new_pad), GST_ELEMENT_NAME (src));
  /* If our converter is already linked, we have nothing to do here */
  /* 检查sink_pad是否已经连接source pad */
  if (gst_pad_is_linked (sink_pad)) {
    g_print ("We are already linked. Ignoring.\n");
    goto exit;
  /* Check the new pad's type */
  new_pad_caps = gst_pad_get_current_caps (new_pad);
  new_pad_struct = gst_caps_get_structure (new_pad_caps, 0);
  new_pad_type = gst_structure_get_name (new_pad_struct);
  if (!g_str_has_prefix (new_pad_type, "audio/x-raw")) {
    g_print ("It has type '%s' which is not raw audio. Ignoring.\n", new_pad_type);
    goto exit;
  /* Attempt the link */
  /* 连接source pad和audioconvert的sink_pad */
  ret = gst_pad_link (new_pad, sink_pad);
  if (GST_PAD_LINK_FAILED (ret)) {
    g_print ("Type is '%s' but link failed.\n", new_pad_type);
  } else {
    g_print ("Link succeeded (type '%s').\n", new_pad_type);
exit:
  /* Unreference the new pad's caps, if we got them */
  if (new_pad_caps != NULL)
    gst_caps_unref (new_pad_caps);
  /* Unreference the sink pad */
  gst_object_unref (sink_pad);
    
     同样建立一个Makefile文件，将APP:= basic-tutorial-1改成APP:= basic-tutorial-2即可。
    
    
     这里管道pipeline除了PLAYING状态外还有三种状态
    
    
     如果我们要修改和重新编译deepstream-app的话，需要添加依赖
    
    
     apt-get install libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev \
   libgstrtspserver-1.0-dev libx11-dev libjson-glib-dev
    
    
     在/opt/nvidia/deepstream/deepstream-5.0/sources/apps
     
      /sample_apps/deepstream-app目录中有着我们调用的主要程序deepstream-app的源码,其中有4个代码文件——deepstream_app.c，deepstream_app_config_parser.c，deepstream_app.h，deepstream_app_main.c。
     
    
    
     
      deepstream_app.h是deepstream_app.c的头文件，它有着对deepstream_app.c中的函数定义。
     
    
    
     deepstream_app.c 是deepstream的操作如调用TensorRT等。pipeline的一些操作在这里，pipeline的建立
    
    
     deepstream_app_config_parser.c 是对配置文件的解析。
    
    
     deepstream_app_main.c为deepstream主函数
    
    
     我们先来看一下deepstream_app_main.c中的主函数
    
    main (int argc, char *argv[])
  GOptionContext *ctx = NULL;  //命令行参数解析器，如-c ....
  GOptionGroup *group = NULL;  //命令行后的参数组，可以包含很多参数
  GError *error = NULL;
  guint i;
  /* 新建一个新的命令行参数解析器 */
  ctx = g_option_context_new ("Nvidia DeepStream Demo");
  /* 新建一个参数组 */
  group = g_option_group_new ("abc", NULL, NULL, NULL, NULL);
  /* 往这个参数组中添加条目 */
  g_option_group_add_entries (group, entries);
  /* 将该参数组设置给解析器 */
  g_option_context_set_main_group (ctx, group);
  /* 给解析器添加参数组 */
  g_option_context_add_group (ctx, gst_init_get_option_group ());
  /* 添加调试 */
  GST_DEBUG_CATEGORY_INIT (NVDS_APP, "NVDS_APP", 0, NULL);
  /* 解析main函数参数如果报错，直接退出 */
  if (!g_option_context_parse (ctx, &argc, &argv, &error)) {
    NVGSTDS_ERR_MSG_V ("%s", error->message);
    return -1;
  /* 打印版本号 */
  if (print_version) {
    g_print ("deepstream-app version %d.%d.%d\n",
        NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO);
    nvds_version_print ();
    return 0;
  /* 打印依赖版本号 */
  if (print_dependencies_version) {
    g_print ("deepstream-app version %d.%d.%d\n",
        NVDS_APP_VERSION_MAJOR, NVDS_APP_VERSION_MINOR, NVDS_APP_VERSION_MICRO);
    nvds_version_print ();
    nvds_dependencies_version_print ();
    return 0;
  /* static gchar **cfg_files = NULL; 
     cfg_files指的就是类似于source1_csi_dec_infer_yolov5.txt的配置文件
     cfg_files是一个数组，指多个文件 */
  if (cfg_files) {
    num_instances = g_strv_length (cfg_files); //获取文件数量
  if (input_files) {
    num_input_files = g_strv_length (input_files);
  /* 如果文件数量为0，直接退出 */
  if (!cfg_files || num_instances == 0) {
    NVGSTDS_ERR_MSG_V ("Specify config file with -c option");
    return_value = -1;
    goto done;
  for (i = 0; i < num_instances; i++) {
    /* AppCtx *appCtx[MAX_INSTANCES]; */
    appCtx[i] = g_malloc0 (sizeof (AppCtx));
    appCtx[i]->person_class_id = -1;
    appCtx[i]->car_class_id = -1;
    appCtx[i]->index = i;
    appCtx[i]->active_source_index = -1;
    if (show_bbox_text) {
      appCtx[i]->show_bbox_text = TRUE;
    if (input_files && input_files[i]) {
      appCtx[i]->config.multi_source_config[0].uri =
          g_strdup_printf ("file://%s", input_files[i]);
      g_free (input_files[i]);
    /* 对配置文件进行解析，并将解析结果放入appCtx的config属性中，这个config是一个NvDsConfig对象 */
    if (!parse_config_file (&appCtx[i]->config, cfg_files[i])) {
      NVGSTDS_ERR_MSG_V ("Failed to parse config file '%s'", cfg_files[i]);
      appCtx[i]->return_value = -1;
      goto done;
  for (i = 0; i < num_instances; i++) {
    /* 根据文件解析结果创建gstreamer管道 */
    if (!create_pipeline (appCtx[i], NULL,
            all_bbox_generated, perf_cb, overlay_graphics)) {
      NVGSTDS_ERR_MSG_V ("Failed to create pipeline");
      return_value = -1;
      goto done;
  /* 创建一个GMainLoop对象 */
  main_loop = g_main_loop_new (NULL, FALSE);
  _intr_setup ();
  /* 每400毫秒运行一次check_for_interrupt函数 */
  g_timeout_add (400, check_for_interrupt, NULL);
  /* 初始化一个互斥锁 */
  g_mutex_init (&disp_lock);
  /* 显示相关 */
  display = XOpenDisplay (NULL);
  for (i = 0; i < num_instances; i++) {
    guint j;
    /* 给管道设置暂停状态 */
    if (gst_element_set_state (appCtx[i]->pipeline.pipeline,
            GST_STATE_PAUSED) == GST_STATE_CHANGE_FAILURE) {
      NVGSTDS_ERR_MSG_V ("Failed to set pipeline to PAUSED");
      return_value = -1;
      goto done;
    /* 如果还无法显示则继续暂停 */
    if (!appCtx[i]->config.tiled_display_config.enable)
      continue;
    /* 处理所有的输入Sink */
    for (j = 0; j < appCtx[i]->config.num_sink_sub_bins; j++) {
      XTextProperty xproperty;  //窗口文字属性
      gchar *title;  //标题
      guint width, height;  //宽，高
      XSizeHints hints = {0};  //窗口在一般情况下的尺寸
      /* 如果有输入延迟，则继续等待 */
      if (!GST_IS_VIDEO_OVERLAY (appCtx[i]->pipeline.instance_bins[0].
              sink_bin.sub_bins[j].sink)) {
        continue;
      if (!display) {
        NVGSTDS_ERR_MSG_V ("Could not open X Display");
        return_value = -1;
        goto done;
      /* 设置显示的宽度 */
      if (appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.width)
        width =
            appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.width;
        width = appCtx[i]->config.tiled_display_config.width;
      /* 设置显示的高度 */
      if (appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.height)
        height =
            appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.height;
        height = appCtx[i]->config.tiled_display_config.height;
      width = (width) ? width : DEFAULT_X_WINDOW_WIDTH;
      height = (height) ? height : DEFAULT_X_WINDOW_HEIGHT;
      /* 设置窗口尺寸参数 */
      hints.flags = PPosition | PSize;
      hints.x = appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.offset_x;
      hints.y = appCtx[i]->config.sink_bin_sub_bin_config[j].render_config.offset_y;
      hints.width = width;
      hints.height = height;
      /* 创建窗口 */
      windows[i] =
          XCreateSimpleWindow (display, RootWindow (display,
              DefaultScreen (display)), hints.x, hints.y, width, height, 2,
              0x00000000, 0x00000000);
      /* 给创建的窗口设置尺寸 */
      XSetNormalHints(display, windows[i], &hints);
      /* 获取窗口名 */
      if (num_instances > 1)
        title = g_strdup_printf (APP_TITLE "-%d", i);
        title = g_strdup (APP_TITLE);
      /* 将标题设置给窗口 */
      if (XStringListToTextProperty ((char **) &title, 1, &xproperty) != 0) {
        XSetWMName (display, windows[i], &xproperty);
        XFree (xproperty.value);
      XSetWindowAttributes attr = { 0 };
      /* 设置窗口事件 */
      if ((appCtx[i]->config.tiled_display_config.enable &&
              appCtx[i]->config.tiled_display_config.rows *
              appCtx[i]->config.tiled_display_config.columns == 1) ||
          (appCtx[i]->config.tiled_display_config.enable == 0 &&
              appCtx[i]->config.num_source_sub_bins == 1)) {
        attr.event_mask = KeyPress;
      } else {
        attr.event_mask = ButtonPress | KeyRelease;
      /* 绑定事件 */
      XChangeWindowAttributes (display, windows[i], CWEventMask, &attr);
      /* 获取窗口id */
      Atom wmDeleteMessage = XInternAtom (display, "WM_DELETE_WINDOW", False);
      if (wmDeleteMessage != None) {
        /* 当窗口退出时，发送ClientMessage消息 */
        XSetWMProtocols (display, windows[i], &wmDeleteMessage, 1);
      XMapRaised (display, windows[i]);
      XSync (display, 1);       //discard the events for now
      /* 将视频在窗口中渲染 */
      gst_video_overlay_set_window_handle (GST_VIDEO_OVERLAY (appCtx
              [i]->pipeline.instance_bins[0].sink_bin.sub_bins[j].sink),
          (gulong) windows[i]);
      gst_video_overlay_expose (GST_VIDEO_OVERLAY (appCtx[i]->
              pipeline.instance_bins[0].sink_bin.sub_bins[j].sink));
      if (!x_event_thread)
        x_event_thread = g_thread_new ("nvds-window-event-thread",
            nvds_x_event_thread, NULL);
  /* Dont try to set playing state if error is observed */
  /* 将管道设置为运行状态 */
  if (return_value != -1) {
    for (i = 0; i < num_instances; i++) {
      if (gst_element_set_state (appCtx[i]->pipeline.pipeline,
              GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
        g_print ("\ncan't set pipeline to playing state.\n");
        return_value = -1;
        goto done;
  print_runtime_commands ();
  changemode (1);
  g_timeout_add (40, event_thread_func, NULL);
  g_main_loop_run (main_loop);
  changemode (0);
done:
  g_print ("Quitting\n");
  for (i = 0; i < num_instances; i++) {
    if (appCtx[i]->return_value == -1)
      return_value = -1;
    destroy_pipeline (appCtx[i]);
    g_mutex_lock (&disp_lock);
    if (windows[i])
      XDestroyWindow (display, windows[i]);
    windows[i] = 0;
    g_mutex_unlock (&disp_lock);
    g_free (appCtx[i]);
  g_mutex_lock (&disp_lock);
  if (display)
    XCloseDisplay (display);
  display = NULL;
  g_mutex_unlock (&disp_lock);
  g_mutex_clear (&disp_lock);
  if (main_loop) {
    g_main_loop_unref (main_loop);
  if (ctx) {
    g_option_context_free (ctx);
  if (return_value == 0) {
    g_print ("App run successful\n");
  } else {
    g_print ("App run failed\n");
  gst_deinit ();
  return return_value;
    
     可接受的所有的命令行参数
    




    

    
     GOptionEntry entries[] = {
  {"version", 'v', 0, G_OPTION_ARG_NONE, &print_version,
      "Print DeepStreamSDK version", NULL}
  {"tiledtext", 't', 0, G_OPTION_ARG_NONE, &show_bbox_text,
      "Display Bounding box labels in tiled mode", NULL}
  {"version-all", 0, 0, G_OPTION_ARG_NONE, &print_dependencies_version,
      "Print DeepStreamSDK and dependencies version", NULL}
  {"cfg-file", 'c', 0, G_OPTION_ARG_FILENAME_ARRAY, &cfg_files,
      "Set the config file", NULL}
  {"input-file", 'i', 0, G_OPTION_ARG_FILENAME_ARRAY, &input_files,
      "Set the input file", NULL}
  {NULL}
     
      第二个字母代表参数类型，如果用第一个单词使用--连接，如果使用第二个字母，使用-连接，如
     
     
      /deepstream-app$deepstream-app --version
deepstream-app version 5.0.0
DeepStreamSDK 5.0.0
/deepstream-app$deepstream-app -v
deepstream-app version 5.0.0
DeepStreamSDK 5.0.0
     
     
      AppCtx是一个结构体，定义在deepstream_app.h的头文件中，结构如下
     
     
      typedef struct _AppCtx AppCtx;
struct _AppCtx
  gboolean version;
  gboolean cintr;
  gboolean show_bbox_text;
  gboolean seeking;
  gboolean quit;
  gint person_class_id;
  gint car_class_id;
  gint return_value;
  guint index;
  gint active_source_index;
  GMutex app_lock;
  GCond app_cond;
  NvDsPipeline pipeline;
  NvDsConfig config;
  NvDsConfig override_config;
  NvDsInstanceData instance_data[MAX_SOURCE_BINS];
  NvDsC2DContext *c2d_ctx[MAX_MESSAGE_CONSUMERS];
  NvDsAppPerfStructInt perf_struct;
  bbox_generated_callback bbox_generated_post_analytics_cb;
  bbox_generated_callback all_bbox_generated_cb;
  overlay_graphics_callback overlay_graphics_cb;
  NvDsFrameLatencyInfo *latency_info;
  GMutex latency_lock;
  GThread *ota_handler_thread;
  guint ota_inotify_fd;
  guint ota_watch_desc;
      
       现在来看一下parse_config_file解析配置文件这个方法，该方法位于deepstream_app_config_parser.c 中
      
      
       gboolean
parse_config_file (NvDsConfig *config, gchar *cfg_file_path)
  GKeyFile *cfg_file = g_key_file_new ();  //创建一个配置文件缓冲区
  GError *error = NULL;
  gboolean ret = FALSE;
  gchar **groups = NULL;
  gchar **group;
  guint i, j;
  config->source_list_enabled = FALSE;
  if (!APP_CFG_PARSER_CAT) {
    GST_DEBUG_CATEGORY_INIT (APP_CFG_PARSER_CAT, "NVDS_CFG_PARSER", 0, NULL);
  /* 将实体配置文件载入到配置文件缓冲区cfg_file */
  if (!g_key_file_load_from_file (cfg_file, cfg_file_path, G_KEY_FILE_NONE,
          &error)) {
    GST_CAT_ERROR (APP_CFG_PARSER_CAT, "Failed to load uri file: %s",
        error->message);
    goto done;
  /* 判断源分组列表是否存在 */
  if (g_key_file_has_group (cfg_file, CONFIG_GROUP_SOURCE_LIST)) {
    if (!parse_source_list (config, cfg_file, cfg_file_path)) {
      GST_CAT_ERROR (APP_CFG_PARSER_CAT, "Failed to parse '%s' group",
          CONFIG_GROUP_SOURCE_LIST);
      goto done;
    /* 提取源的数量 */
    config->num_source_sub_bins = config->total_num_sources;
    config->source_list_enabled = TRUE;
    /* 判断源多分组是否存在，不存在报错 */
    if (!g_key_file_has_group (cfg_file, CONFIG_GROUP_SOURCE_ALL)) {
      NVGSTDS_ERR_MSG_V ("[source-attr-all] group not present.");
      ret = FALSE;
      goto done;
    g_key_file_remove_group (cfg_file, CONFIG_GROUP_SOURCE_LIST, &error);
  /* 如果源多分组存在 */
  if (g_key_file_has_group (cfg_file, CONFIG_GROUP_SOURCE_ALL)) {
    /* 解析源多分组
       global_source_config是一个NvDsSourceConfig对象 */
    if (!parse_source (&global_source_config,
            cfg_file, CONFIG_GROUP_SOURCE_ALL, cfg_file_path)) {
      GST_CAT_ERROR (APP_CFG_PARSER_CAT, "Failed to parse '%s' group",
          CONFIG_GROUP_SOURCE_LIST);
      goto done;
    /* 将解析结果输送给appCtx的config属性中 */
    if (!set_source_all_configs (config, cfg_file_path)) {
      ret = FALSE;
      goto done;
    /* 解析完删除缓冲区中的源多分组 */
    g_key_file_remove_group (cfg_file, CONFIG_GROUP_SOURCE_ALL, &error);
  /* 读取缓冲区中的分组 */
  groups = g_key_file_get_groups (cfg_file, NULL);
  /* 遍历所有的分组 */
  for (group = groups; *group; group++) {
    gboolean parse_err = FALSE;
    GST_CAT_DEBUG (APP_CFG_PARSER_CAT, "Parsing group: %s", *group);
    /* 判断是否是application分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_APP)) {
      /* 解析application分组，将解析结果放入appCtx的config属性中 */
      parse_err = !parse_app (config, cfg_file, cfg_file_path);
    /* 判断是否是source分组 */
    if (!strncmp (*group, CONFIG_GROUP_SOURCE,
            sizeof (CONFIG_GROUP_SOURCE) - 1)) {
      if (config->num_source_sub_bins == MAX_SOURCE_BINS) {
        NVGSTDS_ERR_MSG_V ("App supports max %d sources", MAX_SOURCE_BINS);
        ret = FALSE;
        goto done;
      /* 获取source分组的索引,如source0,source1 */
      gchar *source_id_start_ptr = *group + strlen (CONFIG_GROUP_SOURCE);
      gchar *source_id_end_ptr = NULL;
      guint index =
          g_ascii_strtoull (source_id_start_ptr, &source_id_end_ptr, 10);
      if (source_id_start_ptr == source_id_end_ptr
          || *source_id_end_ptr != '\0') {
        NVGSTDS_ERR_MSG_V
            ("Source group \"[%s]\" is not in the form \"[source<%%d>]\"",
            *group);
        ret = FALSE;
        goto done;
      guint source_id = 0;
      if (config->source_list_enabled) {
        if (index >= config->total_num_sources) {
          NVGSTDS_ERR_MSG_V
              ("Invalid source group index %d, index cannot exceed %d", index,
              config->total_num_sources);
          ret = FALSE;
          goto done;
        source_id = index;
        NVGSTDS_INFO_MSG_V ("Some parameters to be overwritten for group [%s]",
            *group);
      } else {
        source_id = config->num_source_sub_bins;
      /* 解析source分组，将解析结果放入appCtx的config属性中 */
      parse_err = !parse_source (&config->multi_source_config[source_id],
          cfg_file, *group, cfg_file_path);
      if (config->source_list_enabled
          && config->multi_source_config[source_id].type ==
          NV_DS_SOURCE_URI_MULTIPLE) {
        NVGSTDS_ERR_MSG_V
            ("MultiURI support not available if [source-list] is provided");
        ret = FALSE;
        goto done;
      if (config->multi_source_config[source_id].enable
          && !config->source_list_enabled) {
        config->num_source_sub_bins++;
    /* 判断是否是streammux(混流)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_STREAMMUX)) {
      /* 解析streammux分组，将解析结果放入appCtx的config.streammux_config属性中 */
      parse_err = !parse_streammux (&config->streammux_config, cfg_file, cfg_file_path);
    /* 判断是否是osd(每一帧上显示的文本和矩形框)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_OSD)) {
      /* 解析osd分组，将解析结果放入appCtx的config.osd_config属性中 */
      parse_err = !parse_osd (&config->osd_config, cfg_file);
    /* 判断是否是primary-gie(插件)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_PRIMARY_GIE)) {
      /* 解析primary-gie分组，将解析结果放入appCtx的config->primary_gie_config属性中 */
      parse_err =
          !parse_gie (&config->primary_gie_config, cfg_file,
          CONFIG_GROUP_PRIMARY_GIE, cfg_file_path);
    /* 判断是否是tracker(目标跟踪)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_TRACKER)) {
      /* 解析tracker分组，将解析结果放入appCtx的config->tracker_config属性中 */
      parse_err = !parse_tracker (&config->tracker_config, cfg_file, cfg_file_path);
    /* 判断是否是secondary-gie(二级插件)分组 */
    if (!strncmp (*group, CONFIG_GROUP_SECONDARY_GIE,
                  sizeof (CONFIG_GROUP_SECONDARY_GIE) - 1)) {
      if (config->num_secondary_gie_sub_bins == MAX_SECONDARY_GIE_BINS) {
        NVGSTDS_ERR_MSG_V ("App supports max %d secondary GIEs", MAX_SECONDARY_GIE_BINS);
        ret = FALSE;
        goto done;
      /* 解析secondary-gie分组，将解析结果放入appCtx的secondary_gie_sub_bin_config[config->
                                  num_secondary_gie_sub_bins]属性中 */
      parse_err =
          !parse_gie (&config->secondary_gie_sub_bin_config[config->
                                  num_secondary_gie_sub_bins],
                                  cfg_file, *group, cfg_file_path);
      if (config->secondary_gie_sub_bin_config[config->num_secondary_gie_sub_bins].enable){
        config->num_secondary_gie_sub_bins++;
    /* 判断是否是sink(输出)分组 */
    if (!strncmp (*group, CONFIG_GROUP_SINK, sizeof (CONFIG_GROUP_SINK) - 1)) {
      if (config->num_sink_sub_bins == MAX_SINK_BINS) {
        NVGSTDS_ERR_MSG_V ("App supports max %d sinks", MAX_SINK_BINS);
        ret = FALSE;
        goto done;
      /* 解析sink分组，将解析结果放入appCtx的config.
          sink_bin_sub_bin_config[config.num_sink_sub_bins]属性中 */
      parse_err =
          !parse_sink (&config->
          sink_bin_sub_bin_config[config->num_sink_sub_bins], cfg_file, *group,
          cfg_file_path);
      if (config->
          sink_bin_sub_bin_config[config->num_sink_sub_bins].enable){
        config->num_sink_sub_bins++;
    /* 判断是否是msg_consumer(消息队列)分组 */
    if (!strncmp (*group, CONFIG_GROUP_MSG_CONSUMER,
        sizeof (CONFIG_GROUP_MSG_CONSUMER) - 1)) {
      if (config->num_message_consumers == MAX_MESSAGE_CONSUMERS) {
        NVGSTDS_ERR_MSG_V ("App supports max %d consumers", MAX_MESSAGE_CONSUMERS);
        ret = FALSE;
        goto done;
      /* 解析msg_consumer分组，将解析结果放入appCtx的config.message_consumer_config[config.num_message_consumers]属性中 */
      parse_err = !parse_msgconsumer (
                    &config->message_consumer_config[config->num_message_consumers],
                    cfg_file, *group, cfg_file_path);
      if (config->message_consumer_config[config->num_message_consumers].enable) {
        config->num_message_consumers++;
    /* 判断是否是tiled_display(拼接显示)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_TILED_DISPLAY)) {
      /* 解析tiled_display分组，将解析结果放入appCtx的config.tiled_display_config属性中 */
      parse_err = !parse_tiled_display (&config->tiled_display_config, cfg_file);
    /* 判断是否是img_save(图片保存)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_IMG_SAVE)) {
      /* 解析img_save分组，将解析结果放入appCtx的config.image_save_config属性中 */
      parse_err = !parse_image_save (&config->image_save_config , cfg_file, *group, cfg_file_path);
    /* 判断是否是dsanalytics(视频分析插件)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_DSANALYTICS)) {
      /* 解析dsanalytics分组，将解析结果放入appCtx的config.dsanalytics_config属性中 */
      parse_err = !parse_dsanalytics (&config->dsanalytics_config, cfg_file, cfg_file_path);
    /* 判断是否是dsexample(插件具体功能实现)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_DSEXAMPLE)) {
      /* 解析dsexample分组，将解析结果放入appCtx的config.dsexample_config属性中 */
      parse_err = !parse_dsexample (&config->dsexample_config, cfg_file);
    /* 判断是否是msg_converter(消息转化)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_MSG_CONVERTER)) {
      /* 解析msg_converter分组，将解析结果放入appCtx的config.msg_conv_config属性中 */
      parse_err = !parse_msgconv (&config->msg_conv_config, cfg_file, *group, cfg_file_path);
    /* 判断是否是tests(测试)分组 */
    if (!g_strcmp0 (*group, CONFIG_GROUP_TESTS)) {
      /* 解析tests分组，将解析结果放入appCtx的config属性中 */
      parse_err = !parse_tests (config, cfg_file);
    if (parse_err) {
      GST_CAT_ERROR (APP_CFG_PARSER_CAT, "Failed to parse '%s' group", *group);
      goto done;
  /* 以下都是某些分组的细节处理 */
  for (i = 0; i < config->num_secondary_gie_sub_bins; i++) {
    if (config->secondary_gie_sub_bin_config[i].unique_id ==
        config->primary_gie_config.unique_id) {
      NVGSTDS_ERR_MSG_V ("Non unique gie ids found");
      ret = FALSE;
      goto done;
  for (i = 0; i < config->num_secondary_gie_sub_bins; i++) {
    for (j = i + 1; j < config->num_secondary_gie_sub_bins; j++) {
      if (config->secondary_gie_sub_bin_config[i].unique_id ==
          config->secondary_gie_sub_bin_config[j].unique_id) {
        NVGSTDS_ERR_MSG_V ("Non unique gie id %d found",
                            config->secondary_gie_sub_bin_config[i].unique_id);
        ret = FALSE;
        goto done;
  for (i = 0; i < config->num_source_sub_bins; i++) {
    if (config->multi_source_config[i].type == NV_DS_SOURCE_URI_MULTIPLE) {
      if (config->multi_source_config[i].num_sources < 1) {
        config->multi_source_config[i].num_sources = 1;
      for (j = 1; j < config->multi_source_config[i].num_sources; j++) {
        if (config->num_source_sub_bins == MAX_SOURCE_BINS) {
          NVGSTDS_ERR_MSG_V ("App supports max %d sources", MAX_SOURCE_BINS);
          ret = FALSE;
          goto done;
        memcpy (&config->multi_source_config[config->num_source_sub_bins],
            &config->multi_source_config[i],
            sizeof (config->multi_source_config[i]));
        config->multi_source_config[config->num_source_sub_bins].type =
            NV_DS_SOURCE_URI;
        config->multi_source_config[config->num_source_sub_bins].uri =
            g_strdup_printf (config->multi_source_config[config->
                num_source_sub_bins].uri, j);
        config->num_source_sub_bins++;
      config->multi_source_config[i].type = NV_DS_SOURCE_URI;
      config->multi_source_config[i].uri =
          g_strdup_printf (config->multi_source_config[i].uri, 0);
  ret = TRUE;
/* 释放资源 */
done:
  if (cfg_file) {
    g_key_file_free (cfg_file);
  if (groups) {
    g_strfreev (groups);
  if (error) {
    g_error_free (error);
  if (!ret) {
    NVGSTDS_ERR_MSG_V ("%s failed", __func__);
  return ret;
       
        然后看一下create_pipeline创建管道这个方法，该方法位于deepstream_app.c中
       
       
        gboolean
create_pipeline (AppCtx * appCtx,
    bbox_generated_callback bbox_generated_post_analytics_cb,  //边界框生成回调element，bbox_generated_post_analytics_cb为函数指针
    bbox_generated_callback all_bbox_generated_cb, perf_callback perf_cb,
    overlay_graphics_callback overlay_graphics_cb)   //图像覆盖回调element,overlay_graphics_cb为函数指针
  gboolean ret = FALSE;
  NvDsPipeline *pipeline = &appCtx->pipeline; //获取管道对象地址
  NvDsConfig *config = &appCtx->config; //获取配置内容对象地址
  GstBus *bus;  //总线
  /* 创建3个element */
  GstElement *last_elem;
  GstElement *tmp_elem1;
  GstElement *tmp_elem2;
  guint i;
  GstPad *fps_pad;
  gulong latency_probe_id;
  _dsmeta_quark = g_quark_from_static_string (NVDS_META_STRING);
  appCtx->all_bbox_generated_cb = all_bbox_generated_cb;
  appCtx->bbox_generated_post_analytics_cb = bbox_generated_post_analytics_cb;
  appCtx->overlay_graphics_cb = overlay_graphics_cb;
  if (config->osd_config.num_out_buffers < 8) {
    config->osd_config.num_out_buffers = 8;
  /* 创建一个新的通道 */
  pipeline->pipeline = gst_pipeline_new ("pipeline");
  if (!pipeline->pipeline) {
    NVGSTDS_ERR_MSG_V ("Failed to create pipeline");
    goto done;
  /* 获取该通道的总线 */
  bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline->pipeline));
  /* 提供一个在消息可用时将被调用的回调，类似于g_signal_connect */
  pipeline->bus_id = gst_bus_add_watch (bus, bus_callback, appCtx);
  /* 释放总线资源 */
  gst_object_unref (bus);
  /* 如果source视频流需要循环播放，则循环 */
  if (config->file_loop) {
    /* Let each source bin know it needs to loop. */
    guint i;
    for (i = 0; i < config->num_source_sub_bins; i++)
        config->multi_source_config[i].loop = TRUE;
  /* 遍历所有的sink bin */
  for (guint i = 0; i < config->num_sink_sub_bins; i++) {
    NvDsSinkSubBinConfig *sink_config = &config->sink_bin_sub_bin_config[i];
    switch (sink_config->type) {
      case NV_DS_SINK_FAKE:
      case NV_DS_SINK_RENDER_EGL:
      case NV_DS_SINK_RENDER_OVERLAY:
        /* Set the "qos" property of sink, if not explicitly specified in the
           config. */
        /* 对所有的sink类型设置qos(服务质量，指关于测量和调整pipeline的实时性能)
        if (!sink_config->render_config.qos_value_specified) {
          /* QoS events should be generated by sink always in case of live sources
             or with synchronous playback for non-live sources. */
          if (config->streammux_config.live_source || sink_config->render_config.sync) {
            sink_config->render_config.qos = TRUE;
          } else {
            sink_config->render_config.qos = FALSE;
      default:
        break;
   * Add muxer and < N > source components to the pipeline based
   * on the settings in configuration file.
  /* 增加source的多流设置 */
  if (!create_multi_source_bin (config->num_source_sub_bins,
          config->multi_source_config, &pipeline->multi_src_bin))
    goto done;
  /* 在管道中增加一个多流的element bin */
  gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->multi_src_bin.bin);
  /* 链接混流 */
  if (config->streammux_config.is_parsed)
    set_streammux_properties (&config->streammux_config,
        pipeline->multi_src_bin.streammux);
  /* 设置延迟信息 */
  if(appCtx->latency_info == NULL)
    appCtx->latency_info = (NvDsFrameLatencyInfo *)
      calloc(1, config->streammux_config.batch_size *
          sizeof(NvDsFrameLatencyInfo));
  /** a tee after the tiler which shall be connected to sink(s) */
  /* 创建一个tiler_tee的element,filer类型 */
  pipeline->tiler_tee = gst_element_factory_make (NVDS_ELEM_TEE, "tiler_tee");
  if (!pipeline->tiler_tee) {
    NVGSTDS_ERR_MSG_V ("Failed to create element 'tiler_tee'");
    goto done;
  /* 将该tiler_tee的element添加到管道中 */
  gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->tiler_tee);
  /** Tiler + Demux in Parallel Use-Case */
  if (config->tiled_display_config.enable == NV_DS_TILED_DISPLAY_ENABLE_WITH_PARALLEL_DEMUX)
    /* 创建一个demuxer的element，有多个source pad流出,一个sink pad流入 */
    pipeline->demuxer =
        gst_element_factory_make (NVDS_ELEM_STREAM_DEMUX, "demuxer");
    if (!pipeline->demuxer) {
      NVGSTDS_ERR_MSG_V ("Failed to create element 'demuxer'");
      goto done;
    /* 将该demuxer的element添加到管道中 */
    gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->demuxer);
    /** NOTE:
     * demux output is supported for only one source
     * If multiple [sink] groups are configured with
     * link_to_demux=1, only the first [sink]
     * shall be constructed for all occurences of
     * [sink] groups with link_to_demux=1
      gchar pad_name[16];
      GstPad *demux_src_pad;
      i = 0;
      /* 创建混流管道 */
      if (!create_demux_pipeline (appCtx, i)) {
        goto done;
      for (i=0; i < config->num_sink_sub_bins; i++)
        if (config->sink_bin_sub_bin_config[i].link_to_demux == TRUE)
          g_snprintf (pad_name, 16, "src_%02d", config->sink_bin_sub_bin_config[i].source_id);
          break;
      /* 检查demuxer element中sink pad的数量 */
      if (i >= config->num_sink_sub_bins)
        g_print ("\n\nError : sink for demux (use link-to-demux-only property) is not provided in the config file\n\n");
        goto done;
      i = 0;
      /* 将demuxer bin添加到管道中 */
      gst_bin_add (GST_BIN (pipeline->pipeline),
          pipeline->demux_instance_bins[i].bin);
      /* 从demuxer element中获取一个指定的pad */
      demux_src_pad = gst_element_get_request_pad (pipeline->demuxer, pad_name);
      /* 将demuxer与sink进行串联 */
      NVGSTDS_LINK_ELEMENT_FULL (pipeline->demuxer, pad_name,
          pipeline->demux_instance_bins[i].bin, "sink");
      gst_object_unref (demux_src_pad);
      NVGSTDS_ELEM_ADD_PROBE(latency_probe_id,
          appCtx->pipeline.demux_instance_bins[i].demux_sink_bin.bin,
          "sink",
          demux_latency_measurement_buf_prob, GST_PAD_PROBE_TYPE_BUFFER,
          appCtx);
      latency_probe_id = latency_probe_id;
    last_elem = pipeline->demuxer;
    link_element_to_tee_src_pad (pipeline->tiler_tee, last_elem);
    last_elem = pipeline->tiler_tee;
  if (config->tiled_display_config.enable) {
    /* Tiler will generate a single composited buffer for all sources. So need
     * to create only one processing instance. */
    if (!create_processing_instance (appCtx, 0)) {
      goto done;
    // create and add tiling component to pipeline.
    if (config->tiled_display_config.columns *
        config->tiled_display_config.rows < config->num_source_sub_bins) {
      if (config->tiled_display_config.columns == 0) {
        config->tiled_display_config.columns =
            (guint) (sqrt (config->num_source_sub_bins) + 0.5);
      config->tiled_display_config.rows =
          (guint) ceil (1.0 * config->num_source_sub_bins /
          config->tiled_display_config.columns);
      NVGSTDS_WARN_MSG_V
          ("Num of Tiles less than number of sources, readjusting to "
          "%u rows, %u columns", config->tiled_display_config.rows,
          config->tiled_display_config.columns);
    gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->instance_bins[0].bin);
    last_elem = pipeline->instance_bins[0].bin;
    if (!create_tiled_display_bin (&config->tiled_display_config,
            &pipeline->tiled_display_bin)) {
      goto done;
    gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->tiled_display_bin.bin);
    NVGSTDS_LINK_ELEMENT (pipeline->tiled_display_bin.bin, last_elem);
    last_elem = pipeline->tiled_display_bin.bin;
    link_element_to_tee_src_pad (pipeline->tiler_tee, pipeline->tiled_display_bin.bin);
    last_elem = pipeline->tiler_tee;
    NVGSTDS_ELEM_ADD_PROBE (latency_probe_id,
      pipeline->instance_bins->sink_bin.sub_bins[0].sink, "sink",
      latency_measurement_buf_prob, GST_PAD_PROBE_TYPE_BUFFER,
      appCtx);
    latency_probe_id = latency_probe_id;
     * Create demuxer only if tiled display is disabled.
    pipeline->demuxer =
        gst_element_factory_make (NVDS_ELEM_STREAM_DEMUX, "demuxer");
    if (!pipeline->demuxer) {
      NVGSTDS_ERR_MSG_V ("Failed to create element 'demuxer'");
      goto done;
    gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->demuxer);
    for (i = 0; i < config->num_source_sub_bins; i++)
      gchar pad_name[16];
      GstPad *demux_src_pad;
      /* Check if any sink has been configured to render/encode output for
       * source index `i`. The processing instance for that source will be
       * created only if atleast one sink has been configured as such.
      if (!is_sink_available_for_source_id(config, i))
        continue;
      if (!create_processing_instance(appCtx, i))
        goto done;
      gst_bin_add(GST_BIN(pipeline->pipeline),
                  pipeline->instance_bins[i].bin);
      g_snprintf(pad_name, 16, "src_%02d", i);
      demux_src_pad = gst_element_get_request_pad(pipeline->demuxer, pad_name);
      NVGSTDS_LINK_ELEMENT_FULL(pipeline->demuxer, pad_name,
                                pipeline->instance_bins[i].bin, "sink");
      gst_object_unref(demux_src_pad);
      for (int k = 0; k < MAX_SINK_BINS;k++) {
        if(pipeline->instance_bins[i].sink_bin.sub_bins[k].sink){
          NVGSTDS_ELEM_ADD_PROBE(latency_probe_id,
              pipeline->instance_bins[i].sink_bin.sub_bins[k].sink, "sink",
              latency_measurement_buf_prob, GST_PAD_PROBE_TYPE_BUFFER,
              appCtx);
          break;
      latency_probe_id = latency_probe_id;
    last_elem = pipeline->demuxer;
  if (config->tiled_display_config.enable == NV_DS_TILED_DISPLAY_DISABLE) {
    fps_pad = gst_element_get_static_pad (pipeline->demuxer, "sink");
  else {
    fps_pad = gst_element_get_static_pad (pipeline->tiled_display_bin.bin, "sink");
  pipeline->common_elements.appCtx = appCtx;
  // Decide where in the pipeline the element should be added and add only if
  // enabled
  if (config->dsexample_config.enable) {
    // Create dsexample element bin and set properties
    if (!create_dsexample_bin (&config->dsexample_config,
            &pipeline->dsexample_bin)) {
      goto done;
    // Add dsexample bin to instance bin
    gst_bin_add (GST_BIN (pipeline->pipeline), pipeline->dsexample_bin.bin);
    // Link this bin to the last element in the bin
    NVGSTDS_LINK_ELEMENT (pipeline->dsexample_bin.bin, last_elem);
    // Set this bin as the last element
    last_elem = pipeline->dsexample_bin.bin;
  // create and add common components to pipeline.
  if (!create_common_elements (config, pipeline, &tmp_elem1, &tmp_elem2,
          bbox_generated_post_analytics_cb)) {
    goto done;
  if(!add_and_link_broker_sink(appCtx)) {
    goto done;
  if (tmp_elem2) {
    NVGSTDS_LINK_ELEMENT (tmp_elem2, last_elem);
    last_elem = tmp_elem1;
  NVGSTDS_LINK_ELEMENT (pipeline->multi_src_bin.bin, last_elem);
  // enable performance measurement and add call back function to receive
  // performance data.
  if (config->enable_perf_measurement) {
    appCtx->perf_struct.context = appCtx;
    enable_perf_measurement (&appCtx->perf_struct, fps_pad,
        pipeline->multi_src_bin.num_bins,
        config->perf_measurement_interval_sec,
        config->multi_source_config[0].dewarper_config.num_surfaces_per_frame,
        perf_cb);
  latency_probe_id = latency_probe_id;
  if (config->num_message_consumers) {
    for (i = 0; i < config->num_message_consumers; i++) {
      appCtx->c2d_ctx[i] = start_cloud_to_device_messaging (
                                &config->message_consumer_config[i], NULL,
                                &appCtx->pipeline.multi_src_bin);
      if (appCtx->c2d_ctx[i] == NULL) {
        NVGSTDS_ERR_MSG_V ("Failed to create message consumer");
        goto done;
  GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (appCtx->pipeline.pipeline),
      GST_DEBUG_GRAPH_SHOW_ALL, "ds-app-null");
  g_mutex_init (&appCtx->app_lock);
  g_cond_init (&appCtx->app_cond);
  g_mutex_init (&appCtx->latency_lock);
  ret = TRUE;
done:
  if (!ret) {
    NVGSTDS_ERR_MSG_V ("%s failed", __func__);
  return ret;
        
         在上面的代码中，我们可以看到bbox_generated_callback和overlay_graphics_callback两个回调element，它们是定义在deepstream_app.h中的
        




    

        
         typedef void (*bbox_generated_callback) (AppCtx *appCtx, GstBuffer *buf,
    NvDsBatchMeta *batch_meta, guint index);
typedef gboolean (*overlay_graphics_callback) (AppCtx *appCtx, GstBuffer *buf,
    NvDsBatchMeta *batch_meta, guint index);
        
        
         这是两个指针函数。主要用在搭建管道中的通用element作为回调使用的。这上面有一段非常重要,但却很容易被忽视的代码
        
        
         create_demux_pipeline (appCtx, i))
        
        
         我们来看一下它的内容
        
        
         static gboolean
create_demux_pipeline (AppCtx * appCtx, guint index)
  gboolean ret = FALSE;
  NvDsConfig *config = &appCtx->config;
  NvDsInstanceBin *instance_bin = &appCtx->pipeline.demux_instance_bins[index];
  GstElement *last_elem;
  gchar elem_name[32];
  instance_bin->index = index;
  instance_bin->appCtx = appCtx;
  g_snprintf (elem_name, 32, "processing_demux_bin_%d", index);
  instance_bin->bin = gst_bin_new (elem_name);
  if (!create_demux_sink_bin (config->num_sink_sub_bins,
        config->sink_bin_sub_bin_config, &instance_bin->demux_sink_bin,
        config->sink_bin_sub_bin_config[index].source_id)) {
    goto done;
  gst_bin_add (GST_BIN (instance_bin->bin), instance_bin->demux_sink_bin.bin);
  last_elem = instance_bin->demux_sink_bin.bin;
  if (config->osd_config.enable) {
    if (!create_osd_bin (&config->osd_config, &instance_bin->osd_bin)) {
      goto done;
    gst_bin_add (GST_BIN (instance_bin->bin), instance_bin->osd_bin.bin);
    NVGSTDS_LINK_ELEMENT (instance_bin->osd_bin.bin, last_elem);
    last_elem = instance_bin->osd_bin.bin;
  NVGSTDS_BIN_ADD_GHOST_PAD (instance_bin->bin, last_elem, "sink");
  /* 上面都是去构建element，并且连接element的通道操作，可以忽略 */
  /* 插件处理 */
  if (config->osd_config.enable) {
    NVGSTDS_ELEM_ADD_PROBE (instance_bin->all_bbox_buffer_probe_id,
        instance_bin->osd_bin.nvosd, "sink",
        gie_processing_done_buf_prob, GST_PAD_PROBE_TYPE_BUFFER, instance_bin);
  } else {
    NVGSTDS_ELEM_ADD_PROBE (instance_bin->all_bbox_buffer_probe_id,
        instance_bin->demux_sink_bin.bin, "sink",
        gie_processing_done_buf_prob, GST_PAD_PROBE_TYPE_BUFFER, instance_bin);
  ret = TRUE;
done:
  if (!ret) {
    NVGSTDS_ERR_MSG_V ("%s failed", __func__);
  return ret;
         
          这里有一个gie_processing_done_buf_prob的插件处理过程的函数，我们来看一下它的内容
         
         
          static GstPadProbeReturn
gie_processing_done_buf_prob (GstPad * pad, GstPadProbeInfo * info,
    gpointer u_data)
  GstBuffer *buf = (GstBuffer *) info->data;
  NvDsInstanceBin *bin = (NvDsInstanceBin *) u_data;
  guint index = bin->index;
  AppCtx *appCtx = bin->appCtx;
  if (gst_buffer_is_writable (buf))
    process_buffer (buf, appCtx, index);
  return GST_PAD_PROBE_OK;
          
           这里面又有一个process_buffer的函数，继续看它的内容
          
          
           static void
process_buffer (GstBuffer * buf, AppCtx * appCtx, guint index)
  NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf);
  if (!batch_meta) {
    NVGSTDS_WARN_MSG_V ("Batch meta not found for buffer %p", buf);
    return;
  process_meta (appCtx, batch_meta);
  //NvDsInstanceData *data = &appCtx->instance_data[index];
  //guint i;
  //  data->frame_num++;
  /* 文本，外接矩形框生成
  if (appCtx->all_bbox_generated_cb) {
    appCtx->all_bbox_generated_cb (appCtx, buf, batch_meta, index);
  //data->bbox_list_size = 0;
   * 其他图像处理
  if (appCtx->overlay_graphics_cb) {
    appCtx->overlay_graphics_cb (appCtx, buf, batch_meta, index);
           
            再返回deepstream_app_main.c中，我们会看到传入创建管道的参数是
           
           
            create_pipeline (appCtx[i], NULL, all_bbox_generated, perf_cb, overlay_graphics)
           
           
            我们来具体看一下all_bbox_generated这个函数
           
           
            static void
all_bbox_generated (AppCtx * appCtx, GstBuffer * buf,
    NvDsBatchMeta * batch_meta, guint index)
  guint num_male = 0; //男人数量
  guint num_female = 0; //女人数量
  guint num_objects[128];
  memset (num_objects, 0, sizeof (num_objects));
  /* 遍历每一帧元数据 */
  for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame != NULL;
      l_frame = l_frame->next) {
    NvDsFrameMeta *frame_meta = l_frame->data;
    /* 遍历每一帧中的对象元数据 */
    for (NvDsMetaList * l_obj = frame_meta->obj_meta_list; l_obj != NULL;
        l_obj = l_obj->next) {
      NvDsObjectMeta *obj = (NvDsObjectMeta *) l_obj->data;
      /* 如果对象元数据的组件id为配置中的插件id */
      if (obj->unique_component_id ==
          (gint) appCtx->config.primary_gie_config.unique_id) {
        /* 对象元数据的分类id在0和128之间 */
        if (obj->class_id >= 0 && obj->class_id < 128) {
          num_objects[obj->class_id]++;
        /* 如果对象元数据的分类是人 */
        if (appCtx->person_class_id > -1
            && obj->class_id == appCtx->person_class_id) {
          /* 如果对象元数据的显示字符中包含男人,则用男人替换掉人 */
          if (strstr (obj->text_params.display_text, "Man")) {
            str_replace (obj->text_params.display_text, "Man", "");
            str_replace (obj->text_params.display_text, "Person", "Man");
            /* 男人数量加1 */
            num_male++;
          } /* 如果对象元数据的显示字符中包含女人,则用女人替换掉人 */
          else if (strstr (obj->text_params.display_text, "Woman")) {
            str_replace (obj->text_params.display_text, "Woman", "");
            str_replace (obj->text_params.display_text, "Person", "Woman");
            /* 女人数量加1 */
            num_female++;
            
             然后是overlay_graphics这个函数
            
            
             static gboolean
overlay_graphics (AppCtx * appCtx, GstBuffer * buf,
    NvDsBatchMeta * batch_meta, guint index)
  /* 获取可用的源的索引 */
  int srcIndex = appCtx->active_source_index;
  if (srcIndex == -1)
    return TRUE;
  /* 帧延迟信息 */
  NvDsFrameLatencyInfo *latency_info = NULL;
  /* 从NvDsBatchMeta元数据中获取显示元数据 */
  NvDsDisplayMeta *display_meta =
      nvds_acquire_display_meta_from_pool (batch_meta);
  /* 将该显示元数据的标签数设置为1 */
  display_meta->num_labels = 1;
  /* 设置该显示元数据的文字 */
  display_meta->text_params[0].display_text = g_strdup_printf ("Source: %s",
      appCtx->config.multi_source_config[srcIndex].uri);
  /* 设置该显示元数据的文字y轴偏移量 */
  display_meta->text_params[0].y_offset = 20;
  /* 设置该显示元数据的文字x轴偏移量 */
  display_meta->text_params[0].x_offset = 20;
  /* 设置该显示元数据的文字字体颜色 */
  display_meta->text_params[0].font_params.font_color = (NvOSD_ColorParams) {
  0, 1, 0, 1};
  /* 设置该显示元数据的文字字体大小 */
  display_meta->text_params[0].font_params.font_size =
      appCtx->config.osd_config.text_size * 1.5;
  /* 设置该显示元数据的文字字体 */
  display_meta->text_params[0].font_params.font_name = "Serif";
  display_meta->text_params[0].set_bg_clr = 1;
  display_meta->text_params[0].text_bg_clr = (NvOSD_ColorParams) {
  0, 0, 0, 1.0};
  /* 如果允许延迟误差 */
  if(nvds_enable_latency_measurement) {
    g_mutex_lock (&appCtx->latency_lock);
    latency_info = &appCtx->latency_info[index];
    display_meta->num_labels++;
    display_meta->text_params[1].display_text = g_strdup_printf ("Latency: %lf",
        latency_info->latency);
    g_mutex_unlock (&appCtx->latency_lock);
    display_meta->text_params[1].y_offset = (display_meta->text_params[0].y_offset * 2 )+
      display_meta->text_params[0].font_params.font_size;
    display_meta->text_params[1].x_offset = 20;
    display_meta->text_params[1].font_params.font_color = (NvOSD_ColorParams) {
      0, 1, 0, 1};
    display_meta->text_params[1].font_params.font_size =
      appCtx->config.osd_config.text_size * 1.5;
    display_meta->text_params[1].font_params.font_name = "Arial";
    display_meta->text_params[1].set_bg_clr = 1;
    display_meta->text_params[1].text_bg_clr = (NvOSD_ColorParams) {
      0, 0, 0, 1.0};
  /* 将该显示元数据添加到帧中 */
  nvds_add_display_meta_to_frame (nvds_get_nth_frame_meta (batch_meta->
          frame_meta_list, 0), display_meta);
  return TRUE;
             
              在以上代码中都包含了一种叫NvDsBatchMeta的结构体，
              
               在
              
              Gst-nvstreammux
              
               插件中创建。
              
              它的frame_meta_list属性是一个列表，列表中的元素为NvDsFrameMeta，是帧的元数据。每一个NvDsFrameMeta又有object_meta_list列表和display_meta_list列表，object_meta_list列表中的元素为NvDsObjectMeta，display_meta_list列表中的元素为NvDsDisplayMeta。
             
             
              NvDsObjectMeta代表对象元数据，它数量的多少等同于该帧图像中，所能检测到的对象的多少，是由nvinfer插件创建的，用于向下传递的数据，传递到OSD。NvDsDisplayMeta代表显示的元数据，作用于图像帧中显示/绘制文字和方框的，也是向下传递到OSD。
             
             
              NvDsObjectMeta有一个Classifier类型的metadata_list列表，列表中的元素为
              
               NvDsClassifierMeta，代表分类元数据，只有存在多个网络的时候才会被创建。
              
             
             
              
               deepstream-app整合OpenCV
              
             
             
              这里以保存管道截图为例来进行说明，我们需要修改的是all_bbox_generated这个函数，首先是deepstream_app_main.c中的头文件。
             
             
              #include "deepstream_app.h"
#include "deepstream_config_file_parser.h"
#include "nvds_version.h"
#include "gstnvdsmeta.h"
#include "nvbufsurface.h"
#include "nvbufsurftransform.h"
#include <string.h>
#include <unistd.h>
#include <termios.h>
#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <opencv2/core.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/imgproc/types_c.h>
             
             
              然后是函数本身
             
             
              static void
all_bbox_generated (AppCtx * appCtx, GstBuffer * buf,
    NvDsBatchMeta * batch_meta, guint index)
  guint num_male = 0;  //男人数量
  guint num_female = 0;  //女人数量
  guint num_person = 0;  //检测到的人的数量
  guint num_objects[128];
  GstMapInfo in_map_info;  //存储映射
  NvBufSurface *surface = NULL; //映射数据接口
  memset (num_objects, 0, sizeof (num_objects));
  memset (&in_map_info, 0, sizeof(in_map_info));
  /* 将缓存数据转映射 */
  if (gst_buffer_map (buf, &in_map_info, GST_MAP_READWRITE)){
    surface = (NvBufSurface *) in_map_info.data;
    g_print("surface%d \n" , NvBufSurfaceMap(surface, -1, -1, NVBUF_MAP_READ_WRITE));
    /* 开启GPU向CPU同步模式 */
    NvBufSurfaceSyncForCpu(surface, -1, -1);
    /* 遍历每一帧元数据 */
    for (NvDsMetaList * l_frame = batch_meta->frame_meta_list; l_frame != NULL;
        l_frame = l_frame->next) {
      NvDsFrameMeta *frame_meta = (NvDsFrameMeta*)l_frame->data;
      /* 遍历每一帧中的对象元数据 */
      for (NvDsMetaList * l_obj = frame_meta->obj_meta_list; l_obj != NULL;
          l_obj = l_obj->next) {
        /* 人的总数加1 */
        num_person++;
        NvDsObjectMeta *obj = (NvDsObjectMeta *) l_obj->data;
        /* 如果对象元数据的组件id为配置中的插件id */
        if (obj->unique_component_id ==
            (gint) appCtx->config.primary_gie_config.unique_id) {
          /* 对象元数据的分类id在0和128之间 */
          if (obj->class_id >= 0 && obj->class_id < 128) {
            num_objects[obj->class_id]++;
          /* 如果对象元数据的分类是人 */
          if (appCtx->person_class_id > -1
              && obj->class_id == appCtx->person_class_id) {
            /* 如果对象元数据的显示字符中包含男人,则用男人替换掉人 */
            if (strstr (obj->text_params.display_text, "Man")) {
              str_replace (obj->text_params.display_text, "Man", "");
              str_replace (obj->text_params.display_text, "Person", "Man");
              /* 男人数量加1 */
              num_male++;
            } /* 如果对象元数据的显示字符中包含女人,则用女人替换掉人 */
            else if (strstr (obj->text_params.display_text, "Woman")) {
              str_replace (obj->text_params.display_text, "Woman", "");
              str_replace (obj->text_params.display_text, "Person", "Woman");
              /* 女人数量加1 */
              num_female++;
      /* 如果检测到人 */
      if (num_person > 0) {
         /* 获取帧图像宽 */
         gint frame_width = (gint)surface->surfaceList[frame_meta->batch_id].width;
         /* 获取帧图像高 */
         gint frame_height = (gint)surface->surfaceList[frame_meta->batch_id].height;
         /* 获取整个帧的图像数据 */
         void *frame_data = surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0];
         /* 获取帧的批次 */
         size_t frame_step = surface->surfaceList[frame_meta->batch_id].pitch;
         /* 将图像数据转化为opencv的Mat */
         cv::Mat frame = cv::Mat(frame_height, frame_width, CV_8UC4, frame_data, frame_step);
         g_print("channels%d\n",frame.channels());
         g_print("framerows%d\n",frame.rows);
         g_print("framecols%d\n",frame.cols);
         cv::Mat out_mat = cv::Mat (cv::Size(frame_width, frame_height), CV_8UC3);
         cv::cvtColor(frame, out_mat, CV_RGBA2BGR);
         int w = out_mat.cols;
         int h = out_mat.rows;
         cv::Point p1(100, 100);
         cv::Point p2(300, 150);
         cv::Point p3(300, 350);
         cv::Point p4(250, 450);
         cv::Point p5(50, 450);
         std::vector<cv::Point> pts;
         pts.push_back(p1);
         pts.push_back(p2);
         pts.push_back(p3);
         pts.push_back(p4);
         pts.push_back(p5);
         /* 在图像中画一个多边形 */
         cv::polylines(out_mat, pts, true, cv::Scalar(0,0,255), 2, 8, 0);
         /* 将截取的图像进行保存 */
         cv::imwrite("/home/nano/Documents/test.jpg", out_mat);
  /* 解除映射 */
  NvBufSurfaceUnMap(surface, -1, -1);
              
               由于deepstream本身是由C写的，而使用opencv必须使用C++编译器编译，故修改Makefile文件如下
              
              
               APP:= deepstream-app
TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -)
NVDS_VERSION:=5.0
LIB_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/lib/
APP_INSTALL_DIR?=/opt/nvidia/deepstream/deepstream-$(NVDS_VERSION)/bin/
ifeq ($(TARGET_DEVICE),aarch64)
  CFLAGS:= -DPLATFORM_TEGRA
endif
SRCS:= $(wildcard *.c)
SRCS+= $(wildcard ../../apps-common/src/*.c)
INCS:= $(wildcard *.h)
PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0 opencv4
OBJS:= $(SRCS:.c=.o)
CFLAGS+= -I./ -I../../apps-common/includes -I../../../includes -DDS_VERSION_MINOR=0 -DDS_VERSION_MAJOR=5 -I/usr/local/cuda-10.2/targets/aarch64-linux/include
LIBS+= -L$(LIB_INSTALL_DIR) -lnvdsgst_meta -lnvds_meta -lnvdsgst_helper -lnvdsgst_smartrecord -lnvds_utils -lnvds_msgbroker -lm \
       -lgstrtspserver-1.0 -ldl -lnvbufsurface -Wl,-rpath,$(LIB_INSTALL_DIR)
CFLAGS+= `pkg-config --cflags $(PKGS)`
LIBS+= `pkg-config --libs $(PKGS)`
all: $(APP)
%.o: %.c $(INCS) Makefile
        $(CXX) -c -o $@ $(CFLAGS) $<
$(APP): $(OBJS) Makefile
        $(CXX) -o $(APP) $(OBJS) $(LIBS)
install: $(APP)
        cp -rv $(APP) $(APP_INSTALL_DIR)
clean:
        rm -rf $(OBJS) $(APP)
              
              
               这里需要注意的是，当我们使用C++来编译C代码时会有一系列的报错，总体分为两种错误，一种为缺失强制转换的错误，一种是goto语句后不允许定义变量的错误，根据提示进行一一修正就好了。
              
              
               展开阅读全文