path: root/recipes-multimedia/gstreamer/gstreamer1.0-plugins-bad/0024-glupload-add-ion-dmabuf-support-in-glupload.patch

From 21b21597d3e2bccd2bf04ec6dd7c89bfe1a88ec5 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Wed, 17 May 2017 14:55:33 +0800
Subject: [PATCH] glupload: add ion dmabuf support in glupload

1. Support one texture for YUV format in dmabuf uploader
2. Propose ion dma-fd buffer pool to upstream to avoid memory copy
3. If upstream don't chose the proposed buffer pool, then create
   our own and do copy to avoid memory copy from CPU to GPU side
4. Add buffer alignmentw

Upstream-Status: Inappropriate [i.MX specific]

Signed-off-by: Haihua Hu <jared.hu@nxp.com>
---
 configure.ac                      |   9 ++
 gst-libs/gst/gl/Makefile.am       |   6 +-
 gst-libs/gst/gl/egl/gsteglimage.c |  77 +++++++++++++++
 gst-libs/gst/gl/egl/gsteglimage.h |   5 +
 gst-libs/gst/gl/gstglupload.c     | 194 +++++++++++++++++++++++++++++++++++++-
 5 files changed, 285 insertions(+), 6 deletions(-)

diff --git a/configure.ac b/configure.ac
index 3f6f100..d17bb64 100644
--- a/configure.ac
+++ b/configure.ac
@@ -857,10 +857,16 @@ PKG_CHECK_MODULES(GST_ALLOCATORS, gstreamer-allocators-1.0,
   HAVE_GST_ALLOCATORS=yes, )
 
 GST_GL_HAVE_DMABUF=0
+GST_GL_HAVE_IONDMA=0
+HAVE_IONDMA=no
 if test "x$HAVE_DRM_FOURCC_HEADER" = "xyes" -a \
         "x$HAVE_GST_ALLOCATORS" = "xyes" -a \
         "x$HAVE_EGL" = "xyes"; then
           GST_GL_HAVE_DMABUF=1
+  if test "x$HAVE_ION" = "xyes"; then
+    HAVE_IONDMA=yes
+    GST_GL_HAVE_IONDMA=1
+  fi
 fi
 
 dnl check for Vivante DirectVIV support
@@ -1301,6 +1307,7 @@ GL_CONFIG_DEFINES="$GL_CONFIG_DEFINES
 
 GL_CONFIG_DEFINES="$GL_CONFIG_DEFINES
 #define GST_GL_HAVE_DMABUF $GST_GL_HAVE_DMABUF
+#define GST_GL_HAVE_IONDMA $GST_GL_HAVE_IONDMA
 #define GST_GL_HAVE_VIV_DIRECTVIV $GST_GL_HAVE_VIV_DIRECTVIV
 #define GST_GL_HAVE_PHYMEM $GST_GL_HAVE_PHYMEM
 "
@@ -1339,6 +1346,7 @@ if test "x$GL_APIS" = "x" -o "x$GL_PLATFORMS" = "x" -o "x$GL_WINDOWS" = "x"; the
   HAVE_WINDOW_EAGL=no
   HAVE_WINDOW_VIV_FB=no
   HAVE_G2D=no
+  HAVE_IONDMA=no
 fi
 
 AC_SUBST(GL_APIS)
@@ -1359,6 +1367,7 @@ AM_CONDITIONAL(HAVE_WINDOW_ANDROID, test "x$HAVE_WINDOW_ANDROID" = "xyes")
 AM_CONDITIONAL(HAVE_WINDOW_EAGL, test "x$HAVE_WINDOW_EAGL" = "xyes")
 AM_CONDITIONAL(HAVE_WINDOW_VIV_FB, test "x$HAVE_WINDOW_VIV_FB" = "xyes")
 AM_CONDITIONAL(HAVE_GL_PHYMEM, test "x$HAVE_G2D" = "xyes" -a "x$HAVE_VIV_DIRECTVIV" = "xyes")
+AM_CONDITIONAL(HAVE_IONDMA, test "x$HAVE_IONDMA" = "xyes")
 
 AM_CONDITIONAL(USE_OPENGL, test "x$USE_OPENGL" = "xyes")
 AM_CONDITIONAL(USE_GLES2, test "x$USE_GLES2" = "xyes")
diff --git a/gst-libs/gst/gl/Makefile.am b/gst-libs/gst/gl/Makefile.am
index e97a386..03ce1b0 100644
--- a/gst-libs/gst/gl/Makefile.am
+++ b/gst-libs/gst/gl/Makefile.am
@@ -177,9 +177,9 @@ GstGL-@GST_API_VERSION@.gir: $(INTROSPECTION_SCANNER) libgstgl-@GST_API_VERSION@
 		-I$(top_builddir)/gst-libs \
 		$(GST_PLUGINS_BASE_CFLAGS) \
 		$(GL_CFLAGS) \
-		--add-include-path=`PKG_CONFIG_PATH="$(GST_PKG_CONFIG_PATH)" $(PKG_CONFIG) --variable=girdir gstreamer-@GST_API_VERSION@` \
-		--add-include-path=`PKG_CONFIG_PATH="$(GST_PKG_CONFIG_PATH)" $(PKG_CONFIG) --variable=girdir gstreamer-base-@GST_API_VERSION@` \
-		--add-include-path=`PKG_CONFIG_PATH="$(GST_PKG_CONFIG_PATH)" $(PKG_CONFIG) --variable=girdir gstreamer-video-@GST_API_VERSION@` \
+		--add-include-path=$(PKG_CONFIG_SYSROOT_DIR)`PKG_CONFIG_PATH="$(GST_PKG_CONFIG_PATH)" $(PKG_CONFIG) --variable=girdir gstreamer-@GST_API_VERSION@` \
+		--add-include-path=$(PKG_CONFIG_SYSROOT_DIR)`PKG_CONFIG_PATH="$(GST_PKG_CONFIG_PATH)" $(PKG_CONFIG) --variable=girdir gstreamer-base-@GST_API_VERSION@` \
+		--add-include-path=$(PKG_CONFIG_SYSROOT_DIR)`PKG_CONFIG_PATH="$(GST_PKG_CONFIG_PATH)" $(PKG_CONFIG) --variable=girdir gstreamer-video-@GST_API_VERSION@` \
 		--library=libgstgl-@GST_API_VERSION@.la \
 		--include=Gst-@GST_API_VERSION@ \
 		--include=GstBase-@GST_API_VERSION@ \
diff --git a/gst-libs/gst/gl/egl/gsteglimage.c b/gst-libs/gst/gl/egl/gsteglimage.c
index f570718..75ed5cf 100644
--- a/gst-libs/gst/gl/egl/gsteglimage.c
+++ b/gst-libs/gst/gl/egl/gsteglimage.c
@@ -462,4 +462,81 @@ gst_egl_image_from_dmabuf (GstGLContext * context,
   return gst_egl_image_new_wrapped (context, img, format, NULL,
       (GstEGLImageDestroyNotify) _destroy_egl_image);
 }
+
+GstEGLImage *
+gst_egl_image_from_dmabuf_singleplaner (GstGLContext * context,
+    GstMemory ** mems, GstVideoInfo * in_info, gint n_planes, gsize * offset)
+{
+  gint fourcc;
+  gint atti = 0;
+  guint i;
+  guintptr attribs[25];
+  guintptr dmafd_flags[] = {
+    EGL_DMA_BUF_PLANE0_FD_EXT,
+    EGL_DMA_BUF_PLANE1_FD_EXT,
+    EGL_DMA_BUF_PLANE2_FD_EXT
+  };
+  guintptr offset_flags[] = {
+    EGL_DMA_BUF_PLANE0_OFFSET_EXT,
+    EGL_DMA_BUF_PLANE1_OFFSET_EXT,
+    EGL_DMA_BUF_PLANE2_OFFSET_EXT
+  };
+  guintptr pitch_flags[] = {
+    EGL_DMA_BUF_PLANE0_PITCH_EXT,
+    EGL_DMA_BUF_PLANE1_PITCH_EXT,
+    EGL_DMA_BUF_PLANE2_PITCH_EXT
+  };
+  EGLImageKHR img = EGL_NO_IMAGE_KHR;
+
+  fourcc = _drm_fourcc_from_info (in_info, 0);
+
+  if(GST_VIDEO_INFO_IS_YUV(in_info)) {
+    fourcc = gst_video_format_to_fourcc (GST_VIDEO_INFO_FORMAT(in_info));
+
+    /* gstreamer fourcc is not compatible with DRM FOURCC*/
+    if(GST_VIDEO_INFO_FORMAT(in_info) == GST_VIDEO_FORMAT_I420)
+      fourcc = DRM_FORMAT_YUV420;
+  }
+
+  GST_DEBUG ("fourcc %.4s (%d) n_planes %d (%dx%d)",
+      (char *) &fourcc, fourcc, n_planes,
+      GST_VIDEO_INFO_COMP_WIDTH (in_info, 0),
+      GST_VIDEO_INFO_COMP_HEIGHT (in_info, 0));
+
+  attribs[atti++] = EGL_WIDTH;
+  attribs[atti++] = GST_VIDEO_INFO_WIDTH (in_info);
+  attribs[atti++] = EGL_HEIGHT;
+  attribs[atti++] = GST_VIDEO_INFO_HEIGHT (in_info);
+
+  attribs[atti++] = EGL_LINUX_DRM_FOURCC_EXT;
+  attribs[atti++] = fourcc;
+
+  for (i = 0; i < n_planes; i++) {
+    attribs[atti++] = dmafd_flags[i];
+    attribs[atti++] = gst_dmabuf_memory_get_fd (mems[i]);
+    attribs[atti++] = offset_flags[i];
+    attribs[atti++] = offset[i];
+    attribs[atti++] = pitch_flags[i];
+    attribs[atti++] = GST_VIDEO_INFO_PLANE_STRIDE (in_info, i);
+  }
+
+  attribs[atti] = EGL_NONE;
+
+  for (int i = 0; i < atti; i++)
+    GST_LOG ("attr %i: %" G_GINTPTR_FORMAT, i, attribs[i]);
+
+  g_assert (atti <= 25);
+
+  img = _gst_egl_image_create (context, EGL_LINUX_DMA_BUF_EXT, NULL, attribs);
+
+  if (!img) {
+    GST_WARNING ("eglCreateImage failed: %s",
+        gst_egl_get_error_string (eglGetError ()));
+    return NULL;
+  }
+
+  /* one texture for YUV format is treat as RGBA texture in imx GPU */
+  return gst_egl_image_new_wrapped (context, img, GST_GL_RGBA,
+      NULL, (GstEGLImageDestroyNotify) _destroy_egl_image);
+}
 #endif /* GST_GL_HAVE_DMABUF */
diff --git a/gst-libs/gst/gl/egl/gsteglimage.h b/gst-libs/gst/gl/egl/gsteglimage.h
index 56ec7e1..5595ba8 100644
--- a/gst-libs/gst/gl/egl/gsteglimage.h
+++ b/gst-libs/gst/gl/egl/gsteglimage.h
@@ -78,6 +78,11 @@ GstEGLImage *           gst_egl_image_from_dmabuf               (GstGLContext *
                                                                  GstVideoInfo * in_info,
                                                                  gint plane,
                                                                  gsize offset);
+GstEGLImage *           gst_egl_image_from_dmabuf_singleplaner  (GstGLContext * context,
+                                                                 GstMemory ** mems,
+                                                                 GstVideoInfo * in_info,
+                                                                 gint n_planes,
+                                                                 gsize * offset);
 #endif
 
 /**
diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index 496ca86..6011695 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -41,6 +41,10 @@
 #include <gst/allocators/gstphysmemory.h>
 #endif
 
+#if GST_GL_HAVE_IONDMA
+#include <gst/allocators/gstionmemory.h>
+#endif
+
 /**
  * SECTION:gstglupload
  * @title: GstGLUpload
@@ -58,6 +62,8 @@
 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
 
+#define DEFAULT_ALIGN 16
+
 GST_DEBUG_CATEGORY_STATIC (gst_gl_upload_debug);
 #define GST_CAT_DEFAULT gst_gl_upload_debug
 
@@ -483,7 +489,9 @@ struct DmabufUpload
   GstGLUpload *upload;
 
   GstEGLImage *eglimage[GST_VIDEO_MAX_PLANES];
+  GstBuffer *inbuf;
   GstBuffer *outbuf;
+  GstBufferPool *pool;
   GstGLVideoAllocationParams *params;
 };
 
@@ -514,6 +522,7 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
         _set_caps_features_with_passthrough (caps,
         GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
 
+    gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
     tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D);
     gst_caps_unref (ret);
     ret = tmp;
@@ -569,6 +578,72 @@ _set_cached_eglimage (GstMemory * mem, GstEGLImage * eglimage, gint plane)
 }
 
 static gboolean
+_dma_buf_upload_setup_buffer_pool (GstBufferPool **pool, GstAllocator *allocator,
+    GstCaps *caps, GstVideoInfo *info)
+{
+  GstAllocationParams params;
+  GstStructure *config;
+  gsize size;
+  guint width, height;
+  GstVideoAlignment alignment;
+ 
+  g_return_val_if_fail (caps != NULL && info != NULL, FALSE);
+
+  width = GST_VIDEO_INFO_WIDTH (info);
+  height = GST_VIDEO_INFO_HEIGHT (info);
+
+  gst_allocation_params_init (&params);
+
+  /* if user not provide an allocator, then use default ion allocator*/
+  if (!allocator) {
+#if GST_GL_HAVE_IONDMA
+    allocator = gst_ion_allocator_obtain ();
+#endif
+  }
+
+  if (!allocator) {
+    GST_WARNING ("Cannot get available allocator");
+    return FALSE;
+  }
+  GST_DEBUG ("got allocator(%p).", allocator);
+
+  if (*pool)
+    gst_object_unref(*pool);
+
+  *pool = gst_video_buffer_pool_new ();
+  if (!*pool) {
+    GST_WARNING ("New video buffer pool failed.");
+    return FALSE;
+  }
+  GST_DEBUG ("create buffer pool(%p).", *pool);
+
+  config = gst_buffer_pool_get_config (*pool);
+
+  /* configure alignment for eglimage to import this dma-fd buffer */
+  memset (&alignment, 0, sizeof (GstVideoAlignment));
+  alignment.padding_right = GST_ROUND_UP_N(width, DEFAULT_ALIGN) - width;
+  alignment.padding_bottom = GST_ROUND_UP_N(height, DEFAULT_ALIGN) - height;
+  GST_DEBUG ("align buffer pool, w(%d) h(%d), padding_right (%d), padding_bottom (%d)",
+      width, height, alignment.padding_right, alignment.padding_bottom);
+
+  /* the normal size of a frame */
+  size = info->size;
+  gst_buffer_pool_config_set_params (config, caps, size, 0, 30);
+  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
+  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
+  gst_buffer_pool_config_set_video_alignment (config, &alignment);
+  gst_buffer_pool_config_set_allocator (config, allocator, &params);
+
+  if (!gst_buffer_pool_set_config (*pool, config)) {
+    GST_WARNING ("buffer pool config failed.");
+    gst_object_unref (*pool);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static gboolean
 _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
     GstCaps * out_caps)
 {
@@ -595,8 +670,52 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
     return FALSE;
 
   /* This will eliminate most non-dmabuf out there */
-  if (!gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, 0)))
-    return FALSE;
+  if (!gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, 0))) {
+    GstVideoFrame frame1, frame2;
+
+    gst_video_frame_map (&frame1, in_info, buffer, GST_MAP_READ);
+
+    if (!dmabuf->pool) {
+      gboolean ret;
+      GstCaps *new_caps = gst_video_info_to_caps(&frame1.info);
+      gst_video_info_from_caps(in_info, new_caps);
+
+      ret = _dma_buf_upload_setup_buffer_pool (&dmabuf->pool, NULL, new_caps, in_info);
+      if (!ret) {
+        gst_video_frame_unmap (&frame1);
+        gst_caps_unref (new_caps);
+        GST_WARNING_OBJECT (dmabuf->upload, "no available buffer pool");
+        return FALSE;
+      }
+    }
+
+    if (!gst_buffer_pool_is_active (dmabuf->pool)
+        && gst_buffer_pool_set_active (dmabuf->pool, TRUE) != TRUE) {
+      gst_video_frame_unmap (&frame1);
+      GST_WARNING_OBJECT (dmabuf->upload, "buffer pool is not ok");
+      return FALSE;
+    }
+
+    if (dmabuf->inbuf)
+      gst_buffer_unref(dmabuf->inbuf);
+    dmabuf->inbuf = NULL;
+
+    gst_buffer_pool_acquire_buffer (dmabuf->pool, &dmabuf->inbuf, NULL);
+    if (!dmabuf->inbuf) {
+      gst_video_frame_unmap (&frame1);
+      GST_WARNING_OBJECT (dmabuf->upload, "acquire_buffer failed");
+      return FALSE;
+    }
+
+    GST_DEBUG_OBJECT (dmabuf->upload, "copy plane resolution (%d)x(%d)\n", in_info->width, in_info->height);
+    gst_video_frame_map (&frame2, in_info, dmabuf->inbuf, GST_MAP_WRITE);
+    gst_video_frame_copy (&frame2, &frame1);
+    gst_video_frame_unmap (&frame1);
+    gst_video_frame_unmap (&frame2);
+
+    buffer = dmabuf->inbuf;
+    meta = gst_buffer_get_video_meta (buffer);
+  }
 
   /* We cannot have multiple dmabuf per plane */
   if (n_mem > n_planes)
@@ -617,7 +736,7 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
     gst_gl_allocation_params_free ((GstGLAllocationParams *) dmabuf->params);
   if (!(dmabuf->params =
           gst_gl_video_allocation_params_new_wrapped_gl_handle (dmabuf->
-              upload->context, NULL, &dmabuf->upload->priv->in_info, -1, NULL,
+              upload->context, NULL, &dmabuf->upload->priv->out_info, -1, NULL,
               GST_GL_TEXTURE_TARGET_2D, 0, NULL, NULL, NULL)))
     return FALSE;
 
@@ -643,6 +762,21 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
       return FALSE;
   }
 
+#ifdef EGL_DMA_BUF_PLANE1_FD_EXT
+  /* Now create one single EGLImage */
+  /* check if one is cached */
+  dmabuf->eglimage[0] = _get_cached_eglimage (mems[0], 0);
+  if (dmabuf->eglimage[0])
+    return TRUE;
+
+  dmabuf->eglimage[0] =
+      gst_egl_image_from_dmabuf_singleplaner (dmabuf->upload->context,
+      mems, in_info, n_planes, mems_skip);
+  if (!dmabuf->eglimage[0])
+    return FALSE;
+
+  _set_cached_eglimage (mems[0], dmabuf->eglimage[0], 0);
+#else
   /* Now create an EGLImage for each dmabufs */
   for (i = 0; i < n_planes; i++) {
     /* check if one is cached */
@@ -661,6 +795,7 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
 
     _set_cached_eglimage (mems[i], dmabuf->eglimage[i], i);
   }
+#endif
 
   return TRUE;
 }
@@ -670,6 +805,47 @@ _dma_buf_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
     GstQuery * query)
 {
   /* nothing to do for now. */
+  struct DmabufUpload *upload = impl;
+  GstBufferPool *pool = NULL;
+  GstAllocator *allocator = NULL;
+  GstCaps *caps;
+  GstVideoInfo info;
+
+  gst_query_parse_allocation (query, &caps, NULL);
+
+  if (!gst_video_info_from_caps (&info, caps))
+    goto invalid_caps;
+
+#if GST_GL_HAVE_IONDMA
+  allocator = gst_ion_allocator_obtain ();
+#endif
+  if (!allocator) {
+    GST_WARNING ("New ion allocator failed.");
+    return;
+  }
+  GST_DEBUG ("create ion allocator(%p).", allocator);
+
+  gst_query_add_allocation_param (query, allocator, NULL);
+
+  if (!_dma_buf_upload_setup_buffer_pool (&pool, allocator, caps, &info))
+    goto setup_failed;
+
+  gst_query_set_nth_allocation_pool (query, 0, pool, info.size, 1, 30);
+
+  if (pool)
+    gst_object_unref (pool);
+
+  return;
+invalid_caps:
+  {
+    GST_WARNING_OBJECT (upload->upload, "invalid caps specified");
+    return;
+  }
+setup_failed:
+  {
+    GST_WARNING_OBJECT (upload->upload, "failed to setup buffer pool");
+    return;
+  }
 }
 
 static void
@@ -677,11 +853,17 @@ _dma_buf_upload_perform_gl_thread (GstGLContext * context,
     struct DmabufUpload *dmabuf)
 {
   GstGLMemoryAllocator *allocator;
+  guint n_mem, i;
 
   allocator =
       GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
       (GST_GL_MEMORY_EGL_ALLOCATOR_NAME));
 
+  n_mem = GST_VIDEO_INFO_N_PLANES (dmabuf->params->v_info);
+  for (i = 0; i < n_mem; i++) {
+    if(!dmabuf->eglimage[i])
+      return;
+  }
   /* FIXME: buffer pool */
   dmabuf->outbuf = gst_buffer_new ();
   gst_gl_memory_setup_buffer (allocator, dmabuf->outbuf, dmabuf->params, NULL,
@@ -716,6 +898,12 @@ _dma_buf_upload_free (gpointer impl)
   if (dmabuf->params)
     gst_gl_allocation_params_free ((GstGLAllocationParams *) dmabuf->params);
 
+  if (dmabuf->inbuf)
+    gst_buffer_unref (dmabuf->inbuf);
+
+  if (dmabuf->pool)
+    gst_object_unref(dmabuf->pool);
+
   g_free (impl);
 }
 
-- 
2.7.4