v4l2videodec: Handle drivers that only round up height

Commit 1f31715c9861 ("v4l2videodec: use visible size, not coded size,
for downstream negotiation filter") added support for removing the
padding obtained as the difference between width/height from G_FMT and
visible width/height from G_SELECTION from the probed caps obtained
via TRY_FMT.
This patch fixes the padding removal for drivers that only round up
height, but not width, to the padded frame size. This might happen
because horizontal padding can be handled by line stride (bytesperline),
but there is no such thing as plane stride in the V4L2 API for
single-buffer planar formats.

https://bugzilla.gnome.org/show_bug.cgi?id=791271

1 files changed, 11 insertions, 4 deletions

diff --git a/sys/v4l2/gstv4l2videodec.c b/sys/v4l2/gstv4l2videodec.c
index c23572642..64037cb67 100644
--- a/sys/v4l2/gstv4l2videodec.c
+++ b/sys/v4l2/gstv4l2videodec.c
@@ -495,13 +495,20 @@ gst_v4l2_video_remove_padding (GstCapsFeatures * features,
     return TRUE;
 
   if (align->padding_left != 0 || align->padding_top != 0 ||
-      width != info->width + align->padding_right ||
       height != info->height + align->padding_bottom)
     return TRUE;
 
-  gst_structure_set (structure,
-      "width", G_TYPE_INT, width - align->padding_right,
-      "height", G_TYPE_INT, height - align->padding_bottom, NULL);
+  if (height == info->height + align->padding_bottom) {
+    /* Some drivers may round up width to the padded with */
+    if (width == info->width + align->padding_right)
+      gst_structure_set (structure,
+          "width", G_TYPE_INT, width - align->padding_right,
+          "height", G_TYPE_INT, height - align->padding_bottom, NULL);
+    /* Some drivers may keep visible width and only round up bytesperline */
+    else if (width == info->width)
+      gst_structure_set (structure,
+          "height", G_TYPE_INT, height - align->padding_bottom, NULL);
+  }
 
   return TRUE;
 }