bytereader: fix peek value when scanning for 00 00 01 with non-0 offset

We would add the offset a second time in _scan_for_start_code()
when we found a result, but it's already been added to the data
pointer at the beginning of _masked_scan_uint32_peek(), so the
peeked value would be wrong if the initial offset was >0, and
we would potentially read memory out-of-bounds.

Add unit test for all of this.

https://bugzilla.gnome.org/show_bug.cgi?id=778365

1 files changed, 10 insertions, 5 deletions

diff --git a/libs/gst/base/gstbytereader.c b/libs/gst/base/gstbytereader.c
index baa152fd1..0045ce93e 100644
--- a/libs/gst/base/gstbytereader.c
+++ b/libs/gst/base/gstbytereader.c
@@ -822,7 +822,7 @@ gst_byte_reader_dup_data (GstByteReader * reader, guint size, guint8 ** val)
 
 /* Special optimized scan for mask 0xffffff00 and pattern 0x00000100 */
 static inline gint
-_scan_for_start_code (const guint8 * data, guint offset, guint size)
+_scan_for_start_code (const guint8 * data, guint size)
 {
   guint8 *pdata = (guint8 *) data;
   guint8 *pend = (guint8 *) (data + size - 4);
@@ -835,7 +835,7 @@ _scan_for_start_code (const guint8 * data, guint offset, guint size)
     } else if (pdata[0] || pdata[2] != 1) {
       pdata++;
     } else {
-      return (pdata - data + offset);
+      return (pdata - data);
     }
   }
 
@@ -863,10 +863,15 @@ _masked_scan_uint32_peek (const GstByteReader * reader,
 
   /* Handle special case found in MPEG and H264 */
   if ((pattern == 0x00000100) && (mask == 0xffffff00)) {
-    guint ret = _scan_for_start_code (data, offset, size);
-    if (G_UNLIKELY (value))
+    gint ret = _scan_for_start_code (data, size);
+
+    if (ret == -1)
+      return ret;
+
+    if (value != NULL)
       *value = (1 << 8) | data[ret + 3];
-    return ret;
+
+    return ret + offset;
   }
 
   /* set the state to something that does not match */