video-scaler: make sure to clamp to max width

When estimating the area that should first be vertically scaled, make
sure we clamp to the max input size or else we get invalid reads.

1 files changed, 11 insertions, 4 deletions

diff --git a/gst-libs/gst/video/video-scaler.c b/gst-libs/gst/video/video-scaler.c
index 881fd0892..3d47da5c0 100644
--- a/gst-libs/gst/video/video-scaler.c
+++ b/gst-libs/gst/video/video-scaler.c
@@ -1137,6 +1137,7 @@ gst_video_scaler_combine_packed_YUV (GstVideoScaler * y_scale,
 
   scale->in_y_offset = get_y_offset (in_format);
   scale->out_y_offset = get_y_offset (out_format);
+  scale->inc = y_scale->inc;
 
   for (i = 0; i < out_size; i++) {
     if ((i & 1) == scale->out_y_offset) {
@@ -1498,24 +1499,30 @@ gst_video_scaler_2d (GstVideoScaler * hscale, GstVideoScaler * vscale,
           vfunc (vscale, lines, LINE (dest, dest_stride, i), i, width, n_elems);
         }
       } else {
-        guint vx, vw, w1;
+        guint vx, vw, w1, ws;
         guint h_taps;
 
         h_taps = hscale->resampler.max_taps;
         w1 = x + width - 1;
+        ws = hscale->resampler.offset[w1];
 
+        /* we need to estimate the area that we first need to scale in the
+         * vertical direction. Scale x and width to find the lower bound and
+         * overshoot the width to find the upper bound */
         vx = (hscale->inc * x) >> 16;
         vx = MIN (vx, hscale->resampler.offset[x]);
         vw = (hscale->inc * (x + width)) >> 16;
         if (hscale->merged) {
           if ((w1 & 1) == hscale->out_y_offset)
-            vw = MAX (vw, hscale->resampler.offset[w1] + (2 * h_taps));
+            vw = MAX (vw, ws + (2 * h_taps));
           else
-            vw = MAX (vw, hscale->resampler.offset[w1] + (4 * h_taps));
+            vw = MAX (vw, ws + (4 * h_taps));
         } else {
-          vw = MAX (vw, hscale->resampler.offset[w1] + h_taps);
+          vw = MAX (vw, ws + h_taps);
         }
         vw += 1;
+        /* but clamp to max size */
+        vw = MIN (vw, hscale->resampler.in_size);
 
         if (vscale->tmpwidth < vw)
           realloc_tmplines (vscale, n_elems, vw);