diff options
| author | Gert Wollny <gert.wollny@collabora.com> | 2023-09-08 14:00:16 +0200 |
|---|---|---|
| committer | Marge Bot <emma+marge@anholt.net> | 2023-09-11 08:47:20 +0000 |
| commit | ec2f8bcd16e4f5975ab7c988637113ebc136d4c3 (patch) | |
| tree | 793a08c87f529e2de63c99c7e9f550e450ab05fb | |
| parent | 8019cc5e4d4e5e5c53ad4a6cd927b35ecd3df1f0 (diff) | |
vrend: Use uint64_t to evaluate readback size
This is to avoid integer overflow. Bail out if the size is larger
than UINT_MAX because glReadnPixels can't take larger buffers.
v2: Fix format specifier
Closes: https://gitlab.freedesktop.org/virgl/virglrenderer/-/issues/459
Signed-off-by: Gert Wollny <gert.wollny@collabora.com>
Part-of: <https://gitlab.freedesktop.org/virgl/virglrenderer/-/merge_requests/1225>
| -rw-r--r-- | src/vrend_renderer.c | 16 |
1 files changed, 13 insertions, 3 deletions
diff --git a/src/vrend_renderer.c b/src/vrend_renderer.c index 4f2876d..cc58f7b 100644 --- a/src/vrend_renderer.c +++ b/src/vrend_renderer.c @@ -9310,7 +9310,7 @@ static int vrend_transfer_send_readpixels(struct vrend_context *ctx, bool actually_invert, separate_invert = false; GLenum format, type; GLint y1; - uint32_t send_size = 0; + uint64_t send_size = 0; uint32_t h = u_minify(res->base.height0, info->level); int elsize = util_format_get_blocksize(res->base.format); float depth_scale; @@ -9341,10 +9341,20 @@ static int vrend_transfer_send_readpixels(struct vrend_context *ctx, need_temp = true; if (need_temp) { - send_size = util_format_get_nblocks(res->base.format, info->box->width, info->box->height) * info->box->depth * util_format_get_blocksize(res->base.format); + send_size = util_format_get_nblocks(res->base.format, info->box->width, info->box->height); + send_size *= info->box->depth; + send_size *= util_format_get_blocksize(res->base.format); + + /* glReadnPixels only supports a buffer size of GLsizei = uint32_t, anything larger + * is bogous */ + if (send_size > UINT_MAX) { + virgl_error("Readback size out of range %ld\n", send_size); + return EINVAL; + } + data = malloc(send_size); if (!data) { - virgl_error("Memory allocation failed for %d\n", send_size); + virgl_error("Memory allocation failed for %ld\n", send_size); return ENOMEM; } } else { |