diff options
| author | Filip Gawin <filip.gawin@collabora.com> | 2023-09-15 15:16:49 +0200 |
|---|---|---|
| committer | Marge Bot <emma+marge@anholt.net> | 2023-09-18 07:52:17 +0000 |
| commit | 4da1859f07b8a47b12035d6d284ee7bae379be9c (patch) | |
| tree | 7c9afbe24ec8a189648a99267cfd31ea0be3171c | |
| parent | 6f6a81dce68d6a308ee408db57997fa5d31c34d1 (diff) | |
vrend: Use uint64_t to evaluate readback size
in vrend_renderer_transfer_write_iov
Part-of: <https://gitlab.freedesktop.org/virgl/virglrenderer/-/merge_requests/1238>
| -rw-r--r-- | src/vrend_renderer.c | 18 |
1 files changed, 15 insertions, 3 deletions
diff --git a/src/vrend_renderer.c b/src/vrend_renderer.c index 4abc3cf..f8ded20 100644 --- a/src/vrend_renderer.c +++ b/src/vrend_renderer.c @@ -8952,7 +8952,7 @@ static int vrend_renderer_transfer_write_iov(struct vrend_context *ctx, bool compressed; bool invert = false; float depth_scale; - GLuint send_size = 0; + uint64_t send_size = 0; uint32_t stride = info->stride; uint32_t layer_stride = info->layer_stride; @@ -8981,7 +8981,9 @@ static int vrend_renderer_transfer_write_iov(struct vrend_context *ctx, } send_size = util_format_get_nblocks(res->base.format, info->box->width, - info->box->height) * elsize; + info->box->height); + send_size *= elsize; + if (res->target == GL_TEXTURE_3D || res->target == GL_TEXTURE_1D_ARRAY || res->target == GL_TEXTURE_2D_ARRAY || @@ -8992,9 +8994,19 @@ static int vrend_renderer_transfer_write_iov(struct vrend_context *ctx, return EINVAL; if (need_temp) { + /* functions like glCompressedTexSubImage3D only support + * a buffer size of GLsizei = uint32_t, anything larger + * is bogous */ + if (send_size > UINT_MAX) { + virgl_error("Used write size out of range %"PRIu64"\n", send_size); + return EINVAL; + } + data = malloc(send_size); - if (!data) + if (!data) { + virgl_error("Memory allocation failed for %"PRIu64"\n", send_size); return ENOMEM; + } read_transfer_data(iov, num_iovs, data, res->base.format, info->offset, stride, layer_stride, info->box, invert); } else { |