diff options
author | Mike Blumenkrantz <michael.blumenkrantz@gmail.com> | 2022-10-26 12:08:08 -0400 |
---|---|---|
committer | Mike Blumenkrantz <michael.blumenkrantz@gmail.com> | 2023-01-03 09:20:45 -0500 |
commit | 7ab5c5d36d2b988470ba85df9ebc7310b986147b (patch) | |
tree | ac427cca08aedf8686eb559b75d8cbe52ea29b7c /src | |
parent | 20e1474c2cc642061567fdc79ff83f4740b20ea2 (diff) |
zink: use EXT_descriptor_buffer with ZINK_DESCRIPTORS=db
this should be bug-free, as it passes cts/piglit/gaming on multiple drivers,
but since it's new, it stays behind an env var for at least one release
Acked-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20489>
Diffstat (limited to 'src')
-rw-r--r-- | src/gallium/drivers/zink/zink_batch.c | 13 | ||||
-rw-r--r-- | src/gallium/drivers/zink/zink_context.c | 150 | ||||
-rw-r--r-- | src/gallium/drivers/zink/zink_descriptors.c | 349 | ||||
-rw-r--r-- | src/gallium/drivers/zink/zink_device_info.py | 1 | ||||
-rw-r--r-- | src/gallium/drivers/zink/zink_resource.c | 36 | ||||
-rw-r--r-- | src/gallium/drivers/zink/zink_resource.h | 3 | ||||
-rw-r--r-- | src/gallium/drivers/zink/zink_screen.c | 46 | ||||
-rw-r--r-- | src/gallium/drivers/zink/zink_types.h | 62 |
8 files changed, 555 insertions, 105 deletions
diff --git a/src/gallium/drivers/zink/zink_batch.c b/src/gallium/drivers/zink/zink_batch.c index 7063463404d..9f2be7c1e17 100644 --- a/src/gallium/drivers/zink/zink_batch.c +++ b/src/gallium/drivers/zink/zink_batch.c @@ -414,6 +414,7 @@ zink_reset_batch(struct zink_context *ctx, struct zink_batch *batch) void zink_start_batch(struct zink_context *ctx, struct zink_batch *batch) { + struct zink_screen *screen = zink_screen(ctx->base.screen); zink_reset_batch(ctx, batch); batch->state->usage.unflushed = true; @@ -438,6 +439,18 @@ zink_start_batch(struct zink_context *ctx, struct zink_batch *batch) if (!ctx->queries_disabled) zink_resume_queries(ctx, batch); + + /* descriptor buffers must always be bound at the start of a batch */ + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB && !(ctx->flags & ZINK_CONTEXT_COPY_ONLY)) { + unsigned count = screen->compact_descriptors ? 3 : 5; + VkDescriptorBufferBindingInfoEXT infos[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES] = {0}; + for (unsigned i = 0; i < count; i++) { + infos[i].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_BUFFER_BINDING_INFO_EXT; + infos[i].address = batch->state->dd.db[i]->obj->bda; + infos[i].usage = batch->state->dd.db[i]->obj->vkusage; + } + VKSCR(CmdBindDescriptorBuffersEXT)(batch->state->cmdbuf, count, infos); + } } /* common operations to run post submit; split out for clarity */ diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index f382865c7bd..70c0e0d22ca 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -562,15 +562,24 @@ update_descriptor_state_ubo(struct zink_context *ctx, gl_shader_stage shader, un bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor; const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_UBO; ctx->di.descriptor_res[type][shader][slot] = res; - ctx->di.t.ubos[shader][slot].offset = ctx->ubos[shader][slot].buffer_offset; - if (res) { - ctx->di.t.ubos[shader][slot].buffer = res->obj->buffer; - ctx->di.t.ubos[shader][slot].range = ctx->ubos[shader][slot].buffer_size; - assert(ctx->di.t.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange); + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + if (res) + ctx->di.db.ubos[shader][slot].address = res->obj->bda + ctx->ubos[shader][slot].buffer_offset; + else + ctx->di.db.ubos[shader][slot].address = 0; + ctx->di.db.ubos[shader][slot].range = ctx->ubos[shader][slot].buffer_size; + assert(ctx->di.db.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange); } else { - VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer; - ctx->di.t.ubos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer; - ctx->di.t.ubos[shader][slot].range = VK_WHOLE_SIZE; + ctx->di.t.ubos[shader][slot].offset = ctx->ubos[shader][slot].buffer_offset; + if (res) { + ctx->di.t.ubos[shader][slot].buffer = res->obj->buffer; + ctx->di.t.ubos[shader][slot].range = ctx->ubos[shader][slot].buffer_size; + assert(ctx->di.t.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange); + } else { + VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer; + ctx->di.t.ubos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer; + ctx->di.t.ubos[shader][slot].range = VK_WHOLE_SIZE; + } } if (!slot) { if (res) @@ -588,14 +597,22 @@ update_descriptor_state_ssbo(struct zink_context *ctx, gl_shader_stage shader, u bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor; const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SSBO; ctx->di.descriptor_res[type][shader][slot] = res; - ctx->di.t.ssbos[shader][slot].offset = ctx->ssbos[shader][slot].buffer_offset; - if (res) { - ctx->di.t.ssbos[shader][slot].buffer = res->obj->buffer; - ctx->di.t.ssbos[shader][slot].range = ctx->ssbos[shader][slot].buffer_size; + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + if (res) + ctx->di.db.ssbos[shader][slot].address = res->obj->bda + ctx->ssbos[shader][slot].buffer_offset; + else + ctx->di.db.ssbos[shader][slot].address = 0; + ctx->di.db.ssbos[shader][slot].range = ctx->ssbos[shader][slot].buffer_size; } else { - VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer; - ctx->di.t.ssbos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer; - ctx->di.t.ssbos[shader][slot].range = VK_WHOLE_SIZE; + ctx->di.t.ssbos[shader][slot].offset = ctx->ssbos[shader][slot].buffer_offset; + if (res) { + ctx->di.t.ssbos[shader][slot].buffer = res->obj->buffer; + ctx->di.t.ssbos[shader][slot].range = ctx->ssbos[shader][slot].buffer_size; + } else { + VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer; + ctx->di.t.ssbos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer; + ctx->di.t.ssbos[shader][slot].range = VK_WHOLE_SIZE; + } } return res; } @@ -609,8 +626,14 @@ update_descriptor_state_sampler(struct zink_context *ctx, gl_shader_stage shader ctx->di.descriptor_res[type][shader][slot] = res; if (res) { if (res->obj->is_buffer) { - struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot); - ctx->di.t.tbos[shader][slot] = bv->buffer_view; + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + ctx->di.db.tbos[shader][slot].address = res->obj->bda + ctx->sampler_views[shader][slot]->u.buf.offset; + ctx->di.db.tbos[shader][slot].range = ctx->sampler_views[shader][slot]->u.buf.size; + ctx->di.db.tbos[shader][slot].format = zink_get_format(screen, ctx->sampler_views[shader][slot]->format); + } else { + struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot); + ctx->di.t.tbos[shader][slot] = bv->buffer_view; + } } else { struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot); ctx->di.textures[shader][slot].imageLayout = get_layout_for_binding(ctx, res, type, shader == MESA_SHADER_COMPUTE); @@ -632,8 +655,13 @@ update_descriptor_state_sampler(struct zink_context *ctx, gl_shader_stage shader if (likely(have_null_descriptors)) { ctx->di.textures[shader][slot].imageView = VK_NULL_HANDLE; ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_UNDEFINED; - ctx->di.t.tbos[shader][slot] = VK_NULL_HANDLE; + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + ctx->di.db.tbos[shader][slot].address = 0; + } else { + ctx->di.t.tbos[shader][slot] = VK_NULL_HANDLE; + } } else { + assert(zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB); struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0); struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview; ctx->di.textures[shader][slot].imageView = null_surface->image_view; @@ -653,8 +681,14 @@ update_descriptor_state_image(struct zink_context *ctx, gl_shader_stage shader, ctx->di.descriptor_res[type][shader][slot] = res; if (res) { if (res->obj->is_buffer) { - struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot); - ctx->di.t.texel_images[shader][slot] = bv->buffer_view; + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + ctx->di.db.texel_images[shader][slot].address = res->obj->bda + ctx->image_views[shader][slot].base.u.buf.offset; + ctx->di.db.texel_images[shader][slot].range = ctx->image_views[shader][slot].base.u.buf.size; + ctx->di.db.texel_images[shader][slot].format = zink_get_format(screen, ctx->image_views[shader][slot].base.format); + } else { + struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot); + ctx->di.t.texel_images[shader][slot] = bv->buffer_view; + } } else { struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot); ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL; @@ -663,8 +697,12 @@ update_descriptor_state_image(struct zink_context *ctx, gl_shader_stage shader, } else { if (likely(have_null_descriptors)) { memset(&ctx->di.images[shader][slot], 0, sizeof(ctx->di.images[shader][slot])); - ctx->di.t.texel_images[shader][slot] = VK_NULL_HANDLE; + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + ctx->di.db.texel_images[shader][slot].address = 0; + else + ctx->di.t.texel_images[shader][slot] = VK_NULL_HANDLE; } else { + assert(zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB); struct zink_surface *null_surface = zink_get_dummy_surface(ctx, 0); struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview; ctx->di.images[shader][slot].imageView = null_surface->image_view; @@ -1042,6 +1080,8 @@ zink_create_sampler_view(struct pipe_context *pctx, struct pipe_resource *pres, } err = !sampler_view->image_view; } else { + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + return &sampler_view->base; VkBufferViewCreateInfo bvci = create_bvci(ctx, res, state->format, state->u.buf.offset, state->u.buf.size); sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci); err = !sampler_view->buffer_view; @@ -1718,14 +1758,29 @@ zink_set_shader_images(struct pipe_context *pctx, res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(p_stage); res->barrier_access[p_stage == MESA_SHADER_COMPUTE] |= access; if (images[i].resource->target == PIPE_BUFFER) { - struct zink_buffer_view *bv = create_image_bufferview(ctx, &images[i]); - assert(bv); - if (image_view->buffer_view != bv) { - update_res_bind_count(ctx, res, p_stage == MESA_SHADER_COMPUTE, false); - res->image_bind_count[p_stage == MESA_SHADER_COMPUTE]++; - unbind_shader_image(ctx, p_stage, start_slot + i); + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + if (zink_resource_access_is_write(access)) + util_range_add(&res->base.b, &res->valid_buffer_range, + images[i].u.buf.offset, images[i].u.buf.offset + images[i].u.buf.size); + if (image_view->base.format != images[i].format || + memcmp(&image_view->base.u.buf, &images[i].u.buf, sizeof(images[i].u.buf)) || + image_view->base.resource != &res->base.b || + zink_resource(image_view->base.resource)->obj != res->obj) { + update_res_bind_count(ctx, res, p_stage == MESA_SHADER_COMPUTE, false); + res->image_bind_count[p_stage == MESA_SHADER_COMPUTE]++; + unbind_shader_image(ctx, p_stage, start_slot + i); + } + pipe_resource_reference(&image_view->base.resource, images[i].resource); + } else { + struct zink_buffer_view *bv = create_image_bufferview(ctx, &images[i]); + assert(bv); + if (image_view->buffer_view != bv) { + update_res_bind_count(ctx, res, p_stage == MESA_SHADER_COMPUTE, false); + res->image_bind_count[p_stage == MESA_SHADER_COMPUTE]++; + unbind_shader_image(ctx, p_stage, start_slot + i); + } + image_view->buffer_view = bv; } - image_view->buffer_view = bv; zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, access, res->gfx_barrier); zink_batch_resource_usage_set(&ctx->batch, res, @@ -1839,7 +1894,11 @@ zink_set_sampler_views(struct pipe_context *pctx, res->barrier_access[shader_type == MESA_SHADER_COMPUTE] |= VK_ACCESS_SHADER_READ_BIT; } if (res->base.b.target == PIPE_BUFFER) { - if (b->buffer_view->bvci.buffer != res->obj->buffer) { + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + if (!a || a->base.texture != b->base.texture || zink_resource(a->base.texture)->obj != res->obj || + memcmp(&a->base.u.buf, &b->base.u.buf, sizeof(b->base.u.buf))) + update = true; + } else if (b->buffer_view->bvci.buffer != res->obj->buffer) { /* if this resource has been rebound while it wasn't set here, * its backing resource will have changed and thus we need to update * the bufferview @@ -4103,10 +4162,12 @@ rebind_tbo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot) if (!sampler_view || sampler_view->base.texture->target != PIPE_BUFFER) return NULL; struct zink_resource *res = zink_resource(sampler_view->base.texture); - VkBufferViewCreateInfo bvci = sampler_view->buffer_view->bvci; - bvci.buffer = res->obj->buffer; - zink_buffer_view_reference(zink_screen(ctx->base.screen), &sampler_view->buffer_view, NULL); - sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci); + if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) { + VkBufferViewCreateInfo bvci = sampler_view->buffer_view->bvci; + bvci.buffer = res->obj->buffer; + zink_buffer_view_reference(zink_screen(ctx->base.screen), &sampler_view->buffer_view, NULL); + sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci); + } update_descriptor_state_sampler(ctx, shader, slot, res); zink_context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1); return res; @@ -4119,15 +4180,20 @@ rebind_ibo(struct zink_context *ctx, gl_shader_stage shader, unsigned slot) struct zink_resource *res = zink_resource(image_view->base.resource); if (!res || res->base.b.target != PIPE_BUFFER) return NULL; - VkBufferViewCreateInfo bvci = image_view->buffer_view->bvci; - bvci.buffer = res->obj->buffer; - zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL); + VkBufferViewCreateInfo bvci; + if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) { + bvci = image_view->buffer_view->bvci; + bvci.buffer = res->obj->buffer; + zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL); + } if (!zink_resource_object_init_storage(ctx, res)) { debug_printf("couldn't create storage image!"); return NULL; } - image_view->buffer_view = get_buffer_view(ctx, res, &bvci); - assert(image_view->buffer_view); + if (zink_descriptor_mode != ZINK_DESCRIPTOR_MODE_DB) { + image_view->buffer_view = get_buffer_view(ctx, res, &bvci); + assert(image_view->buffer_view); + } util_range_add(&res->base.b, &res->valid_buffer_range, image_view->base.u.buf.offset, image_view->base.u.buf.offset + image_view->base.u.buf.size); update_descriptor_state_image(ctx, shader, slot, res); @@ -4915,15 +4981,23 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags) /* need to update these based on screen config for null descriptors */ for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.t.ubos[i]); j++) { update_descriptor_state_ubo(ctx, i, j, NULL); + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + ctx->di.db.ubos[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT; } for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.textures[i]); j++) { update_descriptor_state_sampler(ctx, i, j, NULL); + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + ctx->di.db.tbos[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT; } for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.t.ssbos[i]); j++) { update_descriptor_state_ssbo(ctx, i, j, NULL); + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + ctx->di.db.ssbos[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT; } for (unsigned j = 0; j < ARRAY_SIZE(ctx->di.images[i]); j++) { update_descriptor_state_image(ctx, i, j, NULL); + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + ctx->di.db.texel_images[i][j].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT; } } if (!screen->info.rb2_feats.nullDescriptor) diff --git a/src/gallium/drivers/zink/zink_descriptors.c b/src/gallium/drivers/zink/zink_descriptors.c index 9a64a3e9715..765d758c35c 100644 --- a/src/gallium/drivers/zink/zink_descriptors.c +++ b/src/gallium/drivers/zink/zink_descriptors.c @@ -46,7 +46,10 @@ descriptor_layout_create(struct zink_screen *screen, enum zink_descriptor_type t VkDescriptorSetLayoutBindingFlagsCreateInfo fci = {0}; VkDescriptorBindingFlags flags[ZINK_MAX_DESCRIPTORS_PER_TYPE]; dcslci.pNext = &fci; - if (t == ZINK_DESCRIPTOR_TYPE_UNIFORMS) + /* TODO bindless */ + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB && t != ZINK_DESCRIPTOR_BINDLESS) + dcslci.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_DESCRIPTOR_BUFFER_BIT_EXT; + else if (t == ZINK_DESCRIPTOR_TYPE_UNIFORMS) dcslci.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR; fci.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO; fci.bindingCount = num_bindings; @@ -305,6 +308,62 @@ zink_descriptor_util_alloc_sets(struct zink_screen *screen, VkDescriptorSetLayou } static void +init_db_template_entry(struct zink_context *ctx, struct zink_shader *shader, enum zink_descriptor_type type, + unsigned idx, struct zink_descriptor_template *entry, unsigned *entry_idx) +{ + int index = shader->bindings[type][idx].index; + gl_shader_stage stage = shader->nir->info.stage; + struct zink_screen *screen = zink_screen(ctx->base.screen); + entry->count = shader->bindings[type][idx].size; + + switch (shader->bindings[type][idx].type) { + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + entry->mem = (void*)&ctx->di.db.ubos[stage][index]; + entry->stride = sizeof(VkDescriptorAddressInfoEXT); + entry->db_size = screen->info.db_props.robustUniformBufferDescriptorSize; + break; + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: + entry->mem = (void*)&ctx->di.textures[stage][index]; + entry->stride = sizeof(VkDescriptorImageInfo); + entry->db_size = screen->info.db_props.combinedImageSamplerDescriptorSize; + break; + case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: + entry->mem = (void*)&ctx->di.textures[stage][index]; + entry->stride = sizeof(VkDescriptorImageInfo); + entry->db_size = screen->info.db_props.sampledImageDescriptorSize; + break; + case VK_DESCRIPTOR_TYPE_SAMPLER: + entry->mem = (void*)&ctx->di.textures[stage][index]; + entry->stride = sizeof(VkDescriptorImageInfo); + entry->db_size = screen->info.db_props.samplerDescriptorSize; + break; + case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: + entry->mem = (void*)&ctx->di.db.tbos[stage][index]; + entry->stride = sizeof(VkDescriptorAddressInfoEXT); + entry->db_size = screen->info.db_props.robustUniformTexelBufferDescriptorSize; + break; + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + entry->mem = (void*)&ctx->di.db.ssbos[stage][index]; + entry->stride = sizeof(VkDescriptorAddressInfoEXT); + entry->db_size = screen->info.db_props.robustStorageBufferDescriptorSize; + break; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: + entry->mem = (void*)&ctx->di.images[stage][index]; + entry->stride = sizeof(VkDescriptorImageInfo); + entry->db_size = screen->info.db_props.storageImageDescriptorSize; + break; + case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: + entry->mem = (void*)&ctx->di.db.texel_images[stage][index]; + entry->stride = sizeof(VkDescriptorAddressInfoEXT); + entry->db_size = screen->info.db_props.robustStorageTexelBufferDescriptorSize; + break; + default: + unreachable("unknown type"); + } + (*entry_idx)++; +} + +static void init_template_entry(struct zink_shader *shader, enum zink_descriptor_type type, unsigned idx, VkDescriptorUpdateTemplateEntry *entry, unsigned *entry_idx) { @@ -352,6 +411,19 @@ init_template_entry(struct zink_shader *shader, enum zink_descriptor_type type, (*entry_idx)++; } +static void +init_program_db(struct zink_screen *screen, struct zink_program *pg, enum zink_descriptor_type type, VkDescriptorSetLayoutBinding *bindings, unsigned num_bindings, VkDescriptorSetLayout dsl) +{ + VkDeviceSize val; + VKSCR(GetDescriptorSetLayoutSizeEXT)(screen->dev, dsl, &val); + pg->dd.db_size[type] = val; + pg->dd.db_offset[type] = rzalloc_array(pg, uint32_t, num_bindings); + for (unsigned i = 0; i < num_bindings; i++) { + VKSCR(GetDescriptorSetLayoutBindingOffsetEXT)(screen->dev, dsl, bindings[i].binding, &val); + pg->dd.db_offset[type][i] = val; + } +} + static uint16_t descriptor_program_num_sizes(VkDescriptorPoolSize *sizes, enum zink_descriptor_type type) { @@ -419,6 +491,21 @@ zink_descriptor_program_init(struct zink_context *ctx, struct zink_program *pg) } unsigned entry_idx[ZINK_DESCRIPTOR_BASE_TYPES] = {0}; + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + unsigned desc_set_size[ZINK_DESCRIPTOR_BASE_TYPES]; + for (unsigned i = 0; i < ZINK_DESCRIPTOR_BASE_TYPES; i++) + desc_set_size[i] = zink_program_num_bindings_typed(pg, i); + if (screen->compact_descriptors) { + desc_set_size[ZINK_DESCRIPTOR_TYPE_UBO] += desc_set_size[ZINK_DESCRIPTOR_TYPE_SSBO]; + desc_set_size[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW] += desc_set_size[ZINK_DESCRIPTOR_TYPE_IMAGE]; + desc_set_size[ZINK_DESCRIPTOR_TYPE_SSBO] = 0; + desc_set_size[ZINK_DESCRIPTOR_TYPE_IMAGE] = 0; + } + for (unsigned i = 0; i < ZINK_DESCRIPTOR_BASE_TYPES; i++) { + if (desc_set_size[i]) + pg->dd.db_template[i] = rzalloc_array(pg, struct zink_descriptor_template, desc_set_size[i]); + } + } unsigned num_shaders = pg->is_compute ? 1 : ZINK_GFX_SHADER_COUNT; bool have_push = screen->info.have_KHR_push_descriptor; @@ -450,7 +537,10 @@ zink_descriptor_program_init(struct zink_context *ctx, struct zink_program *pg) zink_vktype_to_size_idx(shader->bindings[j][k].type); sizes[idx].descriptorCount += shader->bindings[j][k].size; sizes[idx].type = shader->bindings[j][k].type; - init_template_entry(shader, j, k, &entries[desc_type][entry_idx[desc_type]], &entry_idx[desc_type]); + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + init_db_template_entry(ctx, shader, j, k, &pg->dd.db_template[desc_type][entry_idx[desc_type]], &entry_idx[desc_type]); + else + init_template_entry(shader, j, k, &entries[desc_type][entry_idx[desc_type]], &entry_idx[desc_type]); num_bindings[desc_type]++; has_bindings |= BITFIELD_BIT(desc_type); } @@ -507,6 +597,8 @@ zink_descriptor_program_init(struct zink_context *ctx, struct zink_program *pg) pg->dd.pool_key[desc_type] = descriptor_util_pool_key_get(ctx, desc_type, key, sz, num_type_sizes[desc_type]); pg->dd.pool_key[desc_type]->use_count++; pg->dsl[pg->num_dsl] = pg->dd.layouts[pg->num_dsl]->layout; + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + init_program_db(screen, pg, desc_type, bindings[desc_type], num_bindings[desc_type], pg->dsl[pg->num_dsl]); pg->num_dsl++; } } @@ -533,6 +625,9 @@ zink_descriptor_program_init(struct zink_context *ctx, struct zink_program *pg) return false; pg->compat_id = _mesa_hash_data(pg->dsl, pg->num_dsl * sizeof(pg->dsl[0])); + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + return true; + VkDescriptorUpdateTemplateCreateInfo template[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES] = {0}; /* type of template */ VkDescriptorUpdateTemplateType types[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES] = {VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET}; @@ -586,7 +681,7 @@ zink_descriptor_program_deinit(struct zink_screen *screen, struct zink_program * pg->dd.pool_key[i]->use_count--; pg->dd.pool_key[i] = NULL; } - if (pg->dd.templates[i]) { + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_LAZY && pg->dd.templates[i]) { VKSCR(DestroyDescriptorUpdateTemplate)(screen->dev, pg->dd.templates[i], NULL); pg->dd.templates[i] = VK_NULL_HANDLE; } @@ -852,6 +947,78 @@ populate_sets(struct zink_context *ctx, struct zink_batch_state *bs, } /* updates the mask of changed_sets and binds the mask of bind_sets */ +static void +zink_descriptors_update_masked_buffer(struct zink_context *ctx, bool is_compute, uint8_t changed_sets, uint8_t bind_sets) +{ + struct zink_screen *screen = zink_screen(ctx->base.screen); + struct zink_batch_state *bs = ctx->batch.state; + struct zink_program *pg = is_compute ? &ctx->curr_compute->base : &ctx->curr_program->base; + + /* skip if no descriptors are updated */ + if (!pg->dd.binding_usage || (!changed_sets && !bind_sets)) + return; + + u_foreach_bit(type, changed_sets | bind_sets) { + assert(type + 1 < pg->num_dsl); + assert(type < ZINK_DESCRIPTOR_BASE_TYPES); + bool changed = (changed_sets & BITFIELD_BIT(type)) > 0; + uint64_t offset = changed ? bs->dd.db_offset[type] : bs->dd.cur_db_offset[type]; + if (pg->dd.db_template[type] && changed) { + const struct zink_descriptor_layout_key *key = pg->dd.pool_key[type]->layout; + VkDescriptorGetInfoEXT info; + info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_GET_INFO_EXT; + info.pNext = NULL; + assert(bs->dd.db[type]->obj->size > bs->dd.db_offset[type] + pg->dd.db_size[type]); + for (unsigned i = 0; i < key->num_bindings; i++) { + info.type = key->bindings[i].descriptorType; + uint64_t desc_offset = offset + pg->dd.db_offset[type][i]; + if (screen->info.db_props.combinedImageSamplerDescriptorSingleArray || + key->bindings[i].descriptorType != VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER || + key->bindings[i].descriptorCount == 1) { + for (unsigned j = 0; j < key->bindings[i].descriptorCount; j++) { + /* VkDescriptorDataEXT is a union of pointers; the member doesn't matter */ + info.data.pSampler = (void*)(pg->dd.db_template[type][i].mem + j * pg->dd.db_template[type][i].stride); + VKSCR(GetDescriptorEXT)(screen->dev, &info, pg->dd.db_template[type][i].db_size, bs->dd.db_map[type] + desc_offset + j * pg->dd.db_template[type][i].db_size); + } + } else { + assert(key->bindings[i].descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER); + char buf[1024]; + uint8_t *db = bs->dd.db_map[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW] + desc_offset; + uint8_t *samplers = db + key->bindings[i].descriptorCount * screen->info.db_props.sampledImageDescriptorSize; + for (unsigned j = 0; j < key->bindings[i].descriptorCount; j++) { + /* VkDescriptorDataEXT is a union of pointers; the member doesn't matter */ + info.data.pSampler = (void*)(pg->dd.db_template[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW][i].mem + + j * pg->dd.db_template[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW][i].stride); + VKSCR(GetDescriptorEXT)(screen->dev, &info, pg->dd.db_template[type][ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW].db_size, buf); + /* drivers that don't support combinedImageSamplerDescriptorSingleArray must have sampler arrays written in memory as + * + * | array_of_samplers[] | array_of_sampled_images[] | + * + * which means each descriptor's data must be split + */ + memcpy(db, buf, screen->info.db_props.samplerDescriptorSize); + memcpy(samplers, &buf[screen->info.db_props.samplerDescriptorSize], screen->info.db_props.sampledImageDescriptorSize); + db += screen->info.db_props.sampledImageDescriptorSize; + samplers += screen->info.db_props.samplerDescriptorSize; + } + } + } + bs->dd.cur_db_offset[type] = bs->dd.db_offset[type]; + bs->dd.db_offset[type] += pg->dd.db_size[type]; + } + /* templates are indexed by the set id, so increment type by 1 + * (this is effectively an optimization of indirecting through screen->desc_set_id) + */ + VKCTX(CmdSetDescriptorBufferOffsetsEXT)(bs->cmdbuf, + is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, + pg->layout, + type + 1, 1, + &type, + &offset); + } +} + +/* updates the mask of changed_sets and binds the mask of bind_sets */ void zink_descriptors_update_masked(struct zink_context *ctx, bool is_compute, uint8_t changed_sets, uint8_t bind_sets) { @@ -946,29 +1113,71 @@ zink_descriptors_update(struct zink_context *ctx, bool is_compute) * VK 14.2.2 */ uint8_t bind_sets = bs->dd.pg[is_compute] && bs->dd.compat_id[is_compute] == pg->compat_id ? 0 : pg->dd.binding_usage; + if (pg->dd.push_usage && (ctx->dd.push_state_changed[is_compute] || bind_sets)) { - if (have_KHR_push_descriptor) { - if (ctx->dd.push_state_changed[is_compute]) - VKCTX(CmdPushDescriptorSetWithTemplateKHR)(bs->cmdbuf, pg->dd.templates[0], - pg->layout, 0, ctx); - } else { + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + uint32_t index = ZINK_DESCRIPTOR_TYPE_UNIFORMS; + uint64_t offset = ctx->dd.push_state_changed[is_compute] ? + bs->dd.db_offset[ZINK_DESCRIPTOR_TYPE_UNIFORMS] : + bs->dd.cur_db_offset[ZINK_DESCRIPTOR_TYPE_UNIFORMS]; if (ctx->dd.push_state_changed[is_compute]) { - struct zink_descriptor_pool *pool = check_push_pool_alloc(ctx, &bs->dd.push_pool[pg->is_compute], bs, pg->is_compute); - VkDescriptorSet push_set = get_descriptor_set(pool); - if (!push_set) - mesa_loge("ZINK: failed to get push descriptor set! prepare to crash!"); - VKCTX(UpdateDescriptorSetWithTemplate)(screen->dev, push_set, pg->dd.templates[0], ctx); - bs->dd.sets[is_compute][0] = push_set; + assert(bs->dd.db[ZINK_DESCRIPTOR_TYPE_UNIFORMS]->obj->size > bs->dd.db_offset[ZINK_DESCRIPTOR_TYPE_UNIFORMS] + ctx->dd.db_size[is_compute]); + for (unsigned i = 0; i < (is_compute ? 1 : ZINK_GFX_SHADER_COUNT); i++) { + VkDescriptorGetInfoEXT info; + info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_GET_INFO_EXT; + info.pNext = NULL; + info.type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; + info.data.pUniformBuffer = &ctx->di.db.ubos[is_compute ? MESA_SHADER_COMPUTE : i][0]; + uint64_t stage_offset = offset + (is_compute ? 0 : ctx->dd.db_offset[i]); + VKSCR(GetDescriptorEXT)(screen->dev, &info, screen->info.db_props.robustUniformBufferDescriptorSize, + bs->dd.db_map[ZINK_DESCRIPTOR_TYPE_UNIFORMS] + stage_offset); + } + if (!is_compute && ctx->dd.has_fbfetch) { + VkDescriptorGetInfoEXT info; + info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_GET_INFO_EXT; + info.pNext = NULL; + info.type = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT; + info.data.pInputAttachmentImage = &ctx->di.fbfetch; + uint64_t stage_offset = offset + ctx->dd.db_offset[MESA_SHADER_FRAGMENT + 1]; + VKSCR(GetDescriptorEXT)(screen->dev, &info, screen->info.db_props.robustUniformBufferDescriptorSize, + bs->dd.db_map[ZINK_DESCRIPTOR_TYPE_UNIFORMS] + stage_offset); + } + bs->dd.cur_db_offset[ZINK_DESCRIPTOR_TYPE_UNIFORMS] = bs->dd.db_offset[ZINK_DESCRIPTOR_TYPE_UNIFORMS]; + bs->dd.db_offset[ZINK_DESCRIPTOR_TYPE_UNIFORMS] += ctx->dd.db_size[is_compute]; + } + VKCTX(CmdSetDescriptorBufferOffsetsEXT)(bs->cmdbuf, + is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, + pg->layout, + 0, 1, + &index, + &offset); + } else { + if (have_KHR_push_descriptor) { + if (ctx->dd.push_state_changed[is_compute]) + VKCTX(CmdPushDescriptorSetWithTemplateKHR)(bs->cmdbuf, pg->dd.templates[0], + pg->layout, 0, ctx); + } else { + if (ctx->dd.push_state_changed[is_compute]) { + struct zink_descriptor_pool *pool = check_push_pool_alloc(ctx, &bs->dd.push_pool[pg->is_compute], bs, pg->is_compute); + VkDescriptorSet push_set = get_descriptor_set(pool); + if (!push_set) + mesa_loge("ZINK: failed to get push descriptor set! prepare to crash!"); + VKCTX(UpdateDescriptorSetWithTemplate)(screen->dev, push_set, pg->dd.templates[0], ctx); + bs->dd.sets[is_compute][0] = push_set; + } + assert(bs->dd.sets[is_compute][0]); + VKCTX(CmdBindDescriptorSets)(bs->cmdbuf, + is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, + pg->layout, 0, 1, &bs->dd.sets[is_compute][0], + 0, NULL); } - assert(bs->dd.sets[is_compute][0]); - VKCTX(CmdBindDescriptorSets)(bs->cmdbuf, - is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, - pg->layout, 0, 1, &bs->dd.sets[is_compute][0], - 0, NULL); } } ctx->dd.push_state_changed[is_compute] = false; - zink_descriptors_update_masked(ctx, is_compute, changed_sets, bind_sets); + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + zink_descriptors_update_masked_buffer(ctx, is_compute, changed_sets, bind_sets); + else + zink_descriptors_update_masked(ctx, is_compute, changed_sets, bind_sets); /* bindless descriptors are context-based and get updated elsewhere */ if (pg->dd.bindless && unlikely(!ctx->dd.bindless_bound)) { VKCTX(CmdBindDescriptorSets)(ctx->batch.state->cmdbuf, is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, @@ -1023,6 +1232,11 @@ zink_batch_descriptor_deinit(struct zink_screen *screen, struct zink_batch_state pool_destroy(screen, bs->dd.push_pool[i].pool); deinit_multi_pool_overflow(screen, &bs->dd.push_pool[i]); } + + for (unsigned i = 0; i < ARRAY_SIZE(bs->dd.db); i++) { + if (bs->dd.db[i]) + screen->base.resource_destroy(&screen->base, &bs->dd.db[i]->base.b); + } } /* ensure the idle/usable overflow set array always has as many members as possible by merging both arrays on batch state reset */ @@ -1055,35 +1269,39 @@ consolidate_pool_alloc(struct zink_screen *screen, struct zink_descriptor_pool_m void zink_batch_descriptor_reset(struct zink_screen *screen, struct zink_batch_state *bs) { - for (unsigned i = 0; i < ZINK_DESCRIPTOR_BASE_TYPES; i++) { - struct zink_descriptor_pool_multi **mpools = bs->dd.pools[i].data; - for (unsigned j = 0; j < bs->dd.pool_size[i]; j++) { - struct zink_descriptor_pool_multi *mpool = mpools[j]; - if (!mpool) - continue; - consolidate_pool_alloc(screen, mpool); - - /* if the pool is still in use, reset the current set index */ - if (mpool->pool_key->use_count) - mpool->pool->set_idx = 0; - else { - /* otherwise destroy it to reclaim memory */ - multi_pool_destroy(screen, mpool); - mpools[j] = NULL; + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + memset(bs->dd.db_offset, 0, sizeof(bs->dd.db_offset)); + } else { + for (unsigned i = 0; i < ZINK_DESCRIPTOR_BASE_TYPES; i++) { + struct zink_descriptor_pool_multi **mpools = bs->dd.pools[i].data; + for (unsigned j = 0; j < bs->dd.pool_size[i]; j++) { + struct zink_descriptor_pool_multi *mpool = mpools[j]; + if (!mpool) + continue; + consolidate_pool_alloc(screen, mpool); + + /* if the pool is still in use, reset the current set index */ + if (mpool->pool_key->use_count) + mpool->pool->set_idx = 0; + else { + /* otherwise destroy it to reclaim memory */ + multi_pool_destroy(screen, mpool); + mpools[j] = NULL; + } } } - } - for (unsigned i = 0; i < 2; i++) { - bs->dd.pg[i] = NULL; - if (bs->dd.push_pool[i].reinit_overflow) { - /* these don't match current fbfetch usage and can never be used again */ - clear_multi_pool_overflow(screen, &bs->dd.push_pool[i].overflowed_pools[bs->dd.push_pool[i].overflow_idx]); - } else if (bs->dd.push_pool[i].pool) { - consolidate_pool_alloc(screen, &bs->dd.push_pool[i]); + for (unsigned i = 0; i < 2; i++) { + if (bs->dd.push_pool[i].reinit_overflow) { + /* these don't match current fbfetch usage and can never be used again */ + clear_multi_pool_overflow(screen, &bs->dd.push_pool[i].overflowed_pools[bs->dd.push_pool[i].overflow_idx]); + } else if (bs->dd.push_pool[i].pool) { + consolidate_pool_alloc(screen, &bs->dd.push_pool[i]); + } + if (bs->dd.push_pool[i].pool) + bs->dd.push_pool[i].pool->set_idx = 0; } - if (bs->dd.push_pool[i].pool) - bs->dd.push_pool[i].pool->set_idx = 0; } + memset(bs->dd.pg, 0, sizeof(bs->dd.pg)); } /* called on batch state creation */ @@ -1099,6 +1317,22 @@ zink_batch_descriptor_init(struct zink_screen *screen, struct zink_batch_state * util_dynarray_init(&bs->dd.push_pool[i].overflowed_pools[1], bs); } } + + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB && !(bs->ctx->flags & ZINK_CONTEXT_COPY_ONLY)) { + /* TODO: bindless */ + for (unsigned i = 0; i < ZINK_DESCRIPTOR_NON_BINDLESS_TYPES; i++) { + if (!screen->db_size[i]) + continue; + unsigned bind = i == ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW ? ZINK_BIND_SAMPLER_DESCRIPTOR : ZINK_BIND_RESOURCE_DESCRIPTOR; + if (screen->compact_descriptors && i == ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW) + bind |= ZINK_BIND_RESOURCE_DESCRIPTOR; + struct pipe_resource *pres = pipe_buffer_create(&screen->base, bind, 0, screen->db_size[i]); + if (!pres) + return false; + bs->dd.db[i] = zink_resource(pres); + bs->dd.db_map[i] = pipe_buffer_map(&bs->ctx->base, pres, PIPE_MAP_READ | PIPE_MAP_WRITE, &bs->dd.db_xfer[i]); + } + } return true; } @@ -1116,6 +1350,7 @@ init_push_template_entry(VkDescriptorUpdateTemplateEntry *entry, unsigned i) bool zink_descriptors_init(struct zink_context *ctx) { + struct zink_screen *screen = zink_screen(ctx->base.screen); for (unsigned i = 0; i < ZINK_GFX_SHADER_COUNT; i++) { VkDescriptorUpdateTemplateEntry *entry = &ctx->dd.push_entries[i]; init_push_template_entry(entry, i); @@ -1131,10 +1366,22 @@ zink_descriptors_init(struct zink_context *ctx) if (!zink_descriptor_util_push_layouts_get(ctx, ctx->dd.push_dsl, ctx->dd.push_layout_keys)) return false; - ctx->dd.dummy_dsl = descriptor_util_layout_get(zink_screen(ctx->base.screen), 0, NULL, 0, &layout_key); + ctx->dd.dummy_dsl = descriptor_util_layout_get(screen, 0, NULL, 0, &layout_key); if (!ctx->dd.dummy_dsl) return false; + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + VkDeviceSize val; + for (unsigned i = 0; i < 2; i++) { + VKSCR(GetDescriptorSetLayoutSizeEXT)(screen->dev, ctx->dd.push_dsl[i]->layout, &val); + ctx->dd.db_size[i] = val; + } + for (unsigned i = 0; i < ZINK_GFX_SHADER_COUNT; i++) { + VKSCR(GetDescriptorSetLayoutBindingOffsetEXT)(screen->dev, ctx->dd.push_dsl[0]->layout, i, &val); + ctx->dd.db_offset[i] = val; + } + } + return true; } @@ -1196,6 +1443,16 @@ zink_descriptor_util_init_fbfetch(struct zink_context *ctx) //ralloc_free(ctx->dd.push_layout_keys[0]); ctx->dd.push_dsl[0] = create_gfx_layout(ctx, &ctx->dd.push_layout_keys[0], true); ctx->dd.has_fbfetch = true; + + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + VkDeviceSize val; + VKSCR(GetDescriptorSetLayoutSizeEXT)(screen->dev, ctx->dd.push_dsl[0]->layout, &val); + ctx->dd.db_size[0] = val; + for (unsigned i = 0; i < ARRAY_SIZE(ctx->dd.db_offset); i++) { + VKSCR(GetDescriptorSetLayoutBindingOffsetEXT)(screen->dev, ctx->dd.push_dsl[0]->layout, i, &val); + ctx->dd.db_offset[i] = val; + } + } } /* bindless descriptor bindings have their own struct indexing */ diff --git a/src/gallium/drivers/zink/zink_device_info.py b/src/gallium/drivers/zink/zink_device_info.py index 86f3c8cf4d0..4c6973fda7d 100644 --- a/src/gallium/drivers/zink/zink_device_info.py +++ b/src/gallium/drivers/zink/zink_device_info.py @@ -125,6 +125,7 @@ EXTENSIONS = [ conditions=["$props.fullyCoveredFragmentShaderInputVariable"]), Extension("VK_KHR_shader_draw_parameters"), Extension("VK_KHR_sampler_mirror_clamp_to_edge"), + Extension("VK_EXT_descriptor_buffer", alias="db", features=True, properties=True), Extension("VK_EXT_conditional_rendering", alias="cond_render", features=True, diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c index 37ca941e879..f2e59c90909 100644 --- a/src/gallium/drivers/zink/zink_resource.c +++ b/src/gallium/drivers/zink/zink_resource.c @@ -178,17 +178,27 @@ create_bci(struct zink_screen *screen, const struct pipe_resource *templ, unsign bci.flags = 0; assert(bci.size > 0); - bci.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | - VK_BUFFER_USAGE_TRANSFER_DST_BIT | - VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; - - bci.usage |= VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | - VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | - VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | - VK_BUFFER_USAGE_INDEX_BUFFER_BIT | - VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | - VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | - VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT; + if (bind & ZINK_BIND_DESCRIPTOR) { + /* gallium sizes are all uint32_t, while the total size of this buffer may exceed that limit */ + bci.size *= ZINK_DESCRIPTOR_BUFFER_MULTIPLIER; + bci.usage = 0; + if (bind & ZINK_BIND_SAMPLER_DESCRIPTOR) + bci.usage |= VK_BUFFER_USAGE_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT; + if (bind & ZINK_BIND_RESOURCE_DESCRIPTOR) + bci.usage |= VK_BUFFER_USAGE_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT; + } else { + bci.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + + bci.usage |= VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | + VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | + VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDEX_BUFFER_BIT | + VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | + VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | + VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT; + } if (bind & PIPE_BIND_SHADER_IMAGE) bci.usage |= VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT; @@ -653,7 +663,7 @@ resource_object_create(struct zink_screen *screen, const struct pipe_resource *t goto fail1; } - if (!(templ->bind & PIPE_BIND_SHADER_IMAGE)) { + if (!(templ->bind & (PIPE_BIND_SHADER_IMAGE | ZINK_BIND_DESCRIPTOR))) { bci.usage |= VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT; if (VKSCR(CreateBuffer)(screen->dev, &bci, NULL, &obj->storage_buffer) != VK_SUCCESS) { mesa_loge("ZINK: vkCreateBuffer failed"); @@ -1182,6 +1192,8 @@ resource_create(struct pipe_screen *pscreen, */ res->base.b.flags |= PIPE_RESOURCE_FLAG_DONT_MAP_DIRECTLY; } + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) + zink_resource_get_address(screen, res); } else { if (templ->flags & PIPE_RESOURCE_FLAG_SPARSE) res->base.b.bind |= PIPE_BIND_SHADER_IMAGE; diff --git a/src/gallium/drivers/zink/zink_resource.h b/src/gallium/drivers/zink/zink_resource.h index 652585cc7ab..37da41827c9 100644 --- a/src/gallium/drivers/zink/zink_resource.h +++ b/src/gallium/drivers/zink/zink_resource.h @@ -27,6 +27,9 @@ #include "zink_types.h" #define ZINK_MAP_TEMPORARY (PIPE_MAP_DRV_PRV << 0) +#define ZINK_BIND_SAMPLER_DESCRIPTOR (1u << 26) +#define ZINK_BIND_RESOURCE_DESCRIPTOR (1u << 27) +#define ZINK_BIND_DESCRIPTOR (ZINK_BIND_SAMPLER_DESCRIPTOR | ZINK_BIND_RESOURCE_DESCRIPTOR) #define ZINK_BIND_MUTABLE (1u << 28) #define ZINK_BIND_DMABUF (1u << 29) #define ZINK_BIND_TRANSIENT (1u << 30) //transient fb attachment diff --git a/src/gallium/drivers/zink/zink_screen.c b/src/gallium/drivers/zink/zink_screen.c index cdf5d46b41c..8b3643ff457 100644 --- a/src/gallium/drivers/zink/zink_screen.c +++ b/src/gallium/drivers/zink/zink_screen.c @@ -93,6 +93,7 @@ static const struct debug_named_value zink_descriptor_options[] = { { "auto", ZINK_DESCRIPTOR_MODE_AUTO, "Automatically detect best mode" }, { "lazy", ZINK_DESCRIPTOR_MODE_LAZY, "Don't cache, do least amount of updates" }, + { "db", ZINK_DESCRIPTOR_MODE_DB, "Use descriptor buffers" }, DEBUG_NAMED_VALUE_END }; @@ -2716,6 +2717,51 @@ zink_internal_create_screen(const struct pipe_screen_config *config) screen->resizable_bar = true; } + if (zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_DB) { + if (!screen->info.have_EXT_descriptor_buffer) { + mesa_loge("Cannot use db descriptor mode without EXT_descriptor_buffer"); + goto fail; + } + if (!screen->resizable_bar) { + mesa_loge("Cannot use db descriptor mode without resizable bar"); + goto fail; + } + if (!screen->info.have_EXT_non_seamless_cube_map) { + mesa_loge("Cannot use db descriptor mode without EXT_non_seamless_cube_map"); + goto fail; + } + if (!screen->info.rb2_feats.nullDescriptor) { + mesa_loge("Cannot use db descriptor mode without robustness2.nullDescriptor"); + goto fail; + } + if (screen->compact_descriptors) { + /* TODO: bindless */ + if (screen->info.db_props.maxDescriptorBufferBindings < 3) { + mesa_loge("Cannot use db descriptor mode with compact descriptors with maxDescriptorBufferBindings < 3"); + goto fail; + } + } else { + if (screen->info.db_props.maxDescriptorBufferBindings < 5) { + mesa_loge("Cannot use db descriptor mode with maxDescriptorBufferBindings < 5"); + goto fail; + } + } + const uint32_t sampler_size = MAX2(screen->info.db_props.combinedImageSamplerDescriptorSize, screen->info.db_props.robustUniformTexelBufferDescriptorSize); + const uint32_t image_size = MAX2(screen->info.db_props.storageImageDescriptorSize, screen->info.db_props.robustStorageTexelBufferDescriptorSize); + if (screen->compact_descriptors) { + screen->db_size[ZINK_DESCRIPTOR_TYPE_UBO] = screen->info.db_props.robustUniformBufferDescriptorSize + + screen->info.db_props.robustStorageBufferDescriptorSize; + screen->db_size[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW] = sampler_size + image_size; + } else { + screen->db_size[ZINK_DESCRIPTOR_TYPE_UBO] = screen->info.db_props.robustUniformBufferDescriptorSize; + screen->db_size[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW] = sampler_size; + screen->db_size[ZINK_DESCRIPTOR_TYPE_SSBO] = screen->info.db_props.robustStorageBufferDescriptorSize; + screen->db_size[ZINK_DESCRIPTOR_TYPE_IMAGE] = image_size; + } + screen->db_size[ZINK_DESCRIPTOR_TYPE_UNIFORMS] = screen->info.db_props.robustUniformBufferDescriptorSize; + screen->info.have_KHR_push_descriptor = false; + } + simple_mtx_init(&screen->dt_lock, mtx_plain); util_idalloc_mt_init_tc(&screen->buffer_ids); diff --git a/src/gallium/drivers/zink/zink_types.h b/src/gallium/drivers/zink/zink_types.h index 76e5da49391..1dd29df254d 100644 --- a/src/gallium/drivers/zink/zink_types.h +++ b/src/gallium/drivers/zink/zink_types.h @@ -76,6 +76,10 @@ /* enum zink_descriptor_type */ #define ZINK_MAX_DESCRIPTOR_SETS 6 #define ZINK_MAX_DESCRIPTORS_PER_TYPE (32 * ZINK_GFX_SHADER_COUNT) +/* the number of typed descriptors that can fit in a given batch; + * sized based on max values seen in drawoverhead + */ +#define ZINK_DESCRIPTOR_BUFFER_MULTIPLIER 25000 /* suballocator defines */ #define NUM_SLAB_ALLOCATORS 3 @@ -149,8 +153,12 @@ enum zink_descriptor_type { enum zink_descriptor_mode { ZINK_DESCRIPTOR_MODE_AUTO, ZINK_DESCRIPTOR_MODE_LAZY, + ZINK_DESCRIPTOR_MODE_DB, }; +/* the current mode */ +extern enum zink_descriptor_mode zink_descriptor_mode; + /* indexing for descriptor template management */ enum zink_descriptor_size_index { ZDS_INDEX_UBO, @@ -362,6 +370,14 @@ struct zink_descriptor_pool_key { struct zink_descriptor_layout_key *layout; }; +/* a template used for updating descriptor buffers */ +struct zink_descriptor_template { + uint16_t stride; //the stride between mem pointers + uint16_t db_size; //the size of the entry in the buffer + unsigned count; //the number of descriptors + uint8_t *mem; //the base host pointer to update from +}; + /* ctx->dd; created at context creation */ struct zink_descriptor_data { bool bindless_bound; @@ -382,6 +398,11 @@ struct zink_descriptor_data { VkDescriptorUpdateTemplateEntry push_entries[MESA_SHADER_STAGES]; //gfx+fbfetch VkDescriptorUpdateTemplateEntry compute_push_entry; + + /* push descriptor layout size and binding offsets */ + uint32_t db_size[2]; //gfx, compute + uint32_t db_offset[ZINK_GFX_SHADER_COUNT + 1]; //gfx + fbfetch + /* compute offset is always 0 */ }; /* pg->dd; created at program creation */ @@ -397,7 +418,12 @@ struct zink_program_descriptor_data { /* all the layouts for the program */ struct zink_descriptor_layout *layouts[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; /* all the templates for the program */ - VkDescriptorUpdateTemplate templates[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; + union { + VkDescriptorUpdateTemplate templates[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; + struct zink_descriptor_template *db_template[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; + }; + uint32_t db_size[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the total size of the layout + uint32_t *db_offset[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the offset of each binding in the layout }; struct zink_descriptor_pool { @@ -439,10 +465,18 @@ struct zink_batch_descriptor_data { uint32_t compat_id[2]; //gfx, compute /* the current set layout */ VkDescriptorSetLayout dsl[2][ZINK_DESCRIPTOR_BASE_TYPES]; //gfx, compute - /* the current set for a given type; used for rebinding if pipeline compat id changes and current set must be rebound */ - VkDescriptorSet sets[2][ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //gfx, compute + union { + /* the current set for a given type; used for rebinding if pipeline compat id changes and current set must be rebound */ + VkDescriptorSet sets[2][ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //gfx, compute + uint64_t cur_db_offset[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //gfx, compute; the current offset of a descriptor buffer for rebinds + }; /* mask of push descriptor usage */ unsigned push_usage[2]; //gfx, compute + + struct zink_resource *db[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the descriptor buffer for a given type + uint8_t *db_map[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the host map for the buffer + struct pipe_transfer *db_xfer[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the transfer map for the buffer + uint64_t db_offset[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the "next" offset that will be used when the buffer is updated }; /** batch types */ @@ -1212,6 +1246,8 @@ struct zink_screen { struct set desc_pool_keys[ZINK_DESCRIPTOR_BASE_TYPES]; struct util_live_shader_cache shaders; + uint64_t db_size[ZINK_DESCRIPTOR_ALL_TYPES]; + struct { struct pb_cache bo_cache; struct pb_slabs bo_slabs[NUM_SLAB_ALLOCATORS]; @@ -1646,12 +1682,20 @@ struct zink_context { VkDescriptorImageInfo images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES]; uint8_t num_images[MESA_SHADER_STAGES]; - struct { - VkDescriptorBufferInfo ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS]; - VkDescriptorBufferInfo ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS]; - VkBufferView tbos[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS]; - VkBufferView texel_images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES]; - } t; + union { + struct { + VkDescriptorBufferInfo ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS]; + VkDescriptorBufferInfo ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS]; + VkBufferView tbos[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS]; + VkBufferView texel_images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES]; + } t; + struct { + VkDescriptorAddressInfoEXT ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS]; + VkDescriptorAddressInfoEXT ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS]; + VkDescriptorAddressInfoEXT tbos[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS]; + VkDescriptorAddressInfoEXT texel_images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES]; + } db; + }; VkDescriptorImageInfo fbfetch; |