diff options
Diffstat (limited to 'src/gallium/drivers/crocus/crocus_resolve.c')
| -rw-r--r-- | src/gallium/drivers/crocus/crocus_resolve.c | 1061 |
1 files changed, 1061 insertions, 0 deletions
diff --git a/src/gallium/drivers/crocus/crocus_resolve.c b/src/gallium/drivers/crocus/crocus_resolve.c new file mode 100644 index 00000000000..a38eb4a94a7 --- /dev/null +++ b/src/gallium/drivers/crocus/crocus_resolve.c @@ -0,0 +1,1061 @@ +/* + * Copyright © 2017 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/** + * @file crocus_resolve.c + * + * This file handles resolve tracking for main and auxiliary surfaces. + * + * It also handles our cache tracking. We have sets for the render cache, + * depth cache, and so on. If a BO is in a cache's set, then it may have + * data in that cache. The helpers take care of emitting flushes for + * render-to-texture, format reinterpretation issues, and other situations. + */ + +#include "util/hash_table.h" +#include "util/set.h" +#include "crocus_context.h" +#include "compiler/nir/nir.h" + +#define FILE_DEBUG_FLAG DEBUG_BLORP + +static void +crocus_update_stencil_shadow(struct crocus_context *ice, + struct crocus_resource *res); +/** + * Disable auxiliary buffers if a renderbuffer is also bound as a texture + * or shader image. This causes a self-dependency, where both rendering + * and sampling may concurrently read or write the CCS buffer, causing + * incorrect pixels. + */ +static bool +disable_rb_aux_buffer(struct crocus_context *ice, + bool *draw_aux_buffer_disabled, + struct crocus_resource *tex_res, + unsigned min_level, unsigned num_levels, + const char *usage) +{ + struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer; + bool found = false; + + /* We only need to worry about fast clears. */ + if (tex_res->aux.usage != ISL_AUX_USAGE_CCS_D) + return false; + + for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) { + struct crocus_surface *surf = (void *) cso_fb->cbufs[i]; + if (!surf) + continue; + + struct crocus_resource *rb_res = (void *) surf->base.texture; + + if (rb_res->bo == tex_res->bo && + surf->base.u.tex.level >= min_level && + surf->base.u.tex.level < min_level + num_levels) { + found = draw_aux_buffer_disabled[i] = true; + } + } + + if (found) { + perf_debug(&ice->dbg, + "Disabling CCS because a renderbuffer is also bound %s.\n", + usage); + } + + return found; +} + +static void +resolve_sampler_views(struct crocus_context *ice, + struct crocus_batch *batch, + struct crocus_shader_state *shs, + const struct shader_info *info, + bool *draw_aux_buffer_disabled, + bool consider_framebuffer) +{ + uint32_t views = info ? (shs->bound_sampler_views & info->textures_used[0]) : 0; + + while (views) { + const int i = u_bit_scan(&views); + struct crocus_sampler_view *isv = shs->textures[i]; + + if (isv->res->base.target != PIPE_BUFFER) { + if (consider_framebuffer) { + disable_rb_aux_buffer(ice, draw_aux_buffer_disabled, isv->res, + isv->view.base_level, isv->view.levels, + "for sampling"); + } + + crocus_resource_prepare_texture(ice, isv->res, isv->view.format, + isv->view.base_level, isv->view.levels, + isv->view.base_array_layer, + isv->view.array_len); + } + + crocus_cache_flush_for_read(batch, isv->res->bo); + + if (batch->screen->devinfo.ver >= 7 && + (isv->base.format == PIPE_FORMAT_X24S8_UINT || + isv->base.format == PIPE_FORMAT_X32_S8X24_UINT || + isv->base.format == PIPE_FORMAT_S8_UINT)) { + struct crocus_resource *zres, *sres; + crocus_get_depth_stencil_resources(&batch->screen->devinfo, isv->base.texture, &zres, &sres); + crocus_update_stencil_shadow(ice, sres); + crocus_cache_flush_for_read(batch, sres->shadow->bo); + } + } +} + +static void +resolve_image_views(struct crocus_context *ice, + struct crocus_batch *batch, + struct crocus_shader_state *shs, + bool *draw_aux_buffer_disabled, + bool consider_framebuffer) +{ + /* TODO: Consider images used by program */ + uint32_t views = shs->bound_image_views; + + while (views) { + const int i = u_bit_scan(&views); + struct pipe_image_view *pview = &shs->image[i].base; + struct crocus_resource *res = (void *) pview->resource; + + if (res->base.target != PIPE_BUFFER) { + if (consider_framebuffer) { + disable_rb_aux_buffer(ice, draw_aux_buffer_disabled, + res, pview->u.tex.level, 1, + "as a shader image"); + } + + unsigned num_layers = + pview->u.tex.last_layer - pview->u.tex.first_layer + 1; + + /* The data port doesn't understand any compression */ + crocus_resource_prepare_access(ice, res, + pview->u.tex.level, 1, + pview->u.tex.first_layer, num_layers, + ISL_AUX_USAGE_NONE, false); + } + + crocus_cache_flush_for_read(batch, res->bo); + } +} + +static void +crocus_update_align_res(struct crocus_batch *batch, + struct crocus_surface *surf, + bool copy_to_wa) +{ + struct crocus_screen *screen = (struct crocus_screen *)batch->screen; + struct pipe_blit_info info = { 0 }; + + info.src.resource = copy_to_wa ? surf->base.texture : surf->align_res; + info.src.level = copy_to_wa ? surf->base.u.tex.level : 0; + u_box_2d_zslice(0, 0, copy_to_wa ? surf->base.u.tex.first_layer : 0, + u_minify(surf->base.texture->width0, surf->base.u.tex.level), + u_minify(surf->base.texture->height0, surf->base.u.tex.level), &info.src.box); + info.src.format = surf->base.texture->format; + info.dst.resource = copy_to_wa ? surf->align_res : surf->base.texture; + info.dst.level = copy_to_wa ? 0 : surf->base.u.tex.level; + info.dst.box = info.src.box; + info.dst.box.z = copy_to_wa ? 0 : surf->base.u.tex.first_layer; + info.dst.format = surf->base.texture->format; + info.mask = util_format_is_depth_or_stencil(surf->base.texture->format) ? PIPE_MASK_ZS : PIPE_MASK_RGBA; + info.filter = 0; + if (!screen->vtbl.blit_blt(batch, &info)) { + assert(0); + } +} + +/** + * \brief Resolve buffers before drawing. + * + * Resolve the depth buffer's HiZ buffer, resolve the depth buffer of each + * enabled depth texture, and flush the render cache for any dirty textures. + */ +void +crocus_predraw_resolve_inputs(struct crocus_context *ice, + struct crocus_batch *batch, + bool *draw_aux_buffer_disabled, + gl_shader_stage stage, + bool consider_framebuffer) +{ + struct crocus_shader_state *shs = &ice->state.shaders[stage]; + const struct shader_info *info = crocus_get_shader_info(ice, stage); + + uint64_t stage_dirty = (CROCUS_STAGE_DIRTY_BINDINGS_VS << stage) | + (consider_framebuffer ? CROCUS_STAGE_DIRTY_BINDINGS_FS : 0); + + if (ice->state.stage_dirty & stage_dirty) { + resolve_sampler_views(ice, batch, shs, info, draw_aux_buffer_disabled, + consider_framebuffer); + resolve_image_views(ice, batch, shs, draw_aux_buffer_disabled, + consider_framebuffer); + } +} + +void +crocus_predraw_resolve_framebuffer(struct crocus_context *ice, + struct crocus_batch *batch, + bool *draw_aux_buffer_disabled) +{ + struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer; + struct crocus_screen *screen = (void *) ice->ctx.screen; + struct intel_device_info *devinfo = &screen->devinfo; + struct crocus_uncompiled_shader *ish = + ice->shaders.uncompiled[MESA_SHADER_FRAGMENT]; + const nir_shader *nir = ish->nir; + + if (ice->state.dirty & CROCUS_DIRTY_DEPTH_BUFFER) { + struct pipe_surface *zs_surf = cso_fb->zsbuf; + + if (zs_surf) { + struct crocus_resource *z_res, *s_res; + crocus_get_depth_stencil_resources(devinfo, zs_surf->texture, &z_res, &s_res); + unsigned num_layers = + zs_surf->u.tex.last_layer - zs_surf->u.tex.first_layer + 1; + + if (z_res) { + crocus_resource_prepare_render(ice, z_res, + zs_surf->u.tex.level, + zs_surf->u.tex.first_layer, + num_layers, ice->state.hiz_usage); + crocus_cache_flush_for_depth(batch, z_res->bo); + + if (((struct crocus_surface *)zs_surf)->align_res) { + crocus_update_align_res(batch, (struct crocus_surface *)zs_surf, true); + } + } + + if (s_res) { + crocus_cache_flush_for_depth(batch, s_res->bo); + } + } + } + + if (nir->info.outputs_read != 0) { + for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) { + if (cso_fb->cbufs[i]) { + struct crocus_surface *surf = (void *) cso_fb->cbufs[i]; + struct crocus_resource *res = (void *) cso_fb->cbufs[i]->texture; + + crocus_resource_prepare_texture(ice, res, surf->view.format, + surf->view.base_level, 1, + surf->view.base_array_layer, + surf->view.array_len); + } + } + } + + if (ice->state.stage_dirty & CROCUS_STAGE_DIRTY_BINDINGS_FS) { + for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) { + struct crocus_surface *surf = (void *) cso_fb->cbufs[i]; + if (!surf) + continue; + + struct crocus_resource *res = (void *) surf->base.texture; + + if (surf->align_res) + crocus_update_align_res(batch, surf, true); + + enum isl_aux_usage aux_usage = + crocus_resource_render_aux_usage(ice, res, surf->view.format, + ice->state.blend_enables & (1u << i), + draw_aux_buffer_disabled[i]); + + if (ice->state.draw_aux_usage[i] != aux_usage) { + ice->state.draw_aux_usage[i] = aux_usage; + /* XXX: Need to track which bindings to make dirty */ + ice->state.stage_dirty |= CROCUS_ALL_STAGE_DIRTY_BINDINGS; + } + + crocus_resource_prepare_render(ice, res, surf->view.base_level, + surf->view.base_array_layer, + surf->view.array_len, + aux_usage); + + crocus_cache_flush_for_render(batch, res->bo, surf->view.format, + aux_usage); + } + } +} + +/** + * \brief Call this after drawing to mark which buffers need resolving + * + * If the depth buffer was written to and if it has an accompanying HiZ + * buffer, then mark that it needs a depth resolve. + * + * If the color buffer is a multisample window system buffer, then + * mark that it needs a downsample. + * + * Also mark any render targets which will be textured as needing a render + * cache flush. + */ +void +crocus_postdraw_update_resolve_tracking(struct crocus_context *ice, + struct crocus_batch *batch) +{ + struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer; + struct crocus_screen *screen = (void *) ice->ctx.screen; + struct intel_device_info *devinfo = &screen->devinfo; + // XXX: front buffer drawing? + + bool may_have_resolved_depth = + ice->state.dirty & (CROCUS_DIRTY_DEPTH_BUFFER | + CROCUS_DIRTY_GEN6_WM_DEPTH_STENCIL); + + struct pipe_surface *zs_surf = cso_fb->zsbuf; + if (zs_surf) { + struct crocus_resource *z_res, *s_res; + crocus_get_depth_stencil_resources(devinfo, zs_surf->texture, &z_res, &s_res); + unsigned num_layers = + zs_surf->u.tex.last_layer - zs_surf->u.tex.first_layer + 1; + + if (z_res) { + if (may_have_resolved_depth && ice->state.depth_writes_enabled) { + crocus_resource_finish_render(ice, z_res, zs_surf->u.tex.level, + zs_surf->u.tex.first_layer, num_layers, + ice->state.hiz_usage); + } + + if (ice->state.depth_writes_enabled) + crocus_depth_cache_add_bo(batch, z_res->bo); + + if (((struct crocus_surface *)zs_surf)->align_res) { + crocus_update_align_res(batch, (struct crocus_surface *)zs_surf, false); + } + } + + if (s_res) { + if (may_have_resolved_depth && ice->state.stencil_writes_enabled) { + crocus_resource_finish_write(ice, s_res, zs_surf->u.tex.level, + zs_surf->u.tex.first_layer, num_layers, + s_res->aux.usage); + } + + if (ice->state.stencil_writes_enabled) + crocus_depth_cache_add_bo(batch, s_res->bo); + } + } + + bool may_have_resolved_color = + ice->state.stage_dirty & CROCUS_STAGE_DIRTY_BINDINGS_FS; + + for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) { + struct crocus_surface *surf = (void *) cso_fb->cbufs[i]; + if (!surf) + continue; + + if (surf->align_res) + crocus_update_align_res(batch, surf, false); + struct crocus_resource *res = (void *) surf->base.texture; + enum isl_aux_usage aux_usage = ice->state.draw_aux_usage[i]; + + crocus_render_cache_add_bo(batch, res->bo, surf->view.format, + aux_usage); + + if (may_have_resolved_color) { + union pipe_surface_desc *desc = &surf->base.u; + unsigned num_layers = + desc->tex.last_layer - desc->tex.first_layer + 1; + crocus_resource_finish_render(ice, res, desc->tex.level, + desc->tex.first_layer, num_layers, + aux_usage); + } + } +} + +/** + * Clear the cache-tracking sets. + */ +void +crocus_cache_sets_clear(struct crocus_batch *batch) +{ + hash_table_foreach(batch->cache.render, render_entry) + _mesa_hash_table_remove(batch->cache.render, render_entry); + + set_foreach(batch->cache.depth, depth_entry) + _mesa_set_remove(batch->cache.depth, depth_entry); +} + +/** + * Emits an appropriate flush for a BO if it has been rendered to within the + * same batchbuffer as a read that's about to be emitted. + * + * The GPU has separate, incoherent caches for the render cache and the + * sampler cache, along with other caches. Usually data in the different + * caches don't interact (e.g. we don't render to our driver-generated + * immediate constant data), but for render-to-texture in FBOs we definitely + * do. When a batchbuffer is flushed, the kernel will ensure that everything + * necessary is flushed before another use of that BO, but for reuse from + * different caches within a batchbuffer, it's all our responsibility. + */ +void +crocus_flush_depth_and_render_caches(struct crocus_batch *batch) +{ + const struct intel_device_info *devinfo = &batch->screen->devinfo; + if (devinfo->ver >= 6) { + crocus_emit_pipe_control_flush(batch, + "cache tracker: render-to-texture", + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_RENDER_TARGET_FLUSH | + PIPE_CONTROL_CS_STALL); + + crocus_emit_pipe_control_flush(batch, + "cache tracker: render-to-texture", + PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | + PIPE_CONTROL_CONST_CACHE_INVALIDATE); + } else { + crocus_emit_mi_flush(batch); + } + + crocus_cache_sets_clear(batch); +} + +void +crocus_cache_flush_for_read(struct crocus_batch *batch, + struct crocus_bo *bo) +{ + if (_mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo) || + _mesa_set_search_pre_hashed(batch->cache.depth, bo->hash, bo)) + crocus_flush_depth_and_render_caches(batch); +} + +static void * +format_aux_tuple(enum isl_format format, enum isl_aux_usage aux_usage) +{ + return (void *)(uintptr_t)((uint32_t)format << 8 | aux_usage); +} + +void +crocus_cache_flush_for_render(struct crocus_batch *batch, + struct crocus_bo *bo, + enum isl_format format, + enum isl_aux_usage aux_usage) +{ + if (_mesa_set_search_pre_hashed(batch->cache.depth, bo->hash, bo)) + crocus_flush_depth_and_render_caches(batch); + + /* Check to see if this bo has been used by a previous rendering operation + * but with a different format or aux usage. If it has, flush the render + * cache so we ensure that it's only in there with one format or aux usage + * at a time. + * + * Even though it's not obvious, this can easily happen in practice. + * Suppose a client is blending on a surface with sRGB encode enabled on + * gen9. This implies that you get AUX_USAGE_CCS_D at best. If the client + * then disables sRGB decode and continues blending we will flip on + * AUX_USAGE_CCS_E without doing any sort of resolve in-between (this is + * perfectly valid since CCS_E is a subset of CCS_D). However, this means + * that we have fragments in-flight which are rendering with UNORM+CCS_E + * and other fragments in-flight with SRGB+CCS_D on the same surface at the + * same time and the pixel scoreboard and color blender are trying to sort + * it all out. This ends badly (i.e. GPU hangs). + * + * To date, we have never observed GPU hangs or even corruption to be + * associated with switching the format, only the aux usage. However, + * there are comments in various docs which indicate that the render cache + * isn't 100% resilient to format changes. We may as well be conservative + * and flush on format changes too. We can always relax this later if we + * find it to be a performance problem. + */ + struct hash_entry *entry = + _mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo); + if (entry && entry->data != format_aux_tuple(format, aux_usage)) + crocus_flush_depth_and_render_caches(batch); +} + +void +crocus_render_cache_add_bo(struct crocus_batch *batch, + struct crocus_bo *bo, + enum isl_format format, + enum isl_aux_usage aux_usage) +{ +#ifndef NDEBUG + struct hash_entry *entry = + _mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo); + if (entry) { + /* Otherwise, someone didn't do a flush_for_render and that would be + * very bad indeed. + */ + assert(entry->data == format_aux_tuple(format, aux_usage)); + } +#endif + + _mesa_hash_table_insert_pre_hashed(batch->cache.render, bo->hash, bo, + format_aux_tuple(format, aux_usage)); +} + +void +crocus_cache_flush_for_depth(struct crocus_batch *batch, + struct crocus_bo *bo) +{ + if (_mesa_hash_table_search_pre_hashed(batch->cache.render, bo->hash, bo)) + crocus_flush_depth_and_render_caches(batch); +} + +void +crocus_depth_cache_add_bo(struct crocus_batch *batch, struct crocus_bo *bo) +{ + _mesa_set_add_pre_hashed(batch->cache.depth, bo->hash, bo); +} + +static void +crocus_resolve_color(struct crocus_context *ice, + struct crocus_batch *batch, + struct crocus_resource *res, + unsigned level, unsigned layer, + enum isl_aux_op resolve_op) +{ + struct crocus_screen *screen = batch->screen; + DBG("%s to res %p level %u layer %u\n", __func__, res, level, layer); + + struct blorp_surf surf; + crocus_blorp_surf_for_resource(&screen->vtbl, &batch->screen->isl_dev, &surf, + &res->base, res->aux.usage, level, true); + + crocus_batch_maybe_flush(batch, 1500); + + /* Ivybridge PRM Vol 2, Part 1, "11.7 MCS Buffer for Render Target(s)": + * + * "Any transition from any value in {Clear, Render, Resolve} to a + * different value in {Clear, Render, Resolve} requires end of pipe + * synchronization." + * + * In other words, fast clear ops are not properly synchronized with + * other drawing. We need to use a PIPE_CONTROL to ensure that the + * contents of the previous draw hit the render target before we resolve + * and again afterwards to ensure that the resolve is complete before we + * do any more regular drawing. + */ + crocus_emit_end_of_pipe_sync(batch, "color resolve: pre-flush", + PIPE_CONTROL_RENDER_TARGET_FLUSH); + + struct blorp_batch blorp_batch; + blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0); + blorp_ccs_resolve(&blorp_batch, &surf, level, layer, 1, + isl_format_srgb_to_linear(res->surf.format), + resolve_op); + blorp_batch_finish(&blorp_batch); + + /* See comment above */ + crocus_emit_end_of_pipe_sync(batch, "color resolve: post-flush", + PIPE_CONTROL_RENDER_TARGET_FLUSH); +} + +static void +crocus_mcs_partial_resolve(struct crocus_context *ice, + struct crocus_batch *batch, + struct crocus_resource *res, + uint32_t start_layer, + uint32_t num_layers) +{ + struct crocus_screen *screen = batch->screen; + + DBG("%s to res %p layers %u-%u\n", __func__, res, + start_layer, start_layer + num_layers - 1); + + assert(isl_aux_usage_has_mcs(res->aux.usage)); + + struct blorp_surf surf; + crocus_blorp_surf_for_resource(&screen->vtbl, &batch->screen->isl_dev, &surf, + &res->base, res->aux.usage, 0, true); + + struct blorp_batch blorp_batch; + blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0); + blorp_mcs_partial_resolve(&blorp_batch, &surf, + isl_format_srgb_to_linear(res->surf.format), + start_layer, num_layers); + blorp_batch_finish(&blorp_batch); +} + +/** + * Perform a HiZ or depth resolve operation. + * + * For an overview of HiZ ops, see the following sections of the Sandy Bridge + * PRM, Volume 1, Part 2: + * - 7.5.3.1 Depth Buffer Clear + * - 7.5.3.2 Depth Buffer Resolve + * - 7.5.3.3 Hierarchical Depth Buffer Resolve + */ +void +crocus_hiz_exec(struct crocus_context *ice, + struct crocus_batch *batch, + struct crocus_resource *res, + unsigned int level, unsigned int start_layer, + unsigned int num_layers, enum isl_aux_op op, + bool update_clear_depth) +{ + struct crocus_screen *screen = batch->screen; + const struct intel_device_info *devinfo = &batch->screen->devinfo; + assert(crocus_resource_level_has_hiz(res, level)); + assert(op != ISL_AUX_OP_NONE); + UNUSED const char *name = NULL; + + switch (op) { + case ISL_AUX_OP_FULL_RESOLVE: + name = "depth resolve"; + break; + case ISL_AUX_OP_AMBIGUATE: + name = "hiz ambiguate"; + break; + case ISL_AUX_OP_FAST_CLEAR: + name = "depth clear"; + break; + case ISL_AUX_OP_PARTIAL_RESOLVE: + case ISL_AUX_OP_NONE: + unreachable("Invalid HiZ op"); + } + + DBG("%s %s to res %p level %d layers %d-%d\n", + __func__, name, res, level, start_layer, start_layer + num_layers - 1); + + /* The following stalls and flushes are only documented to be required + * for HiZ clear operations. However, they also seem to be required for + * resolve operations. + * + * From the Ivybridge PRM, volume 2, "Depth Buffer Clear": + * + * "If other rendering operations have preceded this clear, a + * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit + * enabled must be issued before the rectangle primitive used for + * the depth buffer clear operation." + * + * Same applies for Gen8 and Gen9. + * + * In addition, from the Ivybridge PRM, volume 2, 1.10.4.1 + * PIPE_CONTROL, Depth Cache Flush Enable: + * + * "This bit must not be set when Depth Stall Enable bit is set in + * this packet." + * + * This is confirmed to hold for real, Haswell gets immediate gpu hangs. + * + * Therefore issue two pipe control flushes, one for cache flush and + * another for depth stall. + */ + if (devinfo->ver == 6) { + /* From the Sandy Bridge PRM, volume 2 part 1, page 313: + * + * "If other rendering operations have preceded this clear, a + * PIPE_CONTROL with write cache flush enabled and Z-inhibit + * disabled must be issued before the rectangle primitive used for + * the depth buffer clear operation. + */ + crocus_emit_pipe_control_flush(batch, + "hiz op: pre-flushes (1)", + PIPE_CONTROL_RENDER_TARGET_FLUSH | + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_CS_STALL); + } else if (devinfo->ver >= 7) { + crocus_emit_pipe_control_flush(batch, + "hiz op: pre-flushes (1/2)", + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_CS_STALL); + crocus_emit_pipe_control_flush(batch, "hiz op: pre-flushes (2/2)", + PIPE_CONTROL_DEPTH_STALL); + } + + assert(isl_aux_usage_has_hiz(res->aux.usage) && res->aux.bo); + + crocus_batch_maybe_flush(batch, 1500); + + struct blorp_surf surf; + crocus_blorp_surf_for_resource(&screen->vtbl, &batch->screen->isl_dev, &surf, + &res->base, res->aux.usage, level, true); + + struct blorp_batch blorp_batch; + enum blorp_batch_flags flags = 0; + flags |= update_clear_depth ? 0 : BLORP_BATCH_NO_UPDATE_CLEAR_COLOR; + blorp_batch_init(&ice->blorp, &blorp_batch, batch, flags); + blorp_hiz_op(&blorp_batch, &surf, level, start_layer, num_layers, op); + blorp_batch_finish(&blorp_batch); + + /* The following stalls and flushes are only documented to be required + * for HiZ clear operations. However, they also seem to be required for + * resolve operations. + * + * From the Broadwell PRM, volume 7, "Depth Buffer Clear": + * + * "Depth buffer clear pass using any of the methods (WM_STATE, + * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a + * PIPE_CONTROL command with DEPTH_STALL bit and Depth FLUSH bits + * "set" before starting to render. DepthStall and DepthFlush are + * not needed between consecutive depth clear passes nor is it + * required if the depth clear pass was done with + * 'full_surf_clear' bit set in the 3DSTATE_WM_HZ_OP." + * + * TODO: Such as the spec says, this could be conditional. + */ + if (devinfo->ver == 6) { + /* From the Sandy Bridge PRM, volume 2 part 1, page 314: + * + * "DevSNB, DevSNB-B{W/A}]: Depth buffer clear pass must be + * followed by a PIPE_CONTROL command with DEPTH_STALL bit set + * and Then followed by Depth FLUSH' + */ + crocus_emit_pipe_control_flush(batch, + "hiz op: post-flushes (1/2)", + PIPE_CONTROL_DEPTH_STALL); + + crocus_emit_pipe_control_flush(batch, + "hiz op: post-flushes (2/2)", + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + PIPE_CONTROL_CS_STALL); + } +} + +/** + * Does the resource's slice have hiz enabled? + */ +bool +crocus_resource_level_has_hiz(const struct crocus_resource *res, uint32_t level) +{ + crocus_resource_check_level_layer(res, level, 0); + return res->aux.has_hiz & 1 << level; +} + +static bool +crocus_resource_level_has_aux(const struct crocus_resource *res, uint32_t level) +{ + if (isl_aux_usage_has_hiz(res->aux.usage)) + return crocus_resource_level_has_hiz(res, level); + else + return level < res->aux.surf.levels; +} + +/** \brief Assert that the level and layer are valid for the resource. */ +void +crocus_resource_check_level_layer(UNUSED const struct crocus_resource *res, + UNUSED uint32_t level, UNUSED uint32_t layer) +{ + assert(level < res->surf.levels); + assert(layer < util_num_layers(&res->base, level)); +} + +static inline uint32_t +miptree_level_range_length(const struct crocus_resource *res, + uint32_t start_level, uint32_t num_levels) +{ + assert(start_level < res->surf.levels); + + if (num_levels == INTEL_REMAINING_LAYERS) + num_levels = res->surf.levels; + + /* Check for overflow */ + assert(start_level + num_levels >= start_level); + assert(start_level + num_levels <= res->surf.levels); + + return num_levels; +} + +static inline uint32_t +miptree_layer_range_length(const struct crocus_resource *res, uint32_t level, + uint32_t start_layer, uint32_t num_layers) +{ + assert(level <= res->base.last_level); + + const uint32_t total_num_layers = crocus_get_num_logical_layers(res, level); + assert(start_layer < total_num_layers); + if (num_layers == INTEL_REMAINING_LAYERS) + num_layers = total_num_layers - start_layer; + /* Check for overflow */ + assert(start_layer + num_layers >= start_layer); + assert(start_layer + num_layers <= total_num_layers); + + return num_layers; +} + +bool +crocus_has_invalid_primary(const struct crocus_resource *res, + unsigned start_level, unsigned num_levels, + unsigned start_layer, unsigned num_layers) +{ + if (!res->aux.bo) + return false; + + /* Clamp the level range to fit the resource */ + num_levels = miptree_level_range_length(res, start_level, num_levels); + + for (uint32_t l = 0; l < num_levels; l++) { + const uint32_t level = start_level + l; + if (!crocus_resource_level_has_aux(res, level)) + continue; + + const uint32_t level_layers = + miptree_layer_range_length(res, level, start_layer, num_layers); + for (unsigned a = 0; a < level_layers; a++) { + enum isl_aux_state aux_state = + crocus_resource_get_aux_state(res, level, start_layer + a); + if (!isl_aux_state_has_valid_primary(aux_state)) + return true; + } + } + + return false; +} + +void +crocus_resource_prepare_access(struct crocus_context *ice, + struct crocus_resource *res, + uint32_t start_level, uint32_t num_levels, + uint32_t start_layer, uint32_t num_layers, + enum isl_aux_usage aux_usage, + bool fast_clear_supported) +{ + if (!res->aux.bo) + return; + + /* We can't do resolves on the compute engine, so awkwardly, we have to + * do them on the render batch... + */ + struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER]; + + const uint32_t clamped_levels = + miptree_level_range_length(res, start_level, num_levels); + for (uint32_t l = 0; l < clamped_levels; l++) { + const uint32_t level = start_level + l; + if (!crocus_resource_level_has_aux(res, level)) + continue; + + const uint32_t level_layers = + miptree_layer_range_length(res, level, start_layer, num_layers); + for (uint32_t a = 0; a < level_layers; a++) { + const uint32_t layer = start_layer + a; + const enum isl_aux_state aux_state = + crocus_resource_get_aux_state(res, level, layer); + const enum isl_aux_op aux_op = + isl_aux_prepare_access(aux_state, aux_usage, fast_clear_supported); + + /* Prepare the aux buffer for a conditional or unconditional access. + * A conditional access is handled by assuming that the access will + * not evaluate to a no-op. If the access does in fact occur, the aux + * will be in the required state. If it does not, no data is lost + * because the aux_op performed is lossless. + */ + if (aux_op == ISL_AUX_OP_NONE) { + /* Nothing to do here. */ + } else if (isl_aux_usage_has_mcs(res->aux.usage)) { + assert(aux_op == ISL_AUX_OP_PARTIAL_RESOLVE); + crocus_mcs_partial_resolve(ice, batch, res, layer, 1); + } else if (isl_aux_usage_has_hiz(res->aux.usage)) { + crocus_hiz_exec(ice, batch, res, level, layer, 1, aux_op, false); + } else if (res->aux.usage == ISL_AUX_USAGE_STC_CCS) { + unreachable("crocus doesn't resolve STC_CCS resources"); + } else { + assert(isl_aux_usage_has_ccs(res->aux.usage)); + crocus_resolve_color(ice, batch, res, level, layer, aux_op); + } + + const enum isl_aux_state new_state = + isl_aux_state_transition_aux_op(aux_state, res->aux.usage, aux_op); + crocus_resource_set_aux_state(ice, res, level, layer, 1, new_state); + } + } +} + +void +crocus_resource_finish_write(struct crocus_context *ice, + struct crocus_resource *res, uint32_t level, + uint32_t start_layer, uint32_t num_layers, + enum isl_aux_usage aux_usage) +{ + if (res->base.format == PIPE_FORMAT_S8_UINT) + res->shadow_needs_update = true; + + if (!crocus_resource_level_has_aux(res, level)) + return; + + const uint32_t level_layers = + miptree_layer_range_length(res, level, start_layer, num_layers); + + for (uint32_t a = 0; a < level_layers; a++) { + const uint32_t layer = start_layer + a; + const enum isl_aux_state aux_state = + crocus_resource_get_aux_state(res, level, layer); + + /* Transition the aux state for a conditional or unconditional write. A + * conditional write is handled by assuming that the write applies to + * only part of the render target. This prevents the new state from + * losing the types of compression that might exist in the current state + * (e.g. CLEAR). If the write evaluates to a no-op, the state will still + * be able to communicate when resolves are necessary (but it may + * falsely communicate this as well). + */ + const enum isl_aux_state new_aux_state = + isl_aux_state_transition_write(aux_state, aux_usage, false); + + crocus_resource_set_aux_state(ice, res, level, layer, 1, new_aux_state); + } +} + +enum isl_aux_state +crocus_resource_get_aux_state(const struct crocus_resource *res, + uint32_t level, uint32_t layer) +{ + crocus_resource_check_level_layer(res, level, layer); + assert(crocus_resource_level_has_aux(res, level)); + + return res->aux.state[level][layer]; +} + +void +crocus_resource_set_aux_state(struct crocus_context *ice, + struct crocus_resource *res, uint32_t level, + uint32_t start_layer, uint32_t num_layers, + enum isl_aux_state aux_state) +{ + assert(crocus_resource_level_has_aux(res, level)); + + num_layers = miptree_layer_range_length(res, level, start_layer, num_layers); + for (unsigned a = 0; a < num_layers; a++) { + if (res->aux.state[level][start_layer + a] != aux_state) { + res->aux.state[level][start_layer + a] = aux_state; + ice->state.dirty |= CROCUS_DIRTY_RENDER_RESOLVES_AND_FLUSHES | + CROCUS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES; + /* XXX: Need to track which bindings to make dirty */ + ice->state.stage_dirty |= CROCUS_ALL_STAGE_DIRTY_BINDINGS; + } + } +} + +static bool +isl_formats_are_fast_clear_compatible(enum isl_format a, enum isl_format b) +{ + /* On gen8 and earlier, the hardware was only capable of handling 0/1 clear + * values so sRGB curve application was a no-op for all fast-clearable + * formats. + * + * On gen9+, the hardware supports arbitrary clear values. For sRGB clear + * values, the hardware interprets the floats, not as what would be + * returned from the sampler (or written by the shader), but as being + * between format conversion and sRGB curve application. This means that + * we can switch between sRGB and UNORM without having to whack the clear + * color. + */ + return isl_format_srgb_to_linear(a) == isl_format_srgb_to_linear(b); +} + +void +crocus_resource_prepare_texture(struct crocus_context *ice, + struct crocus_resource *res, + enum isl_format view_format, + uint32_t start_level, uint32_t num_levels, + uint32_t start_layer, uint32_t num_layers) +{ + enum isl_aux_usage aux_usage = + crocus_resource_texture_aux_usage(res); + + bool clear_supported = aux_usage != ISL_AUX_USAGE_NONE; + + /* Clear color is specified as ints or floats and the conversion is done by + * the sampler. If we have a texture view, we would have to perform the + * clear color conversion manually. Just disable clear color. + */ + if (!isl_formats_are_fast_clear_compatible(res->surf.format, view_format)) + clear_supported = false; + + crocus_resource_prepare_access(ice, res, start_level, num_levels, + start_layer, num_layers, + aux_usage, clear_supported); +} + +enum isl_aux_usage +crocus_resource_render_aux_usage(struct crocus_context *ice, + struct crocus_resource *res, + enum isl_format render_format, + bool blend_enabled, + bool draw_aux_disabled) +{ + struct crocus_screen *screen = (void *) ice->ctx.screen; + struct intel_device_info *devinfo = &screen->devinfo; + + if (draw_aux_disabled) + return ISL_AUX_USAGE_NONE; + + switch (res->aux.usage) { + case ISL_AUX_USAGE_MCS: + return res->aux.usage; + + case ISL_AUX_USAGE_CCS_D: + /* Otherwise, we try to fall back to CCS_D */ + if (isl_format_supports_ccs_d(devinfo, render_format)) + return ISL_AUX_USAGE_CCS_D; + + return ISL_AUX_USAGE_NONE; + + default: + return ISL_AUX_USAGE_NONE; + } +} + +void +crocus_resource_prepare_render(struct crocus_context *ice, + struct crocus_resource *res, uint32_t level, + uint32_t start_layer, uint32_t layer_count, + enum isl_aux_usage aux_usage) +{ + crocus_resource_prepare_access(ice, res, level, 1, start_layer, + layer_count, aux_usage, + aux_usage != ISL_AUX_USAGE_NONE); +} + +void +crocus_resource_finish_render(struct crocus_context *ice, + struct crocus_resource *res, uint32_t level, + uint32_t start_layer, uint32_t layer_count, + enum isl_aux_usage aux_usage) +{ + crocus_resource_finish_write(ice, res, level, start_layer, layer_count, + aux_usage); +} + +static void +crocus_update_stencil_shadow(struct crocus_context *ice, + struct crocus_resource *res) +{ + struct crocus_screen *screen = (struct crocus_screen *)ice->ctx.screen; + UNUSED const struct intel_device_info *devinfo = &screen->devinfo; + assert(devinfo->ver >= 7); + + if (!res->shadow_needs_update) + return; + + struct pipe_box box; + for (unsigned level = 0; level <= res->base.last_level; level++) { + u_box_2d(0, 0, + u_minify(res->base.width0, level), + u_minify(res->base.height0, level), &box); + const unsigned depth = res->base.target == PIPE_TEXTURE_3D ? + u_minify(res->base.depth0, level) : res->base.array_size; + + for (unsigned layer = 0; layer < depth; layer++) { + box.z = layer; + ice->ctx.resource_copy_region(&ice->ctx, + &res->shadow->base, level, 0, 0, layer, + &res->base, level, &box); + } + } + res->shadow_needs_update = false; +} |