diff options
author | Anuj Phogat <anuj.phogat@gmail.com> | 2021-03-03 09:27:36 -0800 |
---|---|---|
committer | Anuj Phogat <anuj.phogat@gmail.com> | 2021-03-11 10:14:33 -0800 |
commit | 9d95e1bd79a8b6a94e371b808f61d415d0ec1372 (patch) | |
tree | f5ec795fd1f064b05c131baea5b833a53c60512b /src/mesa/drivers/dri/i965/intel_mipmap_tree.c | |
parent | 3096788e5ca9f34d9a5c88f87645ff137dbe1e59 (diff) |
i965: Rename files with "intel_" prefix to "brw_"
v2: Rename intel_batchbuffer.c to intel_batch.c and
intel_batchbuffer.h to intel_batch.h
Signed-off-by: Anuj Phogat <anuj.phogat@gmail.com>
Reviewed-by: Kenneth Graunke <kenneth@whitecape.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9510>
Diffstat (limited to 'src/mesa/drivers/dri/i965/intel_mipmap_tree.c')
-rw-r--r-- | src/mesa/drivers/dri/i965/intel_mipmap_tree.c | 3295 |
1 files changed, 0 insertions, 3295 deletions
diff --git a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c b/src/mesa/drivers/dri/i965/intel_mipmap_tree.c deleted file mode 100644 index 3584c94b4a8..00000000000 --- a/src/mesa/drivers/dri/i965/intel_mipmap_tree.c +++ /dev/null @@ -1,3295 +0,0 @@ -/* - * Copyright 2006 VMware, Inc. - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR - * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include <GL/gl.h> -#include <GL/internal/dri_interface.h> -#include "drm-uapi/drm_fourcc.h" - -#include "intel_batchbuffer.h" -#include "intel_image.h" -#include "intel_mipmap_tree.h" -#include "intel_tex.h" -#include "intel_blit.h" -#include "intel_fbo.h" - -#include "brw_blorp.h" -#include "brw_context.h" -#include "brw_state.h" - -#include "main/enums.h" -#include "main/fbobject.h" -#include "main/formats.h" -#include "main/glformats.h" -#include "main/texcompress_etc.h" -#include "main/teximage.h" -#include "main/streaming-load-memcpy.h" - -#include "util/format_srgb.h" -#include "util/u_memory.h" - -#include "x86/common_x86_asm.h" - -#define FILE_DEBUG_FLAG DEBUG_MIPTREE - -static void *brw_miptree_map_raw(struct brw_context *brw, - struct brw_mipmap_tree *mt, - GLbitfield mode); - -static void brw_miptree_unmap_raw(struct brw_mipmap_tree *mt); - -/** - * Return true if the format that will be used to access the miptree is - * CCS_E-compatible with the miptree's linear/non-sRGB format. - * - * Why use the linear format? Well, although the miptree may be specified with - * an sRGB format, the usage of that color space/format can be toggled. Since - * our HW tends to support more linear formats than sRGB ones, we use this - * format variant for check for CCS_E compatibility. - */ -static bool -format_ccs_e_compat_with_miptree(const struct gen_device_info *devinfo, - const struct brw_mipmap_tree *mt, - enum isl_format access_format) -{ - assert(mt->aux_usage == ISL_AUX_USAGE_CCS_E); - - mesa_format linear_format = _mesa_get_srgb_format_linear(mt->format); - enum isl_format isl_format = brw_isl_format_for_mesa_format(linear_format); - return isl_formats_are_ccs_e_compatible(devinfo, isl_format, access_format); -} - -/* Determine if CCS_E is supported for a given platform and mesa format. */ -static bool -format_supports_ccs_e(const struct brw_context *brw, mesa_format format) -{ - /* For now compression is only enabled for integer formats even though - * there exist supported floating point formats also. This is a heuristic - * decision based on current public benchmarks. In none of the cases these - * formats provided any improvement but a few cases were seen to regress. - * Hence these are left to to be enabled in the future when they are known - * to improve things. - */ - if (_mesa_get_format_datatype(format) == GL_FLOAT) - return false; - - /* Many window system buffers are sRGB even if they are never rendered as - * sRGB. For those, we want CCS_E for when sRGBEncode is false. When the - * surface is used as sRGB, we fall back to CCS_D. - */ - mesa_format linear_format = _mesa_get_srgb_format_linear(format); - enum isl_format isl_format = brw_isl_format_for_mesa_format(linear_format); - return isl_format_supports_ccs_e(&brw->screen->devinfo, isl_format); -} - -/** - * Determine depth format corresponding to a depth+stencil format, - * for separate stencil. - */ -mesa_format -brw_depth_format_for_depthstencil_format(mesa_format format) { - switch (format) { - case MESA_FORMAT_Z24_UNORM_S8_UINT: - return MESA_FORMAT_Z24_UNORM_X8_UINT; - case MESA_FORMAT_Z32_FLOAT_S8X24_UINT: - return MESA_FORMAT_Z_FLOAT32; - default: - return format; - } -} - -static bool -create_mapping_table(GLenum target, unsigned first_level, unsigned last_level, - unsigned depth0, struct brw_mipmap_level *table) -{ - for (unsigned level = first_level; level <= last_level; level++) { - const unsigned d = - target == GL_TEXTURE_3D ? minify(depth0, level) : depth0; - - table[level].slice = calloc(d, sizeof(*table[0].slice)); - if (!table[level].slice) - goto unwind; - } - - return true; - -unwind: - for (unsigned level = first_level; level <= last_level; level++) - free(table[level].slice); - - return false; -} - -static bool -needs_separate_stencil(const struct brw_context *brw, - struct brw_mipmap_tree *mt, - mesa_format format) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - if (_mesa_get_format_base_format(format) != GL_DEPTH_STENCIL) - return false; - - if (devinfo->must_use_separate_stencil) - return true; - - return brw->has_separate_stencil && brw->has_hiz; -} - -/** - * Choose the aux usage for this miptree. This function must be called fairly - * late in the miptree create process after we have a tiling. - */ -static void -brw_miptree_choose_aux_usage(struct brw_context *brw, - struct brw_mipmap_tree *mt) -{ - assert(mt->aux_usage == ISL_AUX_USAGE_NONE); - - if (_mesa_is_format_color_format(mt->format)) { - if (mt->surf.samples > 1) { - mt->aux_usage = ISL_AUX_USAGE_MCS; - } else if (!(INTEL_DEBUG & DEBUG_NO_RBC) && - format_supports_ccs_e(brw, mt->format)) { - mt->aux_usage = ISL_AUX_USAGE_CCS_E; - } else if (brw->mesa_format_supports_render[mt->format]) { - mt->aux_usage = ISL_AUX_USAGE_CCS_D; - } - } else if (isl_surf_usage_is_depth(mt->surf.usage) && brw->has_hiz) { - mt->aux_usage = ISL_AUX_USAGE_HIZ; - } - - /* We can do fast-clear on all auxiliary surface types that are - * allocated through the normal texture creation paths. - */ - if (mt->aux_usage != ISL_AUX_USAGE_NONE) - mt->supports_fast_clear = true; -} - - -/** - * Choose an appropriate uncompressed format for a requested - * compressed format, if unsupported. - */ -mesa_format -brw_lower_compressed_format(struct brw_context *brw, mesa_format format) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - /* No need to lower ETC formats on these platforms, - * they are supported natively. - */ - if (devinfo->gen >= 8 || devinfo->is_baytrail) - return format; - - switch (format) { - case MESA_FORMAT_ETC1_RGB8: - return MESA_FORMAT_R8G8B8X8_UNORM; - case MESA_FORMAT_ETC2_RGB8: - return MESA_FORMAT_R8G8B8X8_UNORM; - case MESA_FORMAT_ETC2_SRGB8: - case MESA_FORMAT_ETC2_SRGB8_ALPHA8_EAC: - case MESA_FORMAT_ETC2_SRGB8_PUNCHTHROUGH_ALPHA1: - return MESA_FORMAT_B8G8R8A8_SRGB; - case MESA_FORMAT_ETC2_RGBA8_EAC: - case MESA_FORMAT_ETC2_RGB8_PUNCHTHROUGH_ALPHA1: - return MESA_FORMAT_R8G8B8A8_UNORM; - case MESA_FORMAT_ETC2_R11_EAC: - return MESA_FORMAT_R_UNORM16; - case MESA_FORMAT_ETC2_SIGNED_R11_EAC: - return MESA_FORMAT_R_SNORM16; - case MESA_FORMAT_ETC2_RG11_EAC: - return MESA_FORMAT_RG_UNORM16; - case MESA_FORMAT_ETC2_SIGNED_RG11_EAC: - return MESA_FORMAT_RG_SNORM16; - default: - /* Non ETC1 / ETC2 format */ - return format; - } -} - -unsigned -brw_get_num_logical_layers(const struct brw_mipmap_tree *mt, unsigned level) -{ - if (mt->surf.dim == ISL_SURF_DIM_3D) - return minify(mt->surf.logical_level0_px.depth, level); - else - return mt->surf.logical_level0_px.array_len; -} - -UNUSED static unsigned -get_num_phys_layers(const struct isl_surf *surf, unsigned level) -{ - /* In case of physical dimensions one needs to consider also the layout. - * See isl_calc_phys_level0_extent_sa(). - */ - if (surf->dim != ISL_SURF_DIM_3D) - return surf->phys_level0_sa.array_len; - - if (surf->dim_layout == ISL_DIM_LAYOUT_GEN4_2D) - return minify(surf->phys_level0_sa.array_len, level); - - return minify(surf->phys_level0_sa.depth, level); -} - -/** \brief Assert that the level and layer are valid for the miptree. */ -void -brw_miptree_check_level_layer(const struct brw_mipmap_tree *mt, - uint32_t level, - uint32_t layer) -{ - (void) mt; - (void) level; - (void) layer; - - assert(level >= mt->first_level); - assert(level <= mt->last_level); - assert(layer < get_num_phys_layers(&mt->surf, level)); -} - -static enum isl_aux_state ** -create_aux_state_map(struct brw_mipmap_tree *mt, - enum isl_aux_state initial) -{ - const uint32_t levels = mt->last_level + 1; - - uint32_t total_slices = 0; - for (uint32_t level = 0; level < levels; level++) - total_slices += brw_get_num_logical_layers(mt, level); - - const size_t per_level_array_size = levels * sizeof(enum isl_aux_state *); - - /* We're going to allocate a single chunk of data for both the per-level - * reference array and the arrays of aux_state. This makes cleanup - * significantly easier. - */ - const size_t total_size = per_level_array_size + - total_slices * sizeof(enum isl_aux_state); - void *data = malloc(total_size); - if (data == NULL) - return NULL; - - enum isl_aux_state **per_level_arr = data; - enum isl_aux_state *s = data + per_level_array_size; - for (uint32_t level = 0; level < levels; level++) { - per_level_arr[level] = s; - const unsigned level_layers = brw_get_num_logical_layers(mt, level); - for (uint32_t a = 0; a < level_layers; a++) - *(s++) = initial; - } - assert((void *)s == data + total_size); - - return per_level_arr; -} - -static void -free_aux_state_map(enum isl_aux_state **state) -{ - free(state); -} - -static bool -need_to_retile_as_linear(struct brw_context *brw, unsigned blt_pitch, - enum isl_tiling tiling, unsigned samples) -{ - if (samples > 1) - return false; - - if (tiling == ISL_TILING_LINEAR) - return false; - - if (blt_pitch >= 32768) { - perf_debug("blt pitch %u too large to blit, falling back to untiled", - blt_pitch); - return true; - } - - return false; -} - -static bool -need_to_retile_as_x(const struct brw_context *brw, uint64_t size, - enum isl_tiling tiling) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - /* If the BO is too large to fit in the aperture, we need to use the - * BLT engine to support it. Prior to Sandybridge, the BLT paths can't - * handle Y-tiling, so we need to fall back to X. - */ - if (devinfo->gen < 6 && size >= brw->max_gtt_map_object_size && - tiling == ISL_TILING_Y0) - return true; - - return false; -} - -static struct brw_mipmap_tree * -make_surface(struct brw_context *brw, GLenum target, mesa_format format, - unsigned first_level, unsigned last_level, - unsigned width0, unsigned height0, unsigned depth0, - unsigned num_samples, isl_tiling_flags_t tiling_flags, - isl_surf_usage_flags_t isl_usage_flags, uint32_t alloc_flags, - unsigned row_pitch_B, struct brw_bo *bo) -{ - struct brw_mipmap_tree *mt = calloc(sizeof(*mt), 1); - if (!mt) - return NULL; - - if (!create_mapping_table(target, first_level, last_level, depth0, - mt->level)) { - free(mt); - return NULL; - } - - mt->refcount = 1; - - if (target == GL_TEXTURE_CUBE_MAP || - target == GL_TEXTURE_CUBE_MAP_ARRAY) - isl_usage_flags |= ISL_SURF_USAGE_CUBE_BIT; - - DBG("%s: %s %s %ux %u:%u:%u %d..%d <-- %p\n", - __func__, - _mesa_enum_to_string(target), - _mesa_get_format_name(format), - num_samples, width0, height0, depth0, - first_level, last_level, mt); - - struct isl_surf_init_info init_info = { - .dim = get_isl_surf_dim(target), - .format = translate_tex_format(brw, format, false), - .width = width0, - .height = height0, - .depth = target == GL_TEXTURE_3D ? depth0 : 1, - .levels = last_level - first_level + 1, - .array_len = target == GL_TEXTURE_3D ? 1 : depth0, - .samples = num_samples, - .row_pitch_B = row_pitch_B, - .usage = isl_usage_flags, - .tiling_flags = tiling_flags, - }; - - if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info)) - goto fail; - - /* Depth surfaces are always Y-tiled and stencil is always W-tiled, although - * on gen7 platforms we also need to create Y-tiled copies of stencil for - * texturing since the hardware can't sample from W-tiled surfaces. For - * everything else, check for corner cases needing special treatment. - */ - bool is_depth_stencil = - mt->surf.usage & (ISL_SURF_USAGE_STENCIL_BIT | ISL_SURF_USAGE_DEPTH_BIT); - if (!is_depth_stencil) { - if (need_to_retile_as_linear(brw, brw_miptree_blt_pitch(mt), - mt->surf.tiling, mt->surf.samples)) { - init_info.tiling_flags = 1u << ISL_TILING_LINEAR; - if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info)) - goto fail; - } else if (need_to_retile_as_x(brw, mt->surf.size_B, mt->surf.tiling)) { - init_info.tiling_flags = 1u << ISL_TILING_X; - if (!isl_surf_init_s(&brw->isl_dev, &mt->surf, &init_info)) - goto fail; - } - } - - /* In case of linear the buffer gets padded by fixed 64 bytes and therefore - * the size may not be multiple of row_pitch. - * See isl_apply_surface_padding(). - */ - if (mt->surf.tiling != ISL_TILING_LINEAR) - assert(mt->surf.size_B % mt->surf.row_pitch_B == 0); - - if (!bo) { - mt->bo = brw_bo_alloc_tiled(brw->bufmgr, "isl-miptree", - mt->surf.size_B, - BRW_MEMZONE_OTHER, - isl_tiling_to_i915_tiling( - mt->surf.tiling), - mt->surf.row_pitch_B, alloc_flags); - if (!mt->bo) - goto fail; - } else { - mt->bo = bo; - } - - mt->first_level = first_level; - mt->last_level = last_level; - mt->target = target; - mt->format = format; - mt->aux_state = NULL; - mt->cpp = isl_format_get_layout(mt->surf.format)->bpb / 8; - mt->compressed = _mesa_is_format_compressed(format); - mt->drm_modifier = DRM_FORMAT_MOD_INVALID; - - return mt; - -fail: - brw_miptree_release(&mt); - return NULL; -} - -/* Return the usual surface usage flags for the given format. */ -static isl_surf_usage_flags_t -mt_surf_usage(mesa_format format) -{ - switch(_mesa_get_format_base_format(format)) { - case GL_DEPTH_COMPONENT: - return ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_TEXTURE_BIT; - case GL_DEPTH_STENCIL: - return ISL_SURF_USAGE_DEPTH_BIT | ISL_SURF_USAGE_STENCIL_BIT | - ISL_SURF_USAGE_TEXTURE_BIT; - case GL_STENCIL_INDEX: - return ISL_SURF_USAGE_STENCIL_BIT | ISL_SURF_USAGE_TEXTURE_BIT; - default: - return ISL_SURF_USAGE_RENDER_TARGET_BIT | ISL_SURF_USAGE_TEXTURE_BIT; - } -} - -static struct brw_mipmap_tree * -miptree_create(struct brw_context *brw, - GLenum target, - mesa_format format, - GLuint first_level, - GLuint last_level, - GLuint width0, - GLuint height0, - GLuint depth0, - GLuint num_samples, - enum brw_miptree_create_flags flags) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - const uint32_t alloc_flags = - (flags & MIPTREE_CREATE_BUSY || num_samples > 1) ? BO_ALLOC_BUSY : 0; - isl_tiling_flags_t tiling_flags = ISL_TILING_ANY_MASK; - - /* TODO: This used to be because there wasn't BLORP to handle Y-tiling. */ - if (devinfo->gen < 6 && _mesa_is_format_color_format(format)) - tiling_flags &= ~ISL_TILING_Y0_BIT; - - mesa_format mt_fmt = format; - if (!_mesa_is_format_color_format(format) && devinfo->gen >= 6) { - /* Fix up the Z miptree format for how we're splitting out separate - * stencil. Gen7 expects there to be no stencil bits in its depth buffer. - */ - mt_fmt = brw_depth_format_for_depthstencil_format(format); - } - - struct brw_mipmap_tree *mt = - make_surface(brw, target, mt_fmt, first_level, last_level, - width0, height0, depth0, num_samples, - tiling_flags, mt_surf_usage(mt_fmt), - alloc_flags, 0, NULL); - - if (mt == NULL) - return NULL; - - if (brw_miptree_needs_fake_etc(brw, mt)) { - mesa_format decomp_format = brw_lower_compressed_format(brw, format); - mt->shadow_mt = make_surface(brw, target, decomp_format, first_level, - last_level, width0, height0, depth0, - num_samples, tiling_flags, - mt_surf_usage(decomp_format), - alloc_flags, 0, NULL); - - if (mt->shadow_mt == NULL) { - brw_miptree_release(&mt); - return NULL; - } - } - - if (needs_separate_stencil(brw, mt, format)) { - mt->stencil_mt = - make_surface(brw, target, MESA_FORMAT_S_UINT8, first_level, last_level, - width0, height0, depth0, num_samples, - ISL_TILING_W_BIT, mt_surf_usage(MESA_FORMAT_S_UINT8), - alloc_flags, 0, NULL); - if (mt->stencil_mt == NULL) { - brw_miptree_release(&mt); - return NULL; - } - } - - if (!(flags & MIPTREE_CREATE_NO_AUX)) - brw_miptree_choose_aux_usage(brw, mt); - - return mt; -} - -struct brw_mipmap_tree * -brw_miptree_create(struct brw_context *brw, - GLenum target, - mesa_format format, - GLuint first_level, - GLuint last_level, - GLuint width0, - GLuint height0, - GLuint depth0, - GLuint num_samples, - enum brw_miptree_create_flags flags) -{ - assert(num_samples > 0); - - struct brw_mipmap_tree *mt = miptree_create( - brw, target, format, - first_level, last_level, - width0, height0, depth0, num_samples, - flags); - if (!mt) - return NULL; - - mt->offset = 0; - - /* Create the auxiliary surface up-front. CCS_D, on the other hand, can only - * compress clear color so we wait until an actual fast-clear to allocate - * it. - */ - if (mt->aux_usage != ISL_AUX_USAGE_CCS_D && - !brw_miptree_alloc_aux(brw, mt)) { - mt->aux_usage = ISL_AUX_USAGE_NONE; - mt->supports_fast_clear = false; - } - - return mt; -} - -struct brw_mipmap_tree * -brw_miptree_create_for_bo(struct brw_context *brw, - struct brw_bo *bo, - mesa_format format, - uint32_t offset, - uint32_t width, - uint32_t height, - uint32_t depth, - int pitch, - enum isl_tiling tiling, - enum brw_miptree_create_flags flags) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - struct brw_mipmap_tree *mt; - const GLenum target = depth > 1 ? GL_TEXTURE_2D_ARRAY : GL_TEXTURE_2D; - const GLenum base_format = _mesa_get_format_base_format(format); - - if ((base_format == GL_DEPTH_COMPONENT || - base_format == GL_DEPTH_STENCIL)) { - const mesa_format mt_fmt = (devinfo->gen < 6) ? format : - brw_depth_format_for_depthstencil_format(format); - mt = make_surface(brw, target, mt_fmt, - 0, 0, width, height, depth, 1, ISL_TILING_Y0_BIT, - mt_surf_usage(mt_fmt), - 0, pitch, bo); - if (!mt) - return NULL; - - brw_bo_reference(bo); - - if (!(flags & MIPTREE_CREATE_NO_AUX)) - brw_miptree_choose_aux_usage(brw, mt); - - return mt; - } else if (format == MESA_FORMAT_S_UINT8) { - mt = make_surface(brw, target, MESA_FORMAT_S_UINT8, - 0, 0, width, height, depth, 1, - ISL_TILING_W_BIT, - mt_surf_usage(MESA_FORMAT_S_UINT8), - 0, pitch, bo); - if (!mt) - return NULL; - - assert(bo->size >= mt->surf.size_B); - - brw_bo_reference(bo); - return mt; - } - - /* Nothing will be able to use this miptree with the BO if the offset isn't - * aligned. - */ - if (tiling != ISL_TILING_LINEAR) - assert(offset % 4096 == 0); - - /* miptrees can't handle negative pitch. If you need flipping of images, - * that's outside of the scope of the mt. - */ - assert(pitch >= 0); - - mt = make_surface(brw, target, format, - 0, 0, width, height, depth, 1, - 1lu << tiling, - mt_surf_usage(format), - 0, pitch, bo); - if (!mt) - return NULL; - - brw_bo_reference(bo); - mt->bo = bo; - mt->offset = offset; - - if (!(flags & MIPTREE_CREATE_NO_AUX)) { - brw_miptree_choose_aux_usage(brw, mt); - - /* Create the auxiliary surface up-front. CCS_D, on the other hand, can - * only compress clear color so we wait until an actual fast-clear to - * allocate it. - */ - if (mt->aux_usage != ISL_AUX_USAGE_CCS_D && - !brw_miptree_alloc_aux(brw, mt)) { - mt->aux_usage = ISL_AUX_USAGE_NONE; - mt->supports_fast_clear = false; - } - } - - return mt; -} - -static struct brw_mipmap_tree * -miptree_create_for_planar_image(struct brw_context *brw, - __DRIimage *image, GLenum target, - enum isl_tiling tiling) -{ - const struct brw_image_format *f = image->planar_format; - struct brw_mipmap_tree *planar_mt = NULL; - - for (int i = 0; i < f->nplanes; i++) { - const int index = f->planes[i].buffer_index; - const uint32_t dri_format = f->planes[i].dri_format; - const mesa_format format = driImageFormatToGLFormat(dri_format); - const uint32_t width = image->width >> f->planes[i].width_shift; - const uint32_t height = image->height >> f->planes[i].height_shift; - - /* Disable creation of the texture's aux buffers because the driver - * exposes no EGL API to manage them. That is, there is no API for - * resolving the aux buffer's content to the main buffer nor for - * invalidating the aux buffer's content. - */ - struct brw_mipmap_tree *mt = - brw_miptree_create_for_bo(brw, image->bo, format, - image->offsets[index], - width, height, 1, - image->strides[index], - tiling, - MIPTREE_CREATE_NO_AUX); - if (mt == NULL) { - brw_miptree_release(&planar_mt); - return NULL; - } - - mt->target = target; - - if (i == 0) - planar_mt = mt; - else - planar_mt->plane[i - 1] = mt; - } - - planar_mt->drm_modifier = image->modifier; - - return planar_mt; -} - -static bool -create_ccs_buf_for_image(struct brw_context *brw, - __DRIimage *image, - struct brw_mipmap_tree *mt, - enum isl_aux_state initial_state) -{ - struct isl_surf temp_ccs_surf = {0,}; - - /* CCS is only supported for very simple miptrees */ - assert(image->aux_offset != 0 && image->aux_pitch != 0); - assert(image->tile_x == 0 && image->tile_y == 0); - assert(mt->surf.samples == 1); - assert(mt->surf.levels == 1); - assert(mt->surf.logical_level0_px.depth == 1); - assert(mt->surf.logical_level0_px.array_len == 1); - assert(mt->first_level == 0); - assert(mt->last_level == 0); - - /* We shouldn't already have a CCS */ - assert(!mt->aux_buf); - - if (!isl_surf_get_ccs_surf(&brw->isl_dev, &mt->surf, &temp_ccs_surf, NULL, - image->aux_pitch)) - return false; - - assert(image->aux_offset < image->bo->size); - assert(temp_ccs_surf.size_B <= image->bo->size - image->aux_offset); - - mt->aux_buf = calloc(sizeof(*mt->aux_buf), 1); - if (mt->aux_buf == NULL) - return false; - - mt->aux_state = create_aux_state_map(mt, initial_state); - if (!mt->aux_state) { - free(mt->aux_buf); - mt->aux_buf = NULL; - return false; - } - - /* On gen10+ we start using an extra space in the aux buffer to store the - * indirect clear color. However, if we imported an image from the window - * system with CCS, we don't have the extra space at the end of the aux - * buffer. So create a new bo here that will store that clear color. - */ - if (brw->isl_dev.ss.clear_color_state_size > 0) { - mt->aux_buf->clear_color_bo = - brw_bo_alloc_tiled(brw->bufmgr, "clear_color_bo", - brw->isl_dev.ss.clear_color_state_size, - BRW_MEMZONE_OTHER, I915_TILING_NONE, 0, - BO_ALLOC_ZEROED); - if (!mt->aux_buf->clear_color_bo) { - free(mt->aux_buf); - mt->aux_buf = NULL; - return false; - } - } - - mt->aux_buf->bo = image->bo; - brw_bo_reference(image->bo); - - mt->aux_buf->offset = image->aux_offset; - mt->aux_buf->surf = temp_ccs_surf; - - return true; -} - -struct brw_mipmap_tree * -brw_miptree_create_for_dri_image(struct brw_context *brw, - __DRIimage *image, GLenum target, - mesa_format format, - bool allow_internal_aux) -{ - uint32_t bo_tiling, bo_swizzle; - brw_bo_get_tiling(image->bo, &bo_tiling, &bo_swizzle); - - const struct isl_drm_modifier_info *mod_info = - isl_drm_modifier_get_info(image->modifier); - - const enum isl_tiling tiling = - mod_info ? mod_info->tiling : isl_tiling_from_i915_tiling(bo_tiling); - - if (image->planar_format && image->planar_format->nplanes > 1) - return miptree_create_for_planar_image(brw, image, target, tiling); - - if (image->planar_format) - assert(image->planar_format->planes[0].dri_format == image->dri_format); - - if (!brw->ctx.TextureFormatSupported[format]) { - /* The texture storage paths in core Mesa detect if the driver does not - * support the user-requested format, and then searches for a - * fallback format. The DRIimage code bypasses core Mesa, though. So we - * do the fallbacks here for important formats. - * - * We must support DRM_FOURCC_XBGR8888 textures because the Android - * framework produces HAL_PIXEL_FORMAT_RGBX8888 winsys surfaces, which - * the Chrome OS compositor consumes as dma_buf EGLImages. - */ - format = _mesa_format_fallback_rgbx_to_rgba(format); - } - - if (!brw->ctx.TextureFormatSupported[format]) - return NULL; - - enum brw_miptree_create_flags mt_create_flags = 0; - - /* If this image comes in from a window system, we have different - * requirements than if it comes in via an EGL import operation. Window - * system images can use any form of auxiliary compression we wish because - * they get "flushed" before being handed off to the window system and we - * have the opportunity to do resolves. Non window-system images, on the - * other hand, have no resolve point so we can't have aux without a - * modifier. - */ - if (!allow_internal_aux) - mt_create_flags |= MIPTREE_CREATE_NO_AUX; - - /* If we have a modifier which specifies aux, don't create one yet */ - if (mod_info && mod_info->aux_usage != ISL_AUX_USAGE_NONE) - mt_create_flags |= MIPTREE_CREATE_NO_AUX; - - /* Disable creation of the texture's aux buffers because the driver exposes - * no EGL API to manage them. That is, there is no API for resolving the aux - * buffer's content to the main buffer nor for invalidating the aux buffer's - * content. - */ - struct brw_mipmap_tree *mt = - brw_miptree_create_for_bo(brw, image->bo, format, - image->offset, image->width, image->height, 1, - image->pitch, tiling, mt_create_flags); - if (mt == NULL) - return NULL; - - mt->target = target; - mt->level[0].level_x = image->tile_x; - mt->level[0].level_y = image->tile_y; - mt->drm_modifier = image->modifier; - - /* From "OES_EGL_image" error reporting. We report GL_INVALID_OPERATION - * for EGL images from non-tile aligned sufaces in gen4 hw and earlier which has - * trouble resolving back to destination image due to alignment issues. - */ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - if (!devinfo->has_surface_tile_offset) { - uint32_t draw_x, draw_y; - brw_miptree_get_tile_offsets(mt, 0, 0, &draw_x, &draw_y); - - if (draw_x != 0 || draw_y != 0) { - _mesa_error(&brw->ctx, GL_INVALID_OPERATION, __func__); - brw_miptree_release(&mt); - return NULL; - } - } - - if (mod_info && mod_info->aux_usage != ISL_AUX_USAGE_NONE) { - assert(mod_info->aux_usage == ISL_AUX_USAGE_CCS_E); - - mt->aux_usage = mod_info->aux_usage; - /* If we are a window system buffer, then we can support fast-clears - * even if the modifier doesn't support them by doing a partial resolve - * as part of the flush operation. - */ - mt->supports_fast_clear = - allow_internal_aux || mod_info->supports_clear_color; - - /* We don't know the actual state of the surface when we get it but we - * can make a pretty good guess based on the modifier. What we do know - * for sure is that it isn't in the AUX_INVALID state, so we just assume - * a worst case of compression. - */ - enum isl_aux_state initial_state = - isl_drm_modifier_get_default_aux_state(image->modifier); - - if (!create_ccs_buf_for_image(brw, image, mt, initial_state)) { - brw_miptree_release(&mt); - return NULL; - } - } - - /* Don't assume coherency for imported EGLimages. We don't know what - * external clients are going to do with it. They may scan it out. - */ - image->bo->cache_coherent = false; - - return mt; -} - -/** - * For a singlesample renderbuffer, this simply wraps the given BO with a - * miptree. - * - * For a multisample renderbuffer, this wraps the window system's - * (singlesample) BO with a singlesample miptree attached to the - * brw_renderbuffer, then creates a multisample miptree attached to irb->mt - * that will contain the actual rendering (which is lazily resolved to - * irb->singlesample_mt). - */ -bool -brw_update_winsys_renderbuffer_miptree(struct brw_context *intel, - struct brw_renderbuffer *irb, - struct brw_mipmap_tree *singlesample_mt, - uint32_t width, uint32_t height, - uint32_t pitch) -{ - struct brw_mipmap_tree *multisample_mt = NULL; - struct gl_renderbuffer *rb = &irb->Base.Base; - mesa_format format = rb->Format; - const unsigned num_samples = MAX2(rb->NumSamples, 1); - - /* Only the front and back buffers, which are color buffers, are allocated - * through the image loader. - */ - assert(_mesa_get_format_base_format(format) == GL_RGB || - _mesa_get_format_base_format(format) == GL_RGBA); - - assert(singlesample_mt); - - if (num_samples == 1) { - brw_miptree_release(&irb->mt); - irb->mt = singlesample_mt; - - assert(!irb->singlesample_mt); - } else { - brw_miptree_release(&irb->singlesample_mt); - irb->singlesample_mt = singlesample_mt; - - if (!irb->mt || - irb->mt->surf.logical_level0_px.width != width || - irb->mt->surf.logical_level0_px.height != height) { - multisample_mt = brw_miptree_create_for_renderbuffer(intel, - format, - width, - height, - num_samples); - if (!multisample_mt) - goto fail; - - irb->need_downsample = false; - brw_miptree_release(&irb->mt); - irb->mt = multisample_mt; - } - } - return true; - -fail: - brw_miptree_release(&irb->mt); - return false; -} - -struct brw_mipmap_tree* -brw_miptree_create_for_renderbuffer(struct brw_context *brw, - mesa_format format, - uint32_t width, - uint32_t height, - uint32_t num_samples) -{ - struct brw_mipmap_tree *mt; - uint32_t depth = 1; - GLenum target = num_samples > 1 ? GL_TEXTURE_2D_MULTISAMPLE : GL_TEXTURE_2D; - - mt = brw_miptree_create(brw, target, format, 0, 0, - width, height, depth, num_samples, - MIPTREE_CREATE_BUSY); - if (!mt) - goto fail; - - return mt; - -fail: - brw_miptree_release(&mt); - return NULL; -} - -void -brw_miptree_reference(struct brw_mipmap_tree **dst, - struct brw_mipmap_tree *src) -{ - if (*dst == src) - return; - - brw_miptree_release(dst); - - if (src) { - src->refcount++; - DBG("%s %p refcount now %d\n", __func__, src, src->refcount); - } - - *dst = src; -} - -static void -brw_miptree_aux_buffer_free(struct brw_miptree_aux_buffer *aux_buf) -{ - if (aux_buf == NULL) - return; - - brw_bo_unreference(aux_buf->bo); - brw_bo_unreference(aux_buf->clear_color_bo); - - free(aux_buf); -} - -void -brw_miptree_release(struct brw_mipmap_tree **mt) -{ - if (!*mt) - return; - - DBG("%s %p refcount will be %d\n", __func__, *mt, (*mt)->refcount - 1); - if (--(*mt)->refcount <= 0) { - GLuint i; - - DBG("%s deleting %p\n", __func__, *mt); - - brw_bo_unreference((*mt)->bo); - brw_miptree_release(&(*mt)->stencil_mt); - brw_miptree_release(&(*mt)->shadow_mt); - brw_miptree_aux_buffer_free((*mt)->aux_buf); - free_aux_state_map((*mt)->aux_state); - - brw_miptree_release(&(*mt)->plane[0]); - brw_miptree_release(&(*mt)->plane[1]); - - for (i = 0; i < MAX_TEXTURE_LEVELS; i++) { - free((*mt)->level[i].slice); - } - - free(*mt); - } - *mt = NULL; -} - - -void -brw_get_image_dims(struct gl_texture_image *image, - int *width, int *height, int *depth) -{ - switch (image->TexObject->Target) { - case GL_TEXTURE_1D_ARRAY: - /* For a 1D Array texture the OpenGL API will treat the image height as - * the number of array slices. For Intel hardware, we treat the 1D array - * as a 2D Array with a height of 1. So, here we want to swap image - * height and depth. - */ - assert(image->Depth == 1); - *width = image->Width; - *height = 1; - *depth = image->Height; - break; - case GL_TEXTURE_CUBE_MAP: - /* For Cube maps, the mesa/main api layer gives us a depth of 1 even - * though we really have 6 slices. - */ - assert(image->Depth == 1); - *width = image->Width; - *height = image->Height; - *depth = 6; - break; - default: - *width = image->Width; - *height = image->Height; - *depth = image->Depth; - break; - } -} - -/** - * Can the image be pulled into a unified mipmap tree? This mirrors - * the completeness test in a lot of ways. - * - * Not sure whether I want to pass gl_texture_image here. - */ -bool -brw_miptree_match_image(struct brw_mipmap_tree *mt, - struct gl_texture_image *image) -{ - struct brw_texture_image *brw_image = brw_texture_image(image); - GLuint level = brw_image->base.Base.Level; - int width, height, depth; - - /* glTexImage* choose the texture object based on the target passed in, and - * objects can't change targets over their lifetimes, so this should be - * true. - */ - assert(image->TexObject->Target == mt->target); - - mesa_format mt_format = mt->format; - if (mt->format == MESA_FORMAT_Z24_UNORM_X8_UINT && mt->stencil_mt) - mt_format = MESA_FORMAT_Z24_UNORM_S8_UINT; - if (mt->format == MESA_FORMAT_Z_FLOAT32 && mt->stencil_mt) - mt_format = MESA_FORMAT_Z32_FLOAT_S8X24_UINT; - - if (_mesa_get_srgb_format_linear(image->TexFormat) != - _mesa_get_srgb_format_linear(mt_format)) - return false; - - brw_get_image_dims(image, &width, &height, &depth); - - if (mt->target == GL_TEXTURE_CUBE_MAP) - depth = 6; - - if (level >= mt->surf.levels) - return false; - - const unsigned level_depth = - mt->surf.dim == ISL_SURF_DIM_3D ? - minify(mt->surf.logical_level0_px.depth, level) : - mt->surf.logical_level0_px.array_len; - - return width == minify(mt->surf.logical_level0_px.width, level) && - height == minify(mt->surf.logical_level0_px.height, level) && - depth == level_depth && - MAX2(image->NumSamples, 1) == mt->surf.samples; -} - -void -brw_miptree_get_image_offset(const struct brw_mipmap_tree *mt, - GLuint level, GLuint slice, - GLuint *x, GLuint *y) -{ - if (level == 0 && slice == 0) { - *x = mt->level[0].level_x; - *y = mt->level[0].level_y; - return; - } - - uint32_t x_offset_sa, y_offset_sa; - - /* Miptree itself can have an offset only if it represents a single - * slice in an imported buffer object. - * See brw_miptree_create_for_dri_image(). - */ - assert(mt->level[0].level_x == 0); - assert(mt->level[0].level_y == 0); - - /* Given level is relative to level zero while the miptree may be - * represent just a subset of all levels starting from 'first_level'. - */ - assert(level >= mt->first_level); - level -= mt->first_level; - - const unsigned z = mt->surf.dim == ISL_SURF_DIM_3D ? slice : 0; - slice = mt->surf.dim == ISL_SURF_DIM_3D ? 0 : slice; - isl_surf_get_image_offset_el(&mt->surf, level, slice, z, - &x_offset_sa, &y_offset_sa); - - *x = x_offset_sa; - *y = y_offset_sa; -} - -/** - * Compute the offset (in bytes) from the start of the BO to the given x - * and y coordinate. For tiled BOs, caller must ensure that x and y are - * multiples of the tile size. - */ -uint32_t -brw_miptree_get_aligned_offset(const struct brw_mipmap_tree *mt, - uint32_t x, uint32_t y) -{ - int cpp = mt->cpp; - uint32_t pitch = mt->surf.row_pitch_B; - - switch (mt->surf.tiling) { - default: - unreachable("not reached"); - case ISL_TILING_LINEAR: - return y * pitch + x * cpp; - case ISL_TILING_X: - assert((x % (512 / cpp)) == 0); - assert((y % 8) == 0); - return y * pitch + x / (512 / cpp) * 4096; - case ISL_TILING_Y0: - assert((x % (128 / cpp)) == 0); - assert((y % 32) == 0); - return y * pitch + x / (128 / cpp) * 4096; - } -} - -/** - * Rendering with tiled buffers requires that the base address of the buffer - * be aligned to a page boundary. For renderbuffers, and sometimes with - * textures, we may want the surface to point at a texture image level that - * isn't at a page boundary. - * - * This function returns an appropriately-aligned base offset - * according to the tiling restrictions, plus any required x/y offset - * from there. - */ -uint32_t -brw_miptree_get_tile_offsets(const struct brw_mipmap_tree *mt, - GLuint level, GLuint slice, - uint32_t *tile_x, - uint32_t *tile_y) -{ - uint32_t x, y; - uint32_t mask_x, mask_y; - - isl_get_tile_masks(mt->surf.tiling, mt->cpp, &mask_x, &mask_y); - brw_miptree_get_image_offset(mt, level, slice, &x, &y); - - *tile_x = x & mask_x; - *tile_y = y & mask_y; - - return brw_miptree_get_aligned_offset(mt, x & ~mask_x, y & ~mask_y); -} - -static void -brw_miptree_copy_slice_sw(struct brw_context *brw, - struct brw_mipmap_tree *src_mt, - unsigned src_level, unsigned src_layer, - struct brw_mipmap_tree *dst_mt, - unsigned dst_level, unsigned dst_layer, - unsigned width, unsigned height) -{ - void *src, *dst; - ptrdiff_t src_stride, dst_stride; - const unsigned cpp = (isl_format_get_layout(dst_mt->surf.format)->bpb / 8); - - brw_miptree_map(brw, src_mt, - src_level, src_layer, - 0, 0, - width, height, - GL_MAP_READ_BIT | BRW_MAP_DIRECT_BIT, - &src, &src_stride); - - brw_miptree_map(brw, dst_mt, - dst_level, dst_layer, - 0, 0, - width, height, - GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_RANGE_BIT | - BRW_MAP_DIRECT_BIT, - &dst, &dst_stride); - - DBG("sw blit %s mt %p %p/%"PRIdPTR" -> %s mt %p %p/%"PRIdPTR" (%dx%d)\n", - _mesa_get_format_name(src_mt->format), - src_mt, src, src_stride, - _mesa_get_format_name(dst_mt->format), - dst_mt, dst, dst_stride, - width, height); - - int row_size = cpp * width; - if (src_stride == row_size && - dst_stride == row_size) { - memcpy(dst, src, row_size * height); - } else { - for (int i = 0; i < height; i++) { - memcpy(dst, src, row_size); - dst += dst_stride; - src += src_stride; - } - } - - brw_miptree_unmap(brw, dst_mt, dst_level, dst_layer); - brw_miptree_unmap(brw, src_mt, src_level, src_layer); - - /* Don't forget to copy the stencil data over, too. We could have skipped - * passing BRW_MAP_DIRECT_BIT, but that would have meant brw_miptree_map - * shuffling the two data sources in/out of temporary storage instead of - * the direct mapping we get this way. - */ - if (dst_mt->stencil_mt) { - assert(src_mt->stencil_mt); - brw_miptree_copy_slice_sw(brw, - src_mt->stencil_mt, src_level, src_layer, - dst_mt->stencil_mt, dst_level, dst_layer, - width, height); - } -} - -void -brw_miptree_copy_slice(struct brw_context *brw, - struct brw_mipmap_tree *src_mt, - unsigned src_level, unsigned src_layer, - struct brw_mipmap_tree *dst_mt, - unsigned dst_level, unsigned dst_layer) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - mesa_format format = src_mt->format; - unsigned width = minify(src_mt->surf.phys_level0_sa.width, - src_level - src_mt->first_level); - unsigned height = minify(src_mt->surf.phys_level0_sa.height, - src_level - src_mt->first_level); - - assert(src_layer < get_num_phys_layers(&src_mt->surf, - src_level - src_mt->first_level)); - - assert(_mesa_get_srgb_format_linear(src_mt->format) == - _mesa_get_srgb_format_linear(dst_mt->format)); - - DBG("validate blit mt %s %p %d,%d -> mt %s %p %d,%d (%dx%d)\n", - _mesa_get_format_name(src_mt->format), - src_mt, src_level, src_layer, - _mesa_get_format_name(dst_mt->format), - dst_mt, dst_level, dst_layer, - width, height); - - if (devinfo->gen >= 6) { - /* On gen6 and above, we just use blorp. It's faster than the blitter - * and can handle everything without software fallbacks. - */ - brw_blorp_copy_miptrees(brw, - src_mt, src_level, src_layer, - dst_mt, dst_level, dst_layer, - 0, 0, 0, 0, width, height); - - if (src_mt->stencil_mt) { - assert(dst_mt->stencil_mt); - brw_blorp_copy_miptrees(brw, - src_mt->stencil_mt, src_level, src_layer, - dst_mt->stencil_mt, dst_level, dst_layer, - 0, 0, 0, 0, width, height); - } - return; - } - - if (dst_mt->compressed) { - unsigned int i, j; - _mesa_get_format_block_size(dst_mt->format, &i, &j); - height = ALIGN_NPOT(height, j) / j; - width = ALIGN_NPOT(width, i) / i; - } - - /* Gen4-5 doesn't support separate stencil */ - assert(!src_mt->stencil_mt); - - uint32_t dst_x, dst_y, src_x, src_y; - brw_miptree_get_image_offset(dst_mt, dst_level, dst_layer, &dst_x, &dst_y); - brw_miptree_get_image_offset(src_mt, src_level, src_layer, &src_x, &src_y); - - DBG("validate blit mt %s %p %d,%d/%d -> mt %s %p %d,%d/%d (%dx%d)\n", - _mesa_get_format_name(src_mt->format), - src_mt, src_x, src_y, src_mt->surf.row_pitch_B, - _mesa_get_format_name(dst_mt->format), - dst_mt, dst_x, dst_y, dst_mt->surf.row_pitch_B, - width, height); - - if (!brw_miptree_blit(brw, - src_mt, src_level, src_layer, 0, 0, false, - dst_mt, dst_level, dst_layer, 0, 0, false, - width, height, COLOR_LOGICOP_COPY)) { - perf_debug("miptree validate blit for %s failed\n", - _mesa_get_format_name(format)); - - brw_miptree_copy_slice_sw(brw, - src_mt, src_level, src_layer, - dst_mt, dst_level, dst_layer, - width, height); - } -} - -/** - * Copies the image's current data to the given miptree, and associates that - * miptree with the image. - */ -void -brw_miptree_copy_teximage(struct brw_context *brw, - struct brw_texture_image *brw_image, - struct brw_mipmap_tree *dst_mt) -{ - struct brw_mipmap_tree *src_mt = brw_image->mt; - struct brw_texture_object *intel_obj = - brw_texture_object(brw_image->base.Base.TexObject); - int level = brw_image->base.Base.Level; - const unsigned face = brw_image->base.Base.Face; - unsigned start_layer, end_layer; - - if (intel_obj->base.Target == GL_TEXTURE_1D_ARRAY) { - assert(face == 0); - assert(brw_image->base.Base.Height); - start_layer = 0; - end_layer = brw_image->base.Base.Height - 1; - } else if (face > 0) { - start_layer = face; - end_layer = face; - } else { - assert(brw_image->base.Base.Depth); - start_layer = 0; - end_layer = brw_image->base.Base.Depth - 1; - } - - for (unsigned i = start_layer; i <= end_layer; i++) { - brw_miptree_copy_slice(brw, src_mt, level, i, dst_mt, level, i); - } - - brw_miptree_reference(&brw_image->mt, dst_mt); - intel_obj->needs_validate = true; -} - -static struct brw_miptree_aux_buffer * -brw_alloc_aux_buffer(struct brw_context *brw, - const struct isl_surf *aux_surf, - bool wants_memset, - uint8_t memset_value) -{ - struct brw_miptree_aux_buffer *buf = calloc(sizeof(*buf), 1); - if (!buf) - return false; - - uint64_t size = aux_surf->size_B; - - const bool has_indirect_clear = brw->isl_dev.ss.clear_color_state_size > 0; - if (has_indirect_clear) { - /* On CNL+, instead of setting the clear color in the SURFACE_STATE, we - * will set a pointer to a dword somewhere that contains the color. So, - * allocate the space for the clear color value here on the aux buffer. - */ - buf->clear_color_offset = size; - size += brw->isl_dev.ss.clear_color_state_size; - } - - /* If the buffer needs to be initialised (requiring the buffer to be - * immediately mapped to cpu space for writing), do not use the gpu access - * flag which can cause an unnecessary delay if the backing pages happened - * to be just used by the GPU. - */ - const bool alloc_zeroed = wants_memset && memset_value == 0; - const bool needs_memset = - !alloc_zeroed && (wants_memset || has_indirect_clear); - const uint32_t alloc_flags = - alloc_zeroed ? BO_ALLOC_ZEROED : (needs_memset ? 0 : BO_ALLOC_BUSY); - - /* ISL has stricter set of alignment rules then the drm allocator. - * Therefore one can pass the ISL dimensions in terms of bytes instead of - * trying to recalculate based on different format block sizes. - */ - buf->bo = brw_bo_alloc_tiled(brw->bufmgr, "aux-miptree", size, - BRW_MEMZONE_OTHER, I915_TILING_Y, - aux_surf->row_pitch_B, alloc_flags); - if (!buf->bo) { - free(buf); - return NULL; - } - - /* Initialize the bo to the desired value */ - if (needs_memset) { - assert(!(alloc_flags & BO_ALLOC_BUSY)); - - void *map = brw_bo_map(brw, buf->bo, MAP_WRITE | MAP_RAW); - if (map == NULL) { - brw_miptree_aux_buffer_free(buf); - return NULL; - } - - /* Memset the aux_surf portion of the BO. */ - if (wants_memset) - memset(map, memset_value, aux_surf->size_B); - - /* Zero the indirect clear color to match ::fast_clear_color. */ - if (has_indirect_clear) { - memset((char *)map + buf->clear_color_offset, 0, - brw->isl_dev.ss.clear_color_state_size); - } - - brw_bo_unmap(buf->bo); - } - - if (has_indirect_clear) { - buf->clear_color_bo = buf->bo; - brw_bo_reference(buf->clear_color_bo); - } - - buf->surf = *aux_surf; - - return buf; -} - - -/** - * Helper for brw_miptree_alloc_aux() that sets - * \c mt->level[level].has_hiz. Return true if and only if - * \c has_hiz was set. - */ -static bool -brw_miptree_level_enable_hiz(struct brw_context *brw, - struct brw_mipmap_tree *mt, - uint32_t level) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - assert(mt->aux_buf); - assert(mt->surf.size_B > 0); - - if (devinfo->gen >= 8 || devinfo->is_haswell) { - uint32_t width = minify(mt->surf.phys_level0_sa.width, level); - uint32_t height = minify(mt->surf.phys_level0_sa.height, level); - - /* Disable HiZ for LOD > 0 unless the width is 8 aligned - * and the height is 4 aligned. This allows our HiZ support - * to fulfill Haswell restrictions for HiZ ops. For LOD == 0, - * we can grow the width & height to allow the HiZ op to - * force the proper size alignments. - */ - if (level > 0 && ((width & 7) || (height & 3))) { - DBG("mt %p level %d: HiZ DISABLED\n", mt, level); - return false; - } - } - - DBG("mt %p level %d: HiZ enabled\n", mt, level); - mt->level[level].has_hiz = true; - return true; -} - - -/** - * Allocate the initial aux surface for a miptree based on mt->aux_usage - * - * Since MCS, HiZ, and CCS_E can compress more than just clear color, we - * create the auxiliary surfaces up-front. CCS_D, on the other hand, can only - * compress clear color so we wait until an actual fast-clear to allocate it. - */ -bool -brw_miptree_alloc_aux(struct brw_context *brw, struct brw_mipmap_tree *mt) -{ - assert(mt->aux_buf == NULL); - - /* Get the aux buf allocation parameters for this miptree. */ - enum isl_aux_state initial_state; - uint8_t memset_value; - struct isl_surf aux_surf = {0,}; - bool aux_surf_ok = false; - - switch (mt->aux_usage) { - case ISL_AUX_USAGE_NONE: - aux_surf.size_B = 0; - aux_surf_ok = true; - break; - case ISL_AUX_USAGE_HIZ: - initial_state = ISL_AUX_STATE_AUX_INVALID; - memset_value = 0; - aux_surf_ok = isl_surf_get_hiz_surf(&brw->isl_dev, &mt->surf, &aux_surf); - break; - case ISL_AUX_USAGE_MCS: - /* From the Ivy Bridge PRM, Vol 2 Part 1 p326: - * - * When MCS buffer is enabled and bound to MSRT, it is required that - * it is cleared prior to any rendering. - * - * Since we don't use the MCS buffer for any purpose other than - * rendering, it makes sense to just clear it immediately upon - * allocation. - * - * Note: the clear value for MCS buffers is all 1's, so we memset to - * 0xff. - */ - initial_state = ISL_AUX_STATE_CLEAR; - memset_value = 0xFF; - aux_surf_ok = isl_surf_get_mcs_surf(&brw->isl_dev, &mt->surf, &aux_surf); - break; - case ISL_AUX_USAGE_CCS_D: - case ISL_AUX_USAGE_CCS_E: - /* When CCS_E is used, we need to ensure that the CCS starts off in a - * valid state. From the Sky Lake PRM, "MCS Buffer for Render - * Target(s)": - * - * "If Software wants to enable Color Compression without Fast - * clear, Software needs to initialize MCS with zeros." - * - * A CCS value of 0 indicates that the corresponding block is in the - * pass-through state which is what we want. - * - * For CCS_D, do the same thing. On gen9+, this avoids having any - * undefined bits in the aux buffer. - */ - initial_state = ISL_AUX_STATE_PASS_THROUGH; - memset_value = 0; - aux_surf_ok = - isl_surf_get_ccs_surf(&brw->isl_dev, &mt->surf, &aux_surf, NULL, 0); - break; - - default: - unreachable("Invalid aux usage"); - } - - /* We should have a valid aux_surf. */ - if (!aux_surf_ok) - return false; - - /* No work is needed for a zero-sized auxiliary buffer. */ - if (aux_surf.size_B == 0) - return true; - - /* Create the aux_state for the auxiliary buffer. */ - mt->aux_state = create_aux_state_map(mt, initial_state); - if (mt->aux_state == NULL) - return false; - - /* Allocate the auxiliary buffer. */ - const bool needs_memset = initial_state != ISL_AUX_STATE_AUX_INVALID; - mt->aux_buf = brw_alloc_aux_buffer(brw, &aux_surf, needs_memset, - memset_value); - if (mt->aux_buf == NULL) { - free_aux_state_map(mt->aux_state); - mt->aux_state = NULL; - return false; - } - - /* Perform aux_usage-specific initialization. */ - if (mt->aux_usage == ISL_AUX_USAGE_HIZ) { - for (unsigned level = mt->first_level; level <= mt->last_level; ++level) - brw_miptree_level_enable_hiz(brw, mt, level); - } - - return true; -} - - -/** - * Can the miptree sample using the hiz buffer? - */ -bool -brw_miptree_sample_with_hiz(struct brw_context *brw, - struct brw_mipmap_tree *mt) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - if (!devinfo->has_sample_with_hiz) { - return false; - } - - if (!mt->aux_buf) { - return false; - } - - for (unsigned level = 0; level < mt->surf.levels; ++level) { - if (!brw_miptree_level_has_hiz(mt, level)) - return false; - } - - /* From the BDW PRM (Volume 2d: Command Reference: Structures - * RENDER_SURFACE_STATE.AuxiliarySurfaceMode): - * - * "If this field is set to AUX_HIZ, Number of Multisamples must be - * MULTISAMPLECOUNT_1, and Surface Type cannot be SURFTYPE_3D. - * - * There is no such blurb for 1D textures, but there is sufficient evidence - * that this is broken on SKL+. - */ - return (mt->surf.samples == 1 && - mt->target != GL_TEXTURE_3D && - mt->target != GL_TEXTURE_1D /* gen9+ restriction */); -} - -static bool -level_has_aux(const struct brw_mipmap_tree *mt, uint32_t level) -{ - return isl_aux_usage_has_hiz(mt->aux_usage) ? - brw_miptree_level_has_hiz(mt, level) : - mt->aux_usage != ISL_AUX_USAGE_NONE && mt->aux_buf; -} - -/** - * Does the miptree slice have hiz enabled? - */ -bool -brw_miptree_level_has_hiz(const struct brw_mipmap_tree *mt, uint32_t level) -{ - brw_miptree_check_level_layer(mt, level, 0); - return mt->level[level].has_hiz; -} - -static inline uint32_t -miptree_level_range_length(const struct brw_mipmap_tree *mt, - uint32_t start_level, uint32_t num_levels) -{ - assert(start_level >= mt->first_level); - assert(start_level <= mt->last_level); - - if (num_levels == INTEL_REMAINING_LAYERS) - num_levels = mt->last_level - start_level + 1; - /* Check for overflow */ - assert(start_level + num_levels >= start_level); - assert(start_level + num_levels <= mt->last_level + 1); - - return num_levels; -} - -static inline uint32_t -miptree_layer_range_length(const struct brw_mipmap_tree *mt, uint32_t level, - uint32_t start_layer, uint32_t num_layers) -{ - assert(level <= mt->last_level); - - const uint32_t total_num_layers = brw_get_num_logical_layers(mt, level); - assert(start_layer < total_num_layers); - if (num_layers == INTEL_REMAINING_LAYERS) - num_layers = total_num_layers - start_layer; - /* Check for overflow */ - assert(start_layer + num_layers >= start_layer); - assert(start_layer + num_layers <= total_num_layers); - - return num_layers; -} - -bool -brw_miptree_has_color_unresolved(const struct brw_mipmap_tree *mt, - unsigned start_level, unsigned num_levels, - unsigned start_layer, unsigned num_layers) -{ - assert(_mesa_is_format_color_format(mt->format)); - - if (!mt->aux_buf) - return false; - - /* Clamp the level range to fit the miptree */ - num_levels = miptree_level_range_length(mt, start_level, num_levels); - - for (uint32_t l = 0; l < num_levels; l++) { - const uint32_t level = start_level + l; - const uint32_t level_layers = - miptree_layer_range_length(mt, level, start_layer, num_layers); - for (unsigned a = 0; a < level_layers; a++) { - enum isl_aux_state aux_state = - brw_miptree_get_aux_state(mt, level, start_layer + a); - assert(aux_state != ISL_AUX_STATE_AUX_INVALID); - if (aux_state != ISL_AUX_STATE_PASS_THROUGH) - return true; - } - } - - return false; -} - -static void -brw_miptree_check_color_resolve(const struct brw_context *brw, - const struct brw_mipmap_tree *mt, - unsigned level, unsigned layer) -{ - if (!mt->aux_buf) - return; - - /* Fast color clear is supported for mipmapped surfaces only on Gen8+. */ - assert(brw->screen->devinfo.gen >= 8 || - (level == 0 && mt->first_level == 0 && mt->last_level == 0)); - - /* Compression of arrayed msaa surfaces is supported. */ - if (mt->surf.samples > 1) - return; - - /* Fast color clear is supported for non-msaa arrays only on Gen8+. */ - assert(brw->screen->devinfo.gen >= 8 || - (layer == 0 && - mt->surf.logical_level0_px.depth == 1 && - mt->surf.logical_level0_px.array_len == 1)); - - (void)level; - (void)layer; -} - -void -brw_miptree_prepare_access(struct brw_context *brw, - struct brw_mipmap_tree *mt, - uint32_t start_level, uint32_t num_levels, - uint32_t start_layer, uint32_t num_layers, - enum isl_aux_usage aux_usage, - bool fast_clear_supported) -{ - const uint32_t clamped_levels = - miptree_level_range_length(mt, start_level, num_levels); - for (uint32_t l = 0; l < clamped_levels; l++) { - const uint32_t level = start_level + l; - if (!level_has_aux(mt, level)) - continue; - - const uint32_t level_layers = - miptree_layer_range_length(mt, level, start_layer, num_layers); - for (uint32_t a = 0; a < level_layers; a++) { - const uint32_t layer = start_layer + a; - const enum isl_aux_state aux_state = - brw_miptree_get_aux_state(mt, level, layer); - const enum isl_aux_op aux_op = - isl_aux_prepare_access(aux_state, aux_usage, fast_clear_supported); - - if (aux_op == ISL_AUX_OP_NONE) { - /* Nothing to do here. */ - } else if (isl_aux_usage_has_mcs(mt->aux_usage)) { - assert(aux_op == ISL_AUX_OP_PARTIAL_RESOLVE); - brw_blorp_mcs_partial_resolve(brw, mt, layer, 1); - } else if (isl_aux_usage_has_hiz(mt->aux_usage)) { - brw_hiz_exec(brw, mt, level, layer, 1, aux_op); - } else { - assert(isl_aux_usage_has_ccs(mt->aux_usage)); - brw_miptree_check_color_resolve(brw, mt, level, layer); - brw_blorp_resolve_color(brw, mt, level, layer, aux_op); - } - - const enum isl_aux_state new_state = - isl_aux_state_transition_aux_op(aux_state, mt->aux_usage, aux_op); - brw_miptree_set_aux_state(brw, mt, level, layer, 1, new_state); - } - } -} - -void -brw_miptree_finish_write(struct brw_context *brw, - struct brw_mipmap_tree *mt, uint32_t level, - uint32_t start_layer, uint32_t num_layers, - enum isl_aux_usage aux_usage) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - if (mt->format == MESA_FORMAT_S_UINT8 && devinfo->gen <= 7) { - mt->shadow_needs_update = true; - } else if (brw_miptree_has_etc_shadow(brw, mt)) { - mt->shadow_needs_update = true; - } - - if (!level_has_aux(mt, level)) - return; - - const uint32_t level_layers = - miptree_layer_range_length(mt, level, start_layer, num_layers); - - for (uint32_t a = 0; a < level_layers; a++) { - const uint32_t layer = start_layer + a; - const enum isl_aux_state aux_state = - brw_miptree_get_aux_state(mt, level, layer); - const enum isl_aux_state new_aux_state = - isl_aux_state_transition_write(aux_state, aux_usage, false); - brw_miptree_set_aux_state(brw, mt, level, layer, 1, new_aux_state); - } -} - -enum isl_aux_state -brw_miptree_get_aux_state(const struct brw_mipmap_tree *mt, - uint32_t level, uint32_t layer) -{ - brw_miptree_check_level_layer(mt, level, layer); - - if (_mesa_is_format_color_format(mt->format)) { - assert(mt->aux_buf != NULL); - assert(mt->surf.samples == 1 || - mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY); - } else if (mt->format == MESA_FORMAT_S_UINT8) { - unreachable("Cannot get aux state for stencil"); - } else { - assert(brw_miptree_level_has_hiz(mt, level)); - } - - return mt->aux_state[level][layer]; -} - -void -brw_miptree_set_aux_state(struct brw_context *brw, - struct brw_mipmap_tree *mt, uint32_t level, - uint32_t start_layer, uint32_t num_layers, - enum isl_aux_state aux_state) -{ - num_layers = miptree_layer_range_length(mt, level, start_layer, num_layers); - - if (_mesa_is_format_color_format(mt->format)) { - assert(mt->aux_buf != NULL); - assert(mt->surf.samples == 1 || - mt->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY); - } else if (mt->format == MESA_FORMAT_S_UINT8) { - unreachable("Cannot get aux state for stencil"); - } else { - assert(brw_miptree_level_has_hiz(mt, level)); - } - - for (unsigned a = 0; a < num_layers; a++) { - if (mt->aux_state[level][start_layer + a] != aux_state) { - mt->aux_state[level][start_layer + a] = aux_state; - brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE; - } - } -} - -/* On Gen9 color buffers may be compressed by the hardware (lossless - * compression). There are, however, format restrictions and care needs to be - * taken that the sampler engine is capable for re-interpreting a buffer with - * format different the buffer was originally written with. - * - * For example, SRGB formats are not compressible and the sampler engine isn't - * capable of treating RGBA_UNORM as SRGB_ALPHA. In such a case the underlying - * color buffer needs to be resolved so that the sampling surface can be - * sampled as non-compressed (i.e., without the auxiliary MCS buffer being - * set). - */ -static bool -can_texture_with_ccs(struct brw_context *brw, - struct brw_mipmap_tree *mt, - enum isl_format view_format) -{ - if (mt->aux_usage != ISL_AUX_USAGE_CCS_E) - return false; - - if (!format_ccs_e_compat_with_miptree(&brw->screen->devinfo, - mt, view_format)) { - perf_debug("Incompatible sampling format (%s) for rbc (%s)\n", - isl_format_get_layout(view_format)->name, - _mesa_get_format_name(mt->format)); - return false; - } - - return true; -} - -enum isl_aux_usage -brw_miptree_texture_aux_usage(struct brw_context *brw, - struct brw_mipmap_tree *mt, - enum isl_format view_format, - enum gen9_astc5x5_wa_tex_type astc5x5_wa_bits) -{ - assert(brw->screen->devinfo.gen == 9 || astc5x5_wa_bits == 0); - - /* On gen9, ASTC 5x5 textures cannot live in the sampler cache along side - * CCS or HiZ compressed textures. See gen9_apply_astc5x5_wa_flush() for - * details. - */ - if ((astc5x5_wa_bits & GEN9_ASTC5X5_WA_TEX_TYPE_ASTC5x5) && - mt->aux_usage != ISL_AUX_USAGE_MCS) - return ISL_AUX_USAGE_NONE; - - switch (mt->aux_usage) { - case ISL_AUX_USAGE_HIZ: - if (brw_miptree_sample_with_hiz(brw, mt)) - return ISL_AUX_USAGE_HIZ; - break; - - case ISL_AUX_USAGE_MCS: - return ISL_AUX_USAGE_MCS; - - case ISL_AUX_USAGE_CCS_D: - case ISL_AUX_USAGE_CCS_E: - if (!mt->aux_buf) { - assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D); - return ISL_AUX_USAGE_NONE; - } - - /* If we don't have any unresolved color, report an aux usage of - * ISL_AUX_USAGE_NONE. This way, texturing won't even look at the - * aux surface and we can save some bandwidth. - */ - if (!brw_miptree_has_color_unresolved(mt, 0, INTEL_REMAINING_LEVELS, - 0, INTEL_REMAINING_LAYERS)) - return ISL_AUX_USAGE_NONE; - - if (can_texture_with_ccs(brw, mt, view_format)) - return ISL_AUX_USAGE_CCS_E; - break; - - default: - break; - } - - return ISL_AUX_USAGE_NONE; -} - -static bool -isl_formats_are_fast_clear_compatible(enum isl_format a, enum isl_format b) -{ - /* On gen8 and earlier, the hardware was only capable of handling 0/1 clear - * values so sRGB curve application was a no-op for all fast-clearable - * formats. - * - * On gen9+, the hardware supports arbitrary clear values. For sRGB clear - * values, the hardware interprets the floats, not as what would be - * returned from the sampler (or written by the shader), but as being - * between format conversion and sRGB curve application. This means that - * we can switch between sRGB and UNORM without having to whack the clear - * color. - */ - return isl_format_srgb_to_linear(a) == isl_format_srgb_to_linear(b); -} - -void -brw_miptree_prepare_texture(struct brw_context *brw, - struct brw_mipmap_tree *mt, - enum isl_format view_format, - uint32_t start_level, uint32_t num_levels, - uint32_t start_layer, uint32_t num_layers, - enum gen9_astc5x5_wa_tex_type astc5x5_wa_bits) -{ - enum isl_aux_usage aux_usage = - brw_miptree_texture_aux_usage(brw, mt, view_format, astc5x5_wa_bits); - - bool clear_supported = aux_usage != ISL_AUX_USAGE_NONE; - - /* Clear color is specified as ints or floats and the conversion is done by - * the sampler. If we have a texture view, we would have to perform the - * clear color conversion manually. Just disable clear color. - */ - if (!isl_formats_are_fast_clear_compatible(mt->surf.format, view_format)) - clear_supported = false; - - brw_miptree_prepare_access(brw, mt, start_level, num_levels, - start_layer, num_layers, - aux_usage, clear_supported); -} - -void -brw_miptree_prepare_image(struct brw_context *brw, struct brw_mipmap_tree *mt) -{ - /* The data port doesn't understand any compression */ - brw_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS, - 0, INTEL_REMAINING_LAYERS, - ISL_AUX_USAGE_NONE, false); -} - -enum isl_aux_usage -brw_miptree_render_aux_usage(struct brw_context *brw, - struct brw_mipmap_tree *mt, - enum isl_format render_format, - bool blend_enabled, - bool draw_aux_disabled) -{ - struct gen_device_info *devinfo = &brw->screen->devinfo; - - if (draw_aux_disabled) - return ISL_AUX_USAGE_NONE; - - switch (mt->aux_usage) { - case ISL_AUX_USAGE_MCS: - assert(mt->aux_buf); - return ISL_AUX_USAGE_MCS; - - case ISL_AUX_USAGE_CCS_D: - case ISL_AUX_USAGE_CCS_E: - if (!mt->aux_buf) { - assert(mt->aux_usage == ISL_AUX_USAGE_CCS_D); - return ISL_AUX_USAGE_NONE; - } - - /* gen9+ hardware technically supports non-0/1 clear colors with sRGB - * formats. However, there are issues with blending where it doesn't - * properly apply the sRGB curve to the clear color when blending. - */ - if (devinfo->gen >= 9 && blend_enabled && - isl_format_is_srgb(render_format) && - !isl_color_value_is_zero_one(mt->fast_clear_color, render_format)) - return ISL_AUX_USAGE_NONE; - - if (mt->aux_usage == ISL_AUX_USAGE_CCS_E && - format_ccs_e_compat_with_miptree(&brw->screen->devinfo, - mt, render_format)) - return ISL_AUX_USAGE_CCS_E; - - /* Otherwise, we have to fall back to CCS_D */ - return ISL_AUX_USAGE_CCS_D; - - default: - return ISL_AUX_USAGE_NONE; - } -} - -void -brw_miptree_prepare_render(struct brw_context *brw, - struct brw_mipmap_tree *mt, uint32_t level, - uint32_t start_layer, uint32_t layer_count, - enum isl_aux_usage aux_usage) -{ - brw_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count, - aux_usage, aux_usage != ISL_AUX_USAGE_NONE); -} - -void -brw_miptree_finish_render(struct brw_context *brw, - struct brw_mipmap_tree *mt, uint32_t level, - uint32_t start_layer, uint32_t layer_count, - enum isl_aux_usage aux_usage) -{ - assert(_mesa_is_format_color_format(mt->format)); - - brw_miptree_finish_write(brw, mt, level, start_layer, layer_count, - aux_usage); -} - -void -brw_miptree_prepare_depth(struct brw_context *brw, - struct brw_mipmap_tree *mt, uint32_t level, - uint32_t start_layer, uint32_t layer_count) -{ - brw_miptree_prepare_access(brw, mt, level, 1, start_layer, layer_count, - mt->aux_usage, mt->aux_buf != NULL); -} - -void -brw_miptree_finish_depth(struct brw_context *brw, - struct brw_mipmap_tree *mt, uint32_t level, - uint32_t start_layer, uint32_t layer_count, - bool depth_written) -{ - if (depth_written) { - brw_miptree_finish_write(brw, mt, level, start_layer, layer_count, - mt->aux_usage); - } -} - -void -brw_miptree_prepare_external(struct brw_context *brw, - struct brw_mipmap_tree *mt) -{ - enum isl_aux_usage aux_usage = ISL_AUX_USAGE_NONE; - bool supports_fast_clear = false; - - const struct isl_drm_modifier_info *mod_info = - isl_drm_modifier_get_info(mt->drm_modifier); - - if (mod_info && mod_info->aux_usage != ISL_AUX_USAGE_NONE) { - /* CCS_E is the only supported aux for external images and it's only - * supported on very simple images. - */ - assert(mod_info->aux_usage == ISL_AUX_USAGE_CCS_E); - assert(_mesa_is_format_color_format(mt->format)); - assert(mt->first_level == 0 && mt->last_level == 0); - assert(mt->surf.logical_level0_px.depth == 1); - assert(mt->surf.logical_level0_px.array_len == 1); - assert(mt->surf.samples == 1); - assert(mt->aux_buf != NULL); - - aux_usage = mod_info->aux_usage; - supports_fast_clear = mod_info->supports_clear_color; - } - - brw_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS, - 0, INTEL_REMAINING_LAYERS, - aux_usage, supports_fast_clear); -} - -void -brw_miptree_finish_external(struct brw_context *brw, - struct brw_mipmap_tree *mt) -{ - if (!mt->aux_buf) - return; - - /* We don't know the actual aux state of the aux surface. The previous - * owner could have given it to us in a number of different states. - * Because we don't know the aux state, we reset the aux state to the - * least common denominator of possible valid states. - */ - enum isl_aux_state default_aux_state = - isl_drm_modifier_get_default_aux_state(mt->drm_modifier); - assert(mt->last_level == mt->first_level); - brw_miptree_set_aux_state(brw, mt, 0, 0, INTEL_REMAINING_LAYERS, - default_aux_state); -} - -/** - * Make it possible to share the BO backing the given miptree with another - * process or another miptree. - * - * Fast color clears are unsafe with shared buffers, so we need to resolve and - * then discard the MCS buffer, if present. We also set the no_ccs flag to - * ensure that no MCS buffer gets allocated in the future. - * - * HiZ is similarly unsafe with shared buffers. - */ -void -brw_miptree_make_shareable(struct brw_context *brw, - struct brw_mipmap_tree *mt) -{ - /* MCS buffers are also used for multisample buffers, but we can't resolve - * away a multisample MCS buffer because it's an integral part of how the - * pixel data is stored. Fortunately this code path should never be - * reached for multisample buffers. - */ - assert(mt->surf.msaa_layout == ISL_MSAA_LAYOUT_NONE || - mt->surf.samples == 1); - - brw_miptree_prepare_access(brw, mt, 0, INTEL_REMAINING_LEVELS, - 0, INTEL_REMAINING_LAYERS, - ISL_AUX_USAGE_NONE, false); - - if (mt->aux_buf) { - brw_miptree_aux_buffer_free(mt->aux_buf); - mt->aux_buf = NULL; - - /* Make future calls of brw_miptree_level_has_hiz() return false. */ - for (uint32_t l = mt->first_level; l <= mt->last_level; ++l) { - mt->level[l].has_hiz = false; - } - - free(mt->aux_state); - mt->aux_state = NULL; - brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE; - } - - mt->aux_usage = ISL_AUX_USAGE_NONE; - mt->supports_fast_clear = false; -} - - -/** - * \brief Get pointer offset into stencil buffer. - * - * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we - * must decode the tile's layout in software. - * - * See - * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile - * Format. - * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm - * - * Even though the returned offset is always positive, the return type is - * signed due to - * commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137 - * mesa: Fix return type of _mesa_get_format_bytes() (#37351) - */ -static intptr_t -brw_offset_S8(uint32_t stride, uint32_t x, uint32_t y, bool swizzled) -{ - uint32_t tile_size = 4096; - uint32_t tile_width = 64; - uint32_t tile_height = 64; - uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */ - - uint32_t tile_x = x / tile_width; - uint32_t tile_y = y / tile_height; - - /* The byte's address relative to the tile's base addres. */ - uint32_t byte_x = x % tile_width; - uint32_t byte_y = y % tile_height; - - uintptr_t u = tile_y * row_size - + tile_x * tile_size - + 512 * (byte_x / 8) - + 64 * (byte_y / 8) - + 32 * ((byte_y / 4) % 2) - + 16 * ((byte_x / 4) % 2) - + 8 * ((byte_y / 2) % 2) - + 4 * ((byte_x / 2) % 2) - + 2 * (byte_y % 2) - + 1 * (byte_x % 2); - - if (swizzled) { - /* adjust for bit6 swizzling */ - if (((byte_x / 8) % 2) == 1) { - if (((byte_y / 8) % 2) == 0) { - u += 64; - } else { - u -= 64; - } - } - } - - return u; -} - -void -brw_miptree_updownsample(struct brw_context *brw, - struct brw_mipmap_tree *src, - struct brw_mipmap_tree *dst) -{ - unsigned src_w = src->surf.logical_level0_px.width; - unsigned src_h = src->surf.logical_level0_px.height; - unsigned dst_w = dst->surf.logical_level0_px.width; - unsigned dst_h = dst->surf.logical_level0_px.height; - - brw_blorp_blit_miptrees(brw, - src, 0 /* level */, 0 /* layer */, - src->format, SWIZZLE_XYZW, - dst, 0 /* level */, 0 /* layer */, dst->format, - 0, 0, src_w, src_h, - 0, 0, dst_w, dst_h, - GL_NEAREST, false, false /*mirror x, y*/, - false, false); - - if (src->stencil_mt) { - src_w = src->stencil_mt->surf.logical_level0_px.width; - src_h = src->stencil_mt->surf.logical_level0_px.height; - dst_w = dst->stencil_mt->surf.logical_level0_px.width; - dst_h = dst->stencil_mt->surf.logical_level0_px.height; - - brw_blorp_blit_miptrees(brw, - src->stencil_mt, 0 /* level */, 0 /* layer */, - src->stencil_mt->format, SWIZZLE_XYZW, - dst->stencil_mt, 0 /* level */, 0 /* layer */, - dst->stencil_mt->format, - 0, 0, src_w, src_h, - 0, 0, dst_w, dst_h, - GL_NEAREST, false, false /*mirror x, y*/, - false, false /* decode/encode srgb */); - } -} - -void -brw_update_r8stencil(struct brw_context *brw, - struct brw_mipmap_tree *mt) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - assert(devinfo->gen >= 7); - struct brw_mipmap_tree *src = - mt->format == MESA_FORMAT_S_UINT8 ? mt : mt->stencil_mt; - if (!src || devinfo->gen >= 8) - return; - - assert(src->surf.size_B > 0); - - if (!mt->shadow_mt) { - assert(devinfo->gen > 6); /* Handle MIPTREE_LAYOUT_GEN6_HIZ_STENCIL */ - mt->shadow_mt = make_surface( - brw, - src->target, - MESA_FORMAT_R_UINT8, - src->first_level, src->last_level, - src->surf.logical_level0_px.width, - src->surf.logical_level0_px.height, - src->surf.dim == ISL_SURF_DIM_3D ? - src->surf.logical_level0_px.depth : - src->surf.logical_level0_px.array_len, - src->surf.samples, - ISL_TILING_Y0_BIT, - ISL_SURF_USAGE_TEXTURE_BIT, - BO_ALLOC_BUSY, 0, NULL); - assert(mt->shadow_mt); - } - - if (src->shadow_needs_update == false) - return; - - struct brw_mipmap_tree *dst = mt->shadow_mt; - - for (int level = src->first_level; level <= src->last_level; level++) { - const unsigned depth = src->surf.dim == ISL_SURF_DIM_3D ? - minify(src->surf.phys_level0_sa.depth, level) : - src->surf.phys_level0_sa.array_len; - - for (unsigned layer = 0; layer < depth; layer++) { - brw_blorp_copy_miptrees(brw, - src, level, layer, - dst, level, layer, - 0, 0, 0, 0, - minify(src->surf.logical_level0_px.width, - level), - minify(src->surf.logical_level0_px.height, - level)); - } - } - - brw_cache_flush_for_read(brw, dst->bo); - src->shadow_needs_update = false; -} - -static void * -brw_miptree_map_raw(struct brw_context *brw, - struct brw_mipmap_tree *mt, - GLbitfield mode) -{ - struct brw_bo *bo = mt->bo; - - if (brw_batch_references(&brw->batch, bo)) - brw_batch_flush(brw); - - return brw_bo_map(brw, bo, mode); -} - -static void -brw_miptree_unmap_raw(struct brw_mipmap_tree *mt) -{ - brw_bo_unmap(mt->bo); -} - -static void -brw_miptree_unmap_map(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, unsigned int slice) -{ - brw_miptree_unmap_raw(mt); -} - -static void -brw_miptree_map_map(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, unsigned int slice) -{ - unsigned int bw, bh; - void *base; - unsigned int image_x, image_y; - intptr_t x = map->x; - intptr_t y = map->y; - - /* For compressed formats, the stride is the number of bytes per - * row of blocks. brw_miptree_get_image_offset() already does - * the divide. - */ - _mesa_get_format_block_size(mt->format, &bw, &bh); - assert(y % bh == 0); - assert(x % bw == 0); - y /= bh; - x /= bw; - - brw_miptree_access_raw(brw, mt, level, slice, - map->mode & GL_MAP_WRITE_BIT); - - base = brw_miptree_map_raw(brw, mt, map->mode); - - if (base == NULL) - map->ptr = NULL; - else { - base += mt->offset; - - /* Note that in the case of cube maps, the caller must have passed the - * slice number referencing the face. - */ - brw_miptree_get_image_offset(mt, level, slice, &image_x, &image_y); - x += image_x; - y += image_y; - - map->stride = mt->surf.row_pitch_B; - map->ptr = base + y * map->stride + x * mt->cpp; - } - - DBG("%s: %d,%d %dx%d from mt %p (%s) " - "%"PRIiPTR",%"PRIiPTR" = %p/%d\n", __func__, - map->x, map->y, map->w, map->h, - mt, _mesa_get_format_name(mt->format), - x, y, map->ptr, map->stride); - - map->unmap = brw_miptree_unmap_map; -} - -static void -brw_miptree_unmap_blit(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, - unsigned int slice) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - struct gl_context *ctx = &brw->ctx; - - brw_miptree_unmap_raw(map->linear_mt); - - if (map->mode & GL_MAP_WRITE_BIT) { - if (devinfo->gen >= 6) { - brw_blorp_copy_miptrees(brw, map->linear_mt, 0, 0, - mt, level, slice, - 0, 0, map->x, map->y, map->w, map->h); - } else { - bool ok = brw_miptree_copy(brw, - map->linear_mt, 0, 0, 0, 0, - mt, level, slice, map->x, map->y, - map->w, map->h); - WARN_ONCE(!ok, "Failed to blit from linear temporary mapping"); - } - } - - brw_miptree_release(&map->linear_mt); -} - -/* Compute extent parameters for use with tiled_memcpy functions. - * xs are in units of bytes and ys are in units of strides. - */ -static inline void -tile_extents(struct brw_mipmap_tree *mt, struct brw_miptree_map *map, - unsigned int level, unsigned int slice, unsigned int *x1_B, - unsigned int *x2_B, unsigned int *y1_el, unsigned int *y2_el) -{ - unsigned int block_width, block_height; - unsigned int x0_el, y0_el; - - _mesa_get_format_block_size(mt->format, &block_width, &block_height); - - assert(map->x % block_width == 0); - assert(map->y % block_height == 0); - - brw_miptree_get_image_offset(mt, level, slice, &x0_el, &y0_el); - *x1_B = (map->x / block_width + x0_el) * mt->cpp; - *y1_el = map->y / block_height + y0_el; - *x2_B = (DIV_ROUND_UP(map->x + map->w, block_width) + x0_el) * mt->cpp; - *y2_el = DIV_ROUND_UP(map->y + map->h, block_height) + y0_el; -} - -static void -brw_miptree_unmap_tiled_memcpy(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, - unsigned int slice) -{ - if (map->mode & GL_MAP_WRITE_BIT) { - unsigned int x1, x2, y1, y2; - tile_extents(mt, map, level, slice, &x1, &x2, &y1, &y2); - - char *dst = brw_miptree_map_raw(brw, mt, map->mode | MAP_RAW); - dst += mt->offset; - - isl_memcpy_linear_to_tiled( - x1, x2, y1, y2, dst, map->ptr, mt->surf.row_pitch_B, map->stride, - brw->has_swizzling, mt->surf.tiling, ISL_MEMCPY); - - brw_miptree_unmap_raw(mt); - } - align_free(map->buffer); - map->buffer = map->ptr = NULL; -} - -/** - * Determine which copy function to use for the given format combination - * - * The only two possible copy functions which are ever returned are a - * direct memcpy and a RGBA <-> BGRA copy function. Since RGBA -> BGRA and - * BGRA -> RGBA are exactly the same operation (and memcpy is obviously - * symmetric), it doesn't matter whether the copy is from the tiled image - * to the untiled or vice versa. The copy function required is the same in - * either case so this function can be used. - * - * \param[in] tiledFormat The format of the tiled image - * \param[in] format The GL format of the client data - * \param[in] type The GL type of the client data - * \param[out] mem_copy Will be set to one of either the standard - * library's memcpy or a different copy function - * that performs an RGBA to BGRA conversion - * \param[out] cpp Number of bytes per channel - * - * \return true if the format and type combination are valid - */ -isl_memcpy_type -brw_miptree_get_memcpy_type(mesa_format tiledFormat, GLenum format, GLenum type, - uint32_t *cpp) -{ - if (type == GL_UNSIGNED_INT_8_8_8_8_REV && - !(format == GL_RGBA || format == GL_BGRA)) - return ISL_MEMCPY_INVALID; /* Invalid type/format combination */ - - if ((tiledFormat == MESA_FORMAT_L_UNORM8 && format == GL_LUMINANCE) || - (tiledFormat == MESA_FORMAT_A_UNORM8 && format == GL_ALPHA)) { - *cpp = 1; - return ISL_MEMCPY; - } else if ((tiledFormat == MESA_FORMAT_B8G8R8A8_UNORM) || - (tiledFormat == MESA_FORMAT_B8G8R8X8_UNORM) || - (tiledFormat == MESA_FORMAT_B8G8R8A8_SRGB) || - (tiledFormat == MESA_FORMAT_B8G8R8X8_SRGB)) { - *cpp = 4; - if (format == GL_BGRA) { - return ISL_MEMCPY; - } else if (format == GL_RGBA) { - return ISL_MEMCPY_BGRA8; - } - } else if ((tiledFormat == MESA_FORMAT_R8G8B8A8_UNORM) || - (tiledFormat == MESA_FORMAT_R8G8B8X8_UNORM) || - (tiledFormat == MESA_FORMAT_R8G8B8A8_SRGB) || - (tiledFormat == MESA_FORMAT_R8G8B8X8_SRGB)) { - *cpp = 4; - if (format == GL_BGRA) { - /* Copying from RGBA to BGRA is the same as BGRA to RGBA so we can - * use the same function. - */ - return ISL_MEMCPY_BGRA8; - } else if (format == GL_RGBA) { - return ISL_MEMCPY; - } - } - - return ISL_MEMCPY_INVALID; -} - -static void -brw_miptree_map_tiled_memcpy(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, unsigned int slice) -{ - brw_miptree_access_raw(brw, mt, level, slice, - map->mode & GL_MAP_WRITE_BIT); - - unsigned int x1, x2, y1, y2; - tile_extents(mt, map, level, slice, &x1, &x2, &y1, &y2); - map->stride = ALIGN(_mesa_format_row_stride(mt->format, map->w), 16); - - /* The tiling and detiling functions require that the linear buffer - * has proper 16-byte alignment (that is, its `x0` is 16-byte - * aligned). Here we over-allocate the linear buffer by enough - * bytes to get the proper alignment. - */ - map->buffer = align_malloc(map->stride * (y2 - y1) + (x1 & 0xf), 16); - map->ptr = (char *)map->buffer + (x1 & 0xf); - assert(map->buffer); - - if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { - char *src = brw_miptree_map_raw(brw, mt, map->mode | MAP_RAW); - src += mt->offset; - - const isl_memcpy_type copy_type = -#if defined(USE_SSE41) - cpu_has_sse4_1 ? ISL_MEMCPY_STREAMING_LOAD : -#endif - ISL_MEMCPY; - - isl_memcpy_tiled_to_linear( - x1, x2, y1, y2, map->ptr, src, map->stride, - mt->surf.row_pitch_B, brw->has_swizzling, mt->surf.tiling, - copy_type); - - brw_miptree_unmap_raw(mt); - } - - map->unmap = brw_miptree_unmap_tiled_memcpy; -} - -static void -brw_miptree_map_blit(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, unsigned int slice) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - map->linear_mt = make_surface(brw, GL_TEXTURE_2D, mt->format, - 0, 0, map->w, map->h, 1, 1, - ISL_TILING_LINEAR_BIT, - ISL_SURF_USAGE_RENDER_TARGET_BIT | - ISL_SURF_USAGE_TEXTURE_BIT, - 0, 0, NULL); - - if (!map->linear_mt) { - fprintf(stderr, "Failed to allocate blit temporary\n"); - goto fail; - } - map->stride = map->linear_mt->surf.row_pitch_B; - - /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no - * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless - * invalidate is set, since we'll be writing the whole rectangle from our - * temporary buffer back out. - */ - if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { - if (devinfo->gen >= 6) { - brw_blorp_copy_miptrees(brw, mt, level, slice, - map->linear_mt, 0, 0, - map->x, map->y, 0, 0, map->w, map->h); - } else { - if (!brw_miptree_copy(brw, - mt, level, slice, map->x, map->y, - map->linear_mt, 0, 0, 0, 0, - map->w, map->h)) { - fprintf(stderr, "Failed to blit\n"); - goto fail; - } - } - } - - map->ptr = brw_miptree_map_raw(brw, map->linear_mt, map->mode); - - DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__, - map->x, map->y, map->w, map->h, - mt, _mesa_get_format_name(mt->format), - level, slice, map->ptr, map->stride); - - map->unmap = brw_miptree_unmap_blit; - return; - -fail: - brw_miptree_release(&map->linear_mt); - map->ptr = NULL; - map->stride = 0; -} - -/** - * "Map" a buffer by copying it to an untiled temporary using MOVNTDQA. - */ -#if defined(USE_SSE41) -static void -brw_miptree_unmap_movntdqa(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, - unsigned int slice) -{ - align_free(map->buffer); - map->buffer = NULL; - map->ptr = NULL; -} - -static void -brw_miptree_map_movntdqa(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, unsigned int slice) -{ - assert(map->mode & GL_MAP_READ_BIT); - assert(!(map->mode & GL_MAP_WRITE_BIT)); - - brw_miptree_access_raw(brw, mt, level, slice, false); - - DBG("%s: %d,%d %dx%d from mt %p (%s) %d,%d = %p/%d\n", __func__, - map->x, map->y, map->w, map->h, - mt, _mesa_get_format_name(mt->format), - level, slice, map->ptr, map->stride); - - /* Map the original image */ - uint32_t image_x; - uint32_t image_y; - brw_miptree_get_image_offset(mt, level, slice, &image_x, &image_y); - image_x += map->x; - image_y += map->y; - - void *src = brw_miptree_map_raw(brw, mt, map->mode); - if (!src) - return; - - src += mt->offset; - - src += image_y * mt->surf.row_pitch_B; - src += image_x * mt->cpp; - - /* Due to the pixel offsets for the particular image being mapped, our - * src pointer may not be 16-byte aligned. However, if the pitch is - * divisible by 16, then the amount by which it's misaligned will remain - * consistent from row to row. - */ - assert((mt->surf.row_pitch_B % 16) == 0); - const int misalignment = ((uintptr_t) src) & 15; - - /* Create an untiled temporary buffer for the mapping. */ - const unsigned width_bytes = _mesa_format_row_stride(mt->format, map->w); - - map->stride = ALIGN(misalignment + width_bytes, 16); - - map->buffer = align_malloc(map->stride * map->h, 16); - /* Offset the destination so it has the same misalignment as src. */ - map->ptr = map->buffer + misalignment; - - assert((((uintptr_t) map->ptr) & 15) == misalignment); - - for (uint32_t y = 0; y < map->h; y++) { - void *dst_ptr = map->ptr + y * map->stride; - void *src_ptr = src + y * mt->surf.row_pitch_B; - - _mesa_streaming_load_memcpy(dst_ptr, src_ptr, width_bytes); - } - - brw_miptree_unmap_raw(mt); - - map->unmap = brw_miptree_unmap_movntdqa; -} -#endif - -static void -brw_miptree_unmap_s8(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, - unsigned int slice) -{ - if (map->mode & GL_MAP_WRITE_BIT) { - unsigned int image_x, image_y; - uint8_t *untiled_s8_map = map->ptr; - uint8_t *tiled_s8_map = brw_miptree_map_raw(brw, mt, GL_MAP_WRITE_BIT); - - brw_miptree_get_image_offset(mt, level, slice, &image_x, &image_y); - - for (uint32_t y = 0; y < map->h; y++) { - for (uint32_t x = 0; x < map->w; x++) { - ptrdiff_t offset = brw_offset_S8(mt->surf.row_pitch_B, - image_x + x + map->x, - image_y + y + map->y, - brw->has_swizzling); - tiled_s8_map[offset] = untiled_s8_map[y * map->w + x]; - } - } - - brw_miptree_unmap_raw(mt); - } - - free(map->buffer); -} - -static void -brw_miptree_map_s8(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, unsigned int slice) -{ - map->stride = map->w; - map->buffer = map->ptr = malloc(map->stride * map->h); - if (!map->buffer) - return; - - brw_miptree_access_raw(brw, mt, level, slice, - map->mode & GL_MAP_WRITE_BIT); - - /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no - * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless - * invalidate is set, since we'll be writing the whole rectangle from our - * temporary buffer back out. - */ - if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { - uint8_t *untiled_s8_map = map->ptr; - uint8_t *tiled_s8_map = brw_miptree_map_raw(brw, mt, GL_MAP_READ_BIT); - unsigned int image_x, image_y; - - brw_miptree_get_image_offset(mt, level, slice, &image_x, &image_y); - - for (uint32_t y = 0; y < map->h; y++) { - for (uint32_t x = 0; x < map->w; x++) { - ptrdiff_t offset = brw_offset_S8(mt->surf.row_pitch_B, - x + image_x + map->x, - y + image_y + map->y, - brw->has_swizzling); - untiled_s8_map[y * map->w + x] = tiled_s8_map[offset]; - } - } - - brw_miptree_unmap_raw(mt); - - DBG("%s: %d,%d %dx%d from mt %p %d,%d = %p/%d\n", __func__, - map->x, map->y, map->w, map->h, - mt, map->x + image_x, map->y + image_y, map->ptr, map->stride); - } else { - DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__, - map->x, map->y, map->w, map->h, - mt, map->ptr, map->stride); - } - - map->unmap = brw_miptree_unmap_s8; -} - -/** - * Mapping functions for packed depth/stencil miptrees backed by real separate - * miptrees for depth and stencil. - * - * On gen7, and to support HiZ pre-gen7, we have to have the stencil buffer - * separate from the depth buffer. Yet at the GL API level, we have to expose - * packed depth/stencil textures and FBO attachments, and Mesa core expects to - * be able to map that memory for texture storage and glReadPixels-type - * operations. We give Mesa core that access by mallocing a temporary and - * copying the data between the actual backing store and the temporary. - */ -static void -brw_miptree_unmap_depthstencil(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, - unsigned int slice) -{ - struct brw_mipmap_tree *z_mt = mt; - struct brw_mipmap_tree *s_mt = mt->stencil_mt; - bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z_FLOAT32; - - if (map->mode & GL_MAP_WRITE_BIT) { - uint32_t *packed_map = map->ptr; - uint8_t *s_map = brw_miptree_map_raw(brw, s_mt, GL_MAP_WRITE_BIT); - uint32_t *z_map = brw_miptree_map_raw(brw, z_mt, GL_MAP_WRITE_BIT); - unsigned int s_image_x, s_image_y; - unsigned int z_image_x, z_image_y; - - brw_miptree_get_image_offset(s_mt, level, slice, - &s_image_x, &s_image_y); - brw_miptree_get_image_offset(z_mt, level, slice, - &z_image_x, &z_image_y); - - for (uint32_t y = 0; y < map->h; y++) { - for (uint32_t x = 0; x < map->w; x++) { - ptrdiff_t s_offset = brw_offset_S8(s_mt->surf.row_pitch_B, - x + s_image_x + map->x, - y + s_image_y + map->y, - brw->has_swizzling); - ptrdiff_t z_offset = ((y + z_image_y + map->y) * - (z_mt->surf.row_pitch_B / 4) + - (x + z_image_x + map->x)); - - if (map_z32f_x24s8) { - z_map[z_offset] = packed_map[(y * map->w + x) * 2 + 0]; - s_map[s_offset] = packed_map[(y * map->w + x) * 2 + 1]; - } else { - uint32_t packed = packed_map[y * map->w + x]; - s_map[s_offset] = packed >> 24; - z_map[z_offset] = packed; - } - } - } - - brw_miptree_unmap_raw(s_mt); - brw_miptree_unmap_raw(z_mt); - - DBG("%s: %d,%d %dx%d from z mt %p (%s) %d,%d, s mt %p %d,%d = %p/%d\n", - __func__, - map->x, map->y, map->w, map->h, - z_mt, _mesa_get_format_name(z_mt->format), - map->x + z_image_x, map->y + z_image_y, - s_mt, map->x + s_image_x, map->y + s_image_y, - map->ptr, map->stride); - } - - free(map->buffer); -} - -static void -brw_miptree_map_depthstencil(struct brw_context *brw, - struct brw_mipmap_tree *mt, - struct brw_miptree_map *map, - unsigned int level, unsigned int slice) -{ - struct brw_mipmap_tree *z_mt = mt; - struct brw_mipmap_tree *s_mt = mt->stencil_mt; - bool map_z32f_x24s8 = mt->format == MESA_FORMAT_Z_FLOAT32; - int packed_bpp = map_z32f_x24s8 ? 8 : 4; - - map->stride = map->w * packed_bpp; - map->buffer = map->ptr = malloc(map->stride * map->h); - if (!map->buffer) - return; - - brw_miptree_access_raw(brw, z_mt, level, slice, - map->mode & GL_MAP_WRITE_BIT); - brw_miptree_access_raw(brw, s_mt, level, slice, - map->mode & GL_MAP_WRITE_BIT); - - /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no - * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless - * invalidate is set, since we'll be writing the whole rectangle from our - * temporary buffer back out. - */ - if (!(map->mode & GL_MAP_INVALIDATE_RANGE_BIT)) { - uint32_t *packed_map = map->ptr; - uint8_t *s_map = brw_miptree_map_raw(brw, s_mt, GL_MAP_READ_BIT); - uint32_t *z_map = brw_miptree_map_raw(brw, z_mt, GL_MAP_READ_BIT); - unsigned int s_image_x, s_image_y; - unsigned int z_image_x, z_image_y; - - brw_miptree_get_image_offset(s_mt, level, slice, - &s_image_x, &s_image_y); - brw_miptree_get_image_offset(z_mt, level, slice, - &z_image_x, &z_image_y); - - for (uint32_t y = 0; y < map->h; y++) { - for (uint32_t x = 0; x < map->w; x++) { - int map_x = map->x + x, map_y = map->y + y; - ptrdiff_t s_offset = brw_offset_S8(s_mt->surf.row_pitch_B, - map_x + s_image_x, - map_y + s_image_y, - brw->has_swizzling); - ptrdiff_t z_offset = ((map_y + z_image_y) * - (z_mt->surf.row_pitch_B / 4) + - (map_x + z_image_x)); - uint8_t s = s_map[s_offset]; - uint32_t z = z_map[z_offset]; - - if (map_z32f_x24s8) { - packed_map[(y * map->w + x) * 2 + 0] = z; - packed_map[(y * map->w + x) * 2 + 1] = s; - } else { - packed_map[y * map->w + x] = (s << 24) | (z & 0x00ffffff); - } - } - } - - brw_miptree_unmap_raw(s_mt); - brw_miptree_unmap_raw(z_mt); - - DBG("%s: %d,%d %dx%d from z mt %p %d,%d, s mt %p %d,%d = %p/%d\n", - __func__, - map->x, map->y, map->w, map->h, - z_mt, map->x + z_image_x, map->y + z_image_y, - s_mt, map->x + s_image_x, map->y + s_image_y, - map->ptr, map->stride); - } else { - DBG("%s: %d,%d %dx%d from mt %p = %p/%d\n", __func__, - map->x, map->y, map->w, map->h, - mt, map->ptr, map->stride); - } - - map->unmap = brw_miptree_unmap_depthstencil; -} - -/** - * Create and attach a map to the miptree at (level, slice). Return the - * attached map. - */ -static struct brw_miptree_map* -brw_miptree_attach_map(struct brw_mipmap_tree *mt, - unsigned int level, - unsigned int slice, - unsigned int x, - unsigned int y, - unsigned int w, - unsigned int h, - GLbitfield mode) -{ - struct brw_miptree_map *map = calloc(1, sizeof(*map)); - - if (!map) - return NULL; - - assert(mt->level[level].slice[slice].map == NULL); - mt->level[level].slice[slice].map = map; - - map->mode = mode; - map->x = x; - map->y = y; - map->w = w; - map->h = h; - - return map; -} - -/** - * Release the map at (level, slice). - */ -static void -brw_miptree_release_map(struct brw_mipmap_tree *mt, - unsigned int level, - unsigned int slice) -{ - struct brw_miptree_map **map; - - map = &mt->level[level].slice[slice].map; - free(*map); - *map = NULL; -} - -static bool -can_blit_slice(struct brw_mipmap_tree *mt, - const struct brw_miptree_map *map) -{ - /* See brw_miptree_blit() for details on the 32k pitch limit. */ - const unsigned src_blt_pitch = brw_miptree_blt_pitch(mt); - const unsigned dst_blt_pitch = ALIGN(map->w * mt->cpp, 64); - return src_blt_pitch < 32768 && dst_blt_pitch < 32768; -} - -static bool -use_blitter_to_map(struct brw_context *brw, - struct brw_mipmap_tree *mt, - const struct brw_miptree_map *map) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - - if (devinfo->has_llc && - /* It's probably not worth swapping to the blit ring because of - * all the overhead involved. - */ - !(map->mode & GL_MAP_WRITE_BIT) && - !mt->compressed && - (mt->surf.tiling == ISL_TILING_X || - /* Prior to Sandybridge, the blitter can't handle Y tiling */ - (devinfo->gen >= 6 && mt->surf.tiling == ISL_TILING_Y0) || - /* Fast copy blit on skl+ supports all tiling formats. */ - devinfo->gen >= 9) && - can_blit_slice(mt, map)) - return true; - - if (mt->surf.tiling != ISL_TILING_LINEAR && - mt->bo->size >= brw->max_gtt_map_object_size) { - assert(can_blit_slice(mt, map)); - return true; - } - - return false; -} - -/** - * Parameter \a out_stride has type ptrdiff_t not because the buffer stride may - * exceed 32 bits but to diminish the likelihood subtle bugs in pointer - * arithmetic overflow. - * - * If you call this function and use \a out_stride, then you're doing pointer - * arithmetic on \a out_ptr. The type of \a out_stride doesn't prevent all - * bugs. The caller must still take care to avoid 32-bit overflow errors in - * all arithmetic expressions that contain buffer offsets and pixel sizes, - * which usually have type uint32_t or GLuint. - */ -void -brw_miptree_map(struct brw_context *brw, - struct brw_mipmap_tree *mt, - unsigned int level, - unsigned int slice, - unsigned int x, - unsigned int y, - unsigned int w, - unsigned int h, - GLbitfield mode, - void **out_ptr, - ptrdiff_t *out_stride) -{ - const struct gen_device_info *devinfo = &brw->screen->devinfo; - struct brw_miptree_map *map; - - assert(mt->surf.samples == 1); - - map = brw_miptree_attach_map(mt, level, slice, x, y, w, h, mode); - if (!map){ - *out_ptr = NULL; - *out_stride = 0; - return; - } - - if (mt->format == MESA_FORMAT_S_UINT8) { - brw_miptree_map_s8(brw, mt, map, level, slice); - } else if (mt->stencil_mt && !(mode & BRW_MAP_DIRECT_BIT)) { - brw_miptree_map_depthstencil(brw, mt, map, level, slice); - } else if (use_blitter_to_map(brw, mt, map)) { - brw_miptree_map_blit(brw, mt, map, level, slice); - } else if (mt->surf.tiling != ISL_TILING_LINEAR && devinfo->gen > 4) { - brw_miptree_map_tiled_memcpy(brw, mt, map, level, slice); -#if defined(USE_SSE41) - } else if (!(mode & GL_MAP_WRITE_BIT) && - !mt->compressed && cpu_has_sse4_1 && - (mt->surf.row_pitch_B % 16 == 0)) { - brw_miptree_map_movntdqa(brw, mt, map, level, slice); -#endif - } else { - if (mt->surf.tiling != ISL_TILING_LINEAR) - perf_debug("brw_miptree_map: mapping via gtt"); - brw_miptree_map_map(brw, mt, map, level, slice); - } - - *out_ptr = map->ptr; - *out_stride = map->stride; - - if (map->ptr == NULL) - brw_miptree_release_map(mt, level, slice); -} - -void -brw_miptree_unmap(struct brw_context *brw, - struct brw_mipmap_tree *mt, - unsigned int level, - unsigned int slice) -{ - struct brw_miptree_map *map = mt->level[level].slice[slice].map; - - assert(mt->surf.samples == 1); - - if (!map) - return; - - DBG("%s: mt %p (%s) level %d slice %d\n", __func__, - mt, _mesa_get_format_name(mt->format), level, slice); - - if (map->unmap) - map->unmap(brw, mt, map, level, slice); - - brw_miptree_release_map(mt, level, slice); -} - -enum isl_surf_dim -get_isl_surf_dim(GLenum target) -{ - switch (target) { - case GL_TEXTURE_1D: - case GL_TEXTURE_1D_ARRAY: - return ISL_SURF_DIM_1D; - - case GL_TEXTURE_2D: - case GL_TEXTURE_2D_ARRAY: - case GL_TEXTURE_RECTANGLE: - case GL_TEXTURE_CUBE_MAP: - case GL_TEXTURE_CUBE_MAP_ARRAY: - case GL_TEXTURE_2D_MULTISAMPLE: - case GL_TEXTURE_2D_MULTISAMPLE_ARRAY: - case GL_TEXTURE_EXTERNAL_OES: - return ISL_SURF_DIM_2D; - - case GL_TEXTURE_3D: - return ISL_SURF_DIM_3D; - } - - unreachable("Invalid texture target"); -} - -enum isl_dim_layout -get_isl_dim_layout(const struct gen_device_info *devinfo, - enum isl_tiling tiling, GLenum target) -{ - switch (target) { - case GL_TEXTURE_1D: - case GL_TEXTURE_1D_ARRAY: - return (devinfo->gen >= 9 && tiling == ISL_TILING_LINEAR ? - ISL_DIM_LAYOUT_GEN9_1D : ISL_DIM_LAYOUT_GEN4_2D); - - case GL_TEXTURE_2D: - case GL_TEXTURE_2D_ARRAY: - case GL_TEXTURE_RECTANGLE: - case GL_TEXTURE_2D_MULTISAMPLE: - case GL_TEXTURE_2D_MULTISAMPLE_ARRAY: - case GL_TEXTURE_EXTERNAL_OES: - return ISL_DIM_LAYOUT_GEN4_2D; - - case GL_TEXTURE_CUBE_MAP: - case GL_TEXTURE_CUBE_MAP_ARRAY: - return (devinfo->gen == 4 ? ISL_DIM_LAYOUT_GEN4_3D : - ISL_DIM_LAYOUT_GEN4_2D); - - case GL_TEXTURE_3D: - return (devinfo->gen >= 9 ? - ISL_DIM_LAYOUT_GEN4_2D : ISL_DIM_LAYOUT_GEN4_3D); - } - - unreachable("Invalid texture target"); -} - -bool -brw_miptree_set_clear_color(struct brw_context *brw, - struct brw_mipmap_tree *mt, - union isl_color_value clear_color) -{ - if (memcmp(&mt->fast_clear_color, &clear_color, sizeof(clear_color)) != 0) { - mt->fast_clear_color = clear_color; - if (mt->aux_buf->clear_color_bo) { - /* We can't update the clear color while the hardware is still using - * the previous one for a resolve or sampling from it. Make sure that - * there are no pending commands at this point. - */ - brw_emit_pipe_control_flush(brw, PIPE_CONTROL_CS_STALL); - for (int i = 0; i < 4; i++) { - brw_store_data_imm32(brw, mt->aux_buf->clear_color_bo, - mt->aux_buf->clear_color_offset + i * 4, - mt->fast_clear_color.u32[i]); - } - brw_emit_pipe_control_flush(brw, PIPE_CONTROL_STATE_CACHE_INVALIDATE); - } - brw->ctx.NewDriverState |= BRW_NEW_AUX_STATE; - return true; - } - return false; -} - -union isl_color_value -brw_miptree_get_clear_color(const struct brw_mipmap_tree *mt, - struct brw_bo **clear_color_bo, - uint64_t *clear_color_offset) -{ - assert(mt->aux_buf); - - *clear_color_bo = mt->aux_buf->clear_color_bo; - *clear_color_offset = mt->aux_buf->clear_color_offset; - return mt->fast_clear_color; -} - -static void -brw_miptree_update_etc_shadow(struct brw_context *brw, - struct brw_mipmap_tree *mt, - unsigned int level, - unsigned int slice, - int level_w, - int level_h) -{ - ptrdiff_t etc_stride, shadow_stride; - void *mptr, *sptr; - struct brw_mipmap_tree *smt = mt->shadow_mt; - - assert(brw_miptree_has_etc_shadow(brw, mt)); - - brw_miptree_map(brw, mt, level, slice, 0, 0, level_w, level_h, - GL_MAP_READ_BIT, &mptr, &etc_stride); - brw_miptree_map(brw, smt, level, slice, 0, 0, level_w, level_h, - GL_MAP_WRITE_BIT | GL_MAP_INVALIDATE_RANGE_BIT, - &sptr, &shadow_stride); - - if (mt->format == MESA_FORMAT_ETC1_RGB8) { - _mesa_etc1_unpack_rgba8888(sptr, shadow_stride, mptr, etc_stride, - level_w, level_h); - } else { - /* destination and source images must have the same swizzle */ - bool is_bgra = (smt->format == MESA_FORMAT_B8G8R8A8_SRGB); - _mesa_unpack_etc2_format(sptr, shadow_stride, mptr, etc_stride, - level_w, level_h, mt->format, is_bgra); - } - - brw_miptree_unmap(brw, mt, level, slice); - brw_miptree_unmap(brw, smt, level, slice); -} - -void -brw_miptree_update_etc_shadow_levels(struct brw_context *brw, - struct brw_mipmap_tree *mt) -{ - struct brw_mipmap_tree *smt; - int num_slices; - - assert(mt); - assert(mt->surf.size_B > 0); - assert(brw_miptree_has_etc_shadow(brw, mt)); - - smt = mt->shadow_mt; - num_slices = smt->surf.logical_level0_px.array_len; - - for (int level = smt->first_level; level <= smt->last_level; level++) { - int level_w = minify(smt->surf.logical_level0_px.width, - level - smt->first_level); - int level_h = minify(smt->surf.logical_level0_px.height, - level - smt->first_level); - - for (unsigned int slice = 0; slice < num_slices; slice++) { - brw_miptree_update_etc_shadow(brw, mt, level, slice, level_w, - level_h); - } - } - - mt->shadow_needs_update = false; -} |