/* * Copyright © 2013-2015 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include "brw_vec4_surface_builder.h" using namespace brw; namespace { namespace array_utils { /** * Copy one every \p src_stride logical components of the argument into * one every \p dst_stride logical components of the result. */ src_reg emit_stride(const vec4_builder &bld, const src_reg &src, unsigned size, unsigned dst_stride, unsigned src_stride) { if (src_stride == 1 && dst_stride == 1) { return src; } else { const dst_reg dst = bld.vgrf(src.type, DIV_ROUND_UP(size * dst_stride, 4)); for (unsigned i = 0; i < size; ++i) bld.MOV(writemask(offset(dst, 8, i * dst_stride / 4), 1 << (i * dst_stride % 4)), swizzle(offset(src, 8, i * src_stride / 4), brw_swizzle_for_mask(1 << (i * src_stride % 4)))); return src_reg(dst); } } /** * Convert a VEC4 into an array of registers with the layout expected by * the recipient shared unit. If \p has_simd4x2 is true the argument is * left unmodified in SIMD4x2 form, otherwise it will be rearranged into * a SIMD8 vector. */ src_reg emit_insert(const vec4_builder &bld, const src_reg &src, unsigned n, bool has_simd4x2) { if (src.file == BAD_FILE || n == 0) { return src_reg(); } else { /* Pad unused components with zeroes. */ const unsigned mask = (1 << n) - 1; const dst_reg tmp = bld.vgrf(src.type); bld.MOV(writemask(tmp, mask), src); if (n < 4) bld.MOV(writemask(tmp, ~mask), brw_imm_d(0)); return emit_stride(bld, src_reg(tmp), n, has_simd4x2 ? 1 : 4, 1); } } /** * Convert an array of registers back into a VEC4 according to the * layout expected from some shared unit. If \p has_simd4x2 is true the * argument is left unmodified in SIMD4x2 form, otherwise it will be * rearranged from SIMD8 form. */ src_reg emit_extract(const vec4_builder &bld, const src_reg src, unsigned n, bool has_simd4x2) { if (src.file == BAD_FILE || n == 0) { return src_reg(); } else { return emit_stride(bld, src, n, 1, has_simd4x2 ? 1 : 4); } } } } namespace brw { namespace surface_access { namespace { using namespace array_utils; /** * Generate a send opcode for a surface message and return the * result. */ src_reg emit_send(const vec4_builder &bld, enum opcode op, const src_reg &header, const src_reg &addr, unsigned addr_sz, const src_reg &src, unsigned src_sz, const src_reg &surface, unsigned arg, unsigned ret_sz, brw_predicate pred = BRW_PREDICATE_NONE) { /* Calculate the total number of components of the payload. */ const unsigned header_sz = (header.file == BAD_FILE ? 0 : 1); const unsigned sz = header_sz + addr_sz + src_sz; /* Construct the payload. */ const dst_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, sz); unsigned n = 0; if (header_sz) bld.exec_all().MOV(offset(payload, 8, n++), retype(header, BRW_REGISTER_TYPE_UD)); for (unsigned i = 0; i < addr_sz; i++) bld.MOV(offset(payload, 8, n++), offset(retype(addr, BRW_REGISTER_TYPE_UD), 8, i)); for (unsigned i = 0; i < src_sz; i++) bld.MOV(offset(payload, 8, n++), offset(retype(src, BRW_REGISTER_TYPE_UD), 8, i)); /* Reduce the dynamically uniform surface index to a single * scalar. */ const src_reg usurface = bld.emit_uniformize(surface); /* Emit the message send instruction. */ const dst_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, ret_sz); vec4_instruction *inst = bld.emit(op, dst, src_reg(payload), usurface, brw_imm_ud(arg)); inst->mlen = sz; inst->size_written = ret_sz * REG_SIZE; inst->header_size = header_sz; inst->predicate = pred; return src_reg(dst); } } /** * Emit an untyped surface read opcode. \p dims determines the number * of components of the address and \p size the number of components of * the returned value. */ src_reg emit_untyped_read(const vec4_builder &bld, const src_reg &surface, const src_reg &addr, unsigned dims, unsigned size, brw_predicate pred) { return emit_send(bld, SHADER_OPCODE_UNTYPED_SURFACE_READ, src_reg(), emit_insert(bld, addr, dims, true), 1, src_reg(), 0, surface, size, 1, pred); } /** * Emit an untyped surface write opcode. \p dims determines the number * of components of the address and \p size the number of components of * the argument. */ void emit_untyped_write(const vec4_builder &bld, const src_reg &surface, const src_reg &addr, const src_reg &src, unsigned dims, unsigned size, brw_predicate pred) { const bool has_simd4x2 = (bld.shader->devinfo->gen >= 8 || bld.shader->devinfo->is_haswell); emit_send(bld, SHADER_OPCODE_UNTYPED_SURFACE_WRITE, src_reg(), emit_insert(bld, addr, dims, has_simd4x2), has_simd4x2 ? 1 : dims, emit_insert(bld, src, size, has_simd4x2), has_simd4x2 ? 1 : size, surface, size, 0, pred); } /** * Emit an untyped surface atomic opcode. \p dims determines the number * of components of the address and \p rsize the number of components of * the returned value (either zero or one). */ src_reg emit_untyped_atomic(const vec4_builder &bld, const src_reg &surface, const src_reg &addr, const src_reg &src0, const src_reg &src1, unsigned dims, unsigned rsize, unsigned op, brw_predicate pred) { const bool has_simd4x2 = (bld.shader->devinfo->gen >= 8 || bld.shader->devinfo->is_haswell); /* Zip the components of both sources, they are represented as the X * and Y components of the same vector. */ const unsigned size = (src0.file != BAD_FILE) + (src1.file != BAD_FILE); const dst_reg srcs = bld.vgrf(BRW_REGISTER_TYPE_UD); if (size >= 1) bld.MOV(writemask(srcs, WRITEMASK_X), src0); if (size >= 2) bld.MOV(writemask(srcs, WRITEMASK_Y), src1); return emit_send(bld, SHADER_OPCODE_UNTYPED_ATOMIC, src_reg(), emit_insert(bld, addr, dims, has_simd4x2), has_simd4x2 ? 1 : dims, emit_insert(bld, src_reg(srcs), size, has_simd4x2), has_simd4x2 && size ? 1 : size, surface, op, rsize, pred); } namespace { /** * Initialize the header present in typed surface messages. */ src_reg emit_typed_message_header(const vec4_builder &bld) { const vec4_builder ubld = bld.exec_all(); const dst_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD); ubld.MOV(dst, brw_imm_d(0)); if (bld.shader->devinfo->gen == 7 && !bld.shader->devinfo->is_haswell) { /* The sample mask is used on IVB for the SIMD8 messages that * have no SIMD4x2 variant. We only use the two X channels * in that case, mask everything else out. */ ubld.MOV(writemask(dst, WRITEMASK_W), brw_imm_d(0x11)); } return src_reg(dst); } } /** * Emit a typed surface read opcode. \p dims determines the number of * components of the address and \p size the number of components of the * returned value. */ src_reg emit_typed_read(const vec4_builder &bld, const src_reg &surface, const src_reg &addr, unsigned dims, unsigned size) { const bool has_simd4x2 = (bld.shader->devinfo->gen >= 8 || bld.shader->devinfo->is_haswell); const src_reg tmp = emit_send(bld, SHADER_OPCODE_TYPED_SURFACE_READ, emit_typed_message_header(bld), emit_insert(bld, addr, dims, has_simd4x2), has_simd4x2 ? 1 : dims, src_reg(), 0, surface, size, has_simd4x2 ? 1 : size); return emit_extract(bld, tmp, size, has_simd4x2); } /** * Emit a typed surface write opcode. \p dims determines the number of * components of the address and \p size the number of components of the * argument. */ void emit_typed_write(const vec4_builder &bld, const src_reg &surface, const src_reg &addr, const src_reg &src, unsigned dims, unsigned size) { const bool has_simd4x2 = (bld.shader->devinfo->gen >= 8 || bld.shader->devinfo->is_haswell); emit_send(bld, SHADER_OPCODE_TYPED_SURFACE_WRITE, emit_typed_message_header(bld), emit_insert(bld, addr, dims, has_simd4x2), has_simd4x2 ? 1 : dims, emit_insert(bld, src, size, has_simd4x2), has_simd4x2 ? 1 : size, surface, size, 0); } /** * Emit a typed surface atomic opcode. \p dims determines the number of * components of the address and \p rsize the number of components of * the returned value (either zero or one). */ src_reg emit_typed_atomic(const vec4_builder &bld, const src_reg &surface, const src_reg &addr, const src_reg &src0, const src_reg &src1, unsigned dims, unsigned rsize, unsigned op, brw_predicate pred) { const bool has_simd4x2 = (bld.shader->devinfo->gen >= 8 || bld.shader->devinfo->is_haswell); /* Zip the components of both sources, they are represented as the X * and Y components of the same vector. */ const unsigned size = (src0.file != BAD_FILE) + (src1.file != BAD_FILE); const dst_reg srcs = bld.vgrf(BRW_REGISTER_TYPE_UD); if (size >= 1) bld.MOV(writemask(srcs, WRITEMASK_X), src0); if (size >= 2) bld.MOV(writemask(srcs, WRITEMASK_Y), src1); return emit_send(bld, SHADER_OPCODE_TYPED_ATOMIC, emit_typed_message_header(bld), emit_insert(bld, addr, dims, has_simd4x2), has_simd4x2 ? 1 : dims, emit_insert(bld, src_reg(srcs), size, has_simd4x2), has_simd4x2 ? 1 : size, surface, op, rsize, pred); } } }