/************************************************************************** * * Copyright 2009 VMware, Inc. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * **************************************************************************/ /** * @file * Helper functions for swizzling/shuffling. * * @author Jose Fonseca */ #include /* for PRIx64 macro */ #include "util/u_debug.h" #include "lp_bld_type.h" #include "lp_bld_const.h" #include "lp_bld_init.h" #include "lp_bld_logic.h" #include "lp_bld_swizzle.h" #include "lp_bld_pack.h" LLVMValueRef lp_build_broadcast(struct gallivm_state *gallivm, LLVMTypeRef vec_type, LLVMValueRef scalar) { LLVMValueRef res; if (LLVMGetTypeKind(vec_type) != LLVMVectorTypeKind) { /* scalar */ assert(vec_type == LLVMTypeOf(scalar)); res = scalar; } else { LLVMBuilderRef builder = gallivm->builder; const unsigned length = LLVMGetVectorSize(vec_type); LLVMValueRef undef = LLVMGetUndef(vec_type); /* The shuffle vector is always made of int32 elements */ LLVMTypeRef i32_type = LLVMInt32TypeInContext(gallivm->context); LLVMTypeRef i32_vec_type = LLVMVectorType(i32_type, length); assert(LLVMGetElementType(vec_type) == LLVMTypeOf(scalar)); res = LLVMBuildInsertElement(builder, undef, scalar, LLVMConstNull(i32_type), ""); res = LLVMBuildShuffleVector(builder, res, undef, LLVMConstNull(i32_vec_type), ""); } return res; } /** * Broadcast */ LLVMValueRef lp_build_broadcast_scalar(struct lp_build_context *bld, LLVMValueRef scalar) { assert(lp_check_elem_type(bld->type, LLVMTypeOf(scalar))); return lp_build_broadcast(bld->gallivm, bld->vec_type, scalar); } /** * Combined extract and broadcast (mere shuffle in most cases) */ LLVMValueRef lp_build_extract_broadcast(struct gallivm_state *gallivm, struct lp_type src_type, struct lp_type dst_type, LLVMValueRef vector, LLVMValueRef index) { LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); LLVMValueRef res; assert(src_type.floating == dst_type.floating); assert(src_type.width == dst_type.width); assert(lp_check_value(src_type, vector)); assert(LLVMTypeOf(index) == i32t); if (src_type.length == 1) { if (dst_type.length == 1) { /* * Trivial scalar -> scalar. */ res = vector; } else { /* * Broadcast scalar -> vector. */ res = lp_build_broadcast(gallivm, lp_build_vec_type(gallivm, dst_type), vector); } } else { if (dst_type.length > 1) { /* * shuffle - result can be of different length. */ LLVMValueRef shuffle; shuffle = lp_build_broadcast(gallivm, LLVMVectorType(i32t, dst_type.length), index); res = LLVMBuildShuffleVector(gallivm->builder, vector, LLVMGetUndef(lp_build_vec_type(gallivm, src_type)), shuffle, ""); } else { /* * Trivial extract scalar from vector. */ res = LLVMBuildExtractElement(gallivm->builder, vector, index, ""); } } return res; } /** * Swizzle one channel into other channels. */ LLVMValueRef lp_build_swizzle_scalar_aos(struct lp_build_context *bld, LLVMValueRef a, unsigned channel, unsigned num_channels) { LLVMBuilderRef builder = bld->gallivm->builder; const struct lp_type type = bld->type; const unsigned n = type.length; unsigned i, j; if(a == bld->undef || a == bld->zero || a == bld->one || num_channels == 1) return a; assert(num_channels == 2 || num_channels == 4); /* XXX: SSE3 has PSHUFB which should be better than bitmasks, but forcing * using shuffles here actually causes worst results. More investigation is * needed. */ if (LLVMIsConstant(a) || type.width >= 16) { /* * Shuffle. */ LLVMTypeRef elem_type = LLVMInt32TypeInContext(bld->gallivm->context); LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; for(j = 0; j < n; j += num_channels) for(i = 0; i < num_channels; ++i) shuffles[j + i] = LLVMConstInt(elem_type, j + channel, 0); return LLVMBuildShuffleVector(builder, a, bld->undef, LLVMConstVector(shuffles, n), ""); } else if (num_channels == 2) { /* * Bit mask and shifts * * XY XY .... XY <= input * 0Y 0Y .... 0Y * YY YY .... YY * YY YY .... YY <= output */ struct lp_type type2; LLVMValueRef tmp = NULL; int shift; a = LLVMBuildAnd(builder, a, lp_build_const_mask_aos(bld->gallivm, type, 1 << channel, num_channels), ""); type2 = type; type2.floating = FALSE; type2.width *= 2; type2.length /= 2; a = LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type2), ""); /* * Vector element 0 is always channel X. * * 76 54 32 10 (array numbering) * Little endian reg in: YX YX YX YX * Little endian reg out: YY YY YY YY if shift right (shift == -1) * XX XX XX XX if shift left (shift == 1) * * 01 23 45 67 (array numbering) * Big endian reg in: XY XY XY XY * Big endian reg out: YY YY YY YY if shift left (shift == 1) * XX XX XX XX if shift right (shift == -1) * */ #ifdef PIPE_ARCH_LITTLE_ENDIAN shift = channel == 0 ? 1 : -1; #else shift = channel == 0 ? -1 : 1; #endif if (shift > 0) { tmp = LLVMBuildShl(builder, a, lp_build_const_int_vec(bld->gallivm, type2, shift * type.width), ""); } else if (shift < 0) { tmp = LLVMBuildLShr(builder, a, lp_build_const_int_vec(bld->gallivm, type2, -shift * type.width), ""); } assert(tmp); if (tmp) { a = LLVMBuildOr(builder, a, tmp, ""); } return LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type), ""); } else { /* * Bit mask and recursive shifts * * Little-endian registers: * * 7654 3210 * WZYX WZYX .... WZYX <= input * 00Y0 00Y0 .... 00Y0 <= mask * 00YY 00YY .... 00YY <= shift right 1 (shift amount -1) * YYYY YYYY .... YYYY <= shift left 2 (shift amount 2) * * Big-endian registers: * * 0123 4567 * XYZW XYZW .... XYZW <= input * 0Y00 0Y00 .... 0Y00 <= mask * YY00 YY00 .... YY00 <= shift left 1 (shift amount 1) * YYYY YYYY .... YYYY <= shift right 2 (shift amount -2) * * shifts[] gives little-endian shift amounts; we need to negate for big-endian. */ struct lp_type type4; const int shifts[4][2] = { { 1, 2}, {-1, 2}, { 1, -2}, {-1, -2} }; unsigned i; a = LLVMBuildAnd(builder, a, lp_build_const_mask_aos(bld->gallivm, type, 1 << channel, 4), ""); /* * Build a type where each element is an integer that cover the four * channels. */ type4 = type; type4.floating = FALSE; type4.width *= 4; type4.length /= 4; a = LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type4), ""); for(i = 0; i < 2; ++i) { LLVMValueRef tmp = NULL; int shift = shifts[channel][i]; /* See endianness diagram above */ #ifdef PIPE_ARCH_BIG_ENDIAN shift = -shift; #endif if(shift > 0) tmp = LLVMBuildShl(builder, a, lp_build_const_int_vec(bld->gallivm, type4, shift*type.width), ""); if(shift < 0) tmp = LLVMBuildLShr(builder, a, lp_build_const_int_vec(bld->gallivm, type4, -shift*type.width), ""); assert(tmp); if(tmp) a = LLVMBuildOr(builder, a, tmp, ""); } return LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type), ""); } } /** * Swizzle a vector consisting of an array of XYZW structs. * * This fills a vector of dst_len length with the swizzled channels from src. * * e.g. with swizzles = { 2, 1, 0 } and swizzle_count = 6 results in * RGBA RGBA = BGR BGR BG * * @param swizzles the swizzle array * @param num_swizzles the number of elements in swizzles * @param dst_len the length of the result */ LLVMValueRef lp_build_swizzle_aos_n(struct gallivm_state* gallivm, LLVMValueRef src, const unsigned char* swizzles, unsigned num_swizzles, unsigned dst_len) { LLVMBuilderRef builder = gallivm->builder; LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH]; unsigned i; assert(dst_len < LP_MAX_VECTOR_WIDTH); for (i = 0; i < dst_len; ++i) { int swizzle = swizzles[i % num_swizzles]; if (swizzle == LP_BLD_SWIZZLE_DONTCARE) { shuffles[i] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context)); } else { shuffles[i] = lp_build_const_int32(gallivm, swizzle); } } return LLVMBuildShuffleVector(builder, src, LLVMGetUndef(LLVMTypeOf(src)), LLVMConstVector(shuffles, dst_len), ""); } LLVMValueRef lp_build_swizzle_aos(struct lp_build_context *bld, LLVMValueRef a, const unsigned char swizzles[4]) { LLVMBuilderRef builder = bld->gallivm->builder; const struct lp_type type = bld->type; const unsigned n = type.length; unsigned i, j; if (swizzles[0] == PIPE_SWIZZLE_RED && swizzles[1] == PIPE_SWIZZLE_GREEN && swizzles[2] == PIPE_SWIZZLE_BLUE && swizzles[3] == PIPE_SWIZZLE_ALPHA) { return a; } if (swizzles[0] == swizzles[1] && swizzles[1] == swizzles[2] && swizzles[2] == swizzles[3]) { switch (swizzles[0]) { case PIPE_SWIZZLE_RED: case PIPE_SWIZZLE_GREEN: case PIPE_SWIZZLE_BLUE: case PIPE_SWIZZLE_ALPHA: return lp_build_swizzle_scalar_aos(bld, a, swizzles[0], 4); case PIPE_SWIZZLE_ZERO: return bld->zero; case PIPE_SWIZZLE_ONE: return bld->one; case LP_BLD_SWIZZLE_DONTCARE: return bld->undef; default: assert(0); return bld->undef; } } if (LLVMIsConstant(a) || type.width >= 16) { /* * Shuffle. */ LLVMValueRef undef = LLVMGetUndef(lp_build_elem_type(bld->gallivm, type)); LLVMTypeRef i32t = LLVMInt32TypeInContext(bld->gallivm->context); LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; LLVMValueRef aux[LP_MAX_VECTOR_LENGTH]; memset(aux, 0, sizeof aux); for(j = 0; j < n; j += 4) { for(i = 0; i < 4; ++i) { unsigned shuffle; switch (swizzles[i]) { default: assert(0); /* fall through */ case PIPE_SWIZZLE_RED: case PIPE_SWIZZLE_GREEN: case PIPE_SWIZZLE_BLUE: case PIPE_SWIZZLE_ALPHA: shuffle = j + swizzles[i]; shuffles[j + i] = LLVMConstInt(i32t, shuffle, 0); break; case PIPE_SWIZZLE_ZERO: shuffle = type.length + 0; shuffles[j + i] = LLVMConstInt(i32t, shuffle, 0); if (!aux[0]) { aux[0] = lp_build_const_elem(bld->gallivm, type, 0.0); } break; case PIPE_SWIZZLE_ONE: shuffle = type.length + 1; shuffles[j + i] = LLVMConstInt(i32t, shuffle, 0); if (!aux[1]) { aux[1] = lp_build_const_elem(bld->gallivm, type, 1.0); } break; case LP_BLD_SWIZZLE_DONTCARE: shuffles[j + i] = LLVMGetUndef(i32t); break; } } } for (i = 0; i < n; ++i) { if (!aux[i]) { aux[i] = undef; } } return LLVMBuildShuffleVector(builder, a, LLVMConstVector(aux, n), LLVMConstVector(shuffles, n), ""); } else { /* * Bit mask and shifts. * * For example, this will convert BGRA to RGBA by doing * * Little endian: * rgba = (bgra & 0x00ff0000) >> 16 * | (bgra & 0xff00ff00) * | (bgra & 0x000000ff) << 16 * * Big endian:A * rgba = (bgra & 0x0000ff00) << 16 * | (bgra & 0x00ff00ff) * | (bgra & 0xff000000) >> 16 * * This is necessary not only for faster cause, but because X86 backend * will refuse shuffles of <4 x i8> vectors */ LLVMValueRef res; struct lp_type type4; unsigned cond = 0; unsigned chan; int shift; /* * Start with a mixture of 1 and 0. */ for (chan = 0; chan < 4; ++chan) { if (swizzles[chan] == PIPE_SWIZZLE_ONE) { cond |= 1 << chan; } } res = lp_build_select_aos(bld, cond, bld->one, bld->zero, 4); /* * Build a type where each element is an integer that cover the four * channels. */ type4 = type; type4.floating = FALSE; type4.width *= 4; type4.length /= 4; a = LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type4), ""); res = LLVMBuildBitCast(builder, res, lp_build_vec_type(bld->gallivm, type4), ""); /* * Mask and shift the channels, trying to group as many channels in the * same shift as possible. The shift amount is positive for shifts left * and negative for shifts right. */ for (shift = -3; shift <= 3; ++shift) { uint64_t mask = 0; assert(type4.width <= sizeof(mask)*8); /* * Vector element numbers follow the XYZW order, so 0 is always X, etc. * After widening 4 times we have: * * 3210 * Little-endian register layout: WZYX * * 0123 * Big-endian register layout: XYZW * * For little-endian, higher-numbered channels are obtained by a shift right * (negative shift amount) and lower-numbered channels by a shift left * (positive shift amount). The opposite is true for big-endian. */ for (chan = 0; chan < 4; ++chan) { if (swizzles[chan] < 4) { /* We need to move channel swizzles[chan] into channel chan */ #ifdef PIPE_ARCH_LITTLE_ENDIAN if (swizzles[chan] - chan == -shift) { mask |= ((1ULL << type.width) - 1) << (swizzles[chan] * type.width); } #else if (swizzles[chan] - chan == shift) { mask |= ((1ULL << type.width) - 1) << (type4.width - type.width) >> (swizzles[chan] * type.width); } #endif } } if (mask) { LLVMValueRef masked; LLVMValueRef shifted; if (0) debug_printf("shift = %i, mask = %" PRIx64 "\n", shift, mask); masked = LLVMBuildAnd(builder, a, lp_build_const_int_vec(bld->gallivm, type4, mask), ""); if (shift > 0) { shifted = LLVMBuildShl(builder, masked, lp_build_const_int_vec(bld->gallivm, type4, shift*type.width), ""); } else if (shift < 0) { shifted = LLVMBuildLShr(builder, masked, lp_build_const_int_vec(bld->gallivm, type4, -shift*type.width), ""); } else { shifted = masked; } res = LLVMBuildOr(builder, res, shifted, ""); } } return LLVMBuildBitCast(builder, res, lp_build_vec_type(bld->gallivm, type), ""); } } /** * Extended swizzle of a single channel of a SoA vector. * * @param bld building context * @param unswizzled array with the 4 unswizzled values * @param swizzle one of the PIPE_SWIZZLE_* * * @return the swizzled value. */ LLVMValueRef lp_build_swizzle_soa_channel(struct lp_build_context *bld, const LLVMValueRef *unswizzled, unsigned swizzle) { switch (swizzle) { case PIPE_SWIZZLE_RED: case PIPE_SWIZZLE_GREEN: case PIPE_SWIZZLE_BLUE: case PIPE_SWIZZLE_ALPHA: return unswizzled[swizzle]; case PIPE_SWIZZLE_ZERO: return bld->zero; case PIPE_SWIZZLE_ONE: return bld->one; default: assert(0); return bld->undef; } } /** * Extended swizzle of a SoA vector. * * @param bld building context * @param unswizzled array with the 4 unswizzled values * @param swizzles array of PIPE_SWIZZLE_* * @param swizzled output swizzled values */ void lp_build_swizzle_soa(struct lp_build_context *bld, const LLVMValueRef *unswizzled, const unsigned char swizzles[4], LLVMValueRef *swizzled) { unsigned chan; for (chan = 0; chan < 4; ++chan) { swizzled[chan] = lp_build_swizzle_soa_channel(bld, unswizzled, swizzles[chan]); } } /** * Do an extended swizzle of a SoA vector inplace. * * @param bld building context * @param values intput/output array with the 4 values * @param swizzles array of PIPE_SWIZZLE_* */ void lp_build_swizzle_soa_inplace(struct lp_build_context *bld, LLVMValueRef *values, const unsigned char swizzles[4]) { LLVMValueRef unswizzled[4]; unsigned chan; for (chan = 0; chan < 4; ++chan) { unswizzled[chan] = values[chan]; } lp_build_swizzle_soa(bld, unswizzled, swizzles, values); } /** * Transpose from AOS <-> SOA * * @param single_type_lp type of pixels * @param src the 4 * n pixel input * @param dst the 4 * n pixel output */ void lp_build_transpose_aos(struct gallivm_state *gallivm, struct lp_type single_type_lp, const LLVMValueRef src[4], LLVMValueRef dst[4]) { struct lp_type double_type_lp = single_type_lp; LLVMTypeRef single_type; LLVMTypeRef double_type; LLVMValueRef t0, t1, t2, t3; double_type_lp.length >>= 1; double_type_lp.width <<= 1; double_type = lp_build_vec_type(gallivm, double_type_lp); single_type = lp_build_vec_type(gallivm, single_type_lp); /* Interleave x, y, z, w -> xy and zw */ t0 = lp_build_interleave2_half(gallivm, single_type_lp, src[0], src[1], 0); t1 = lp_build_interleave2_half(gallivm, single_type_lp, src[2], src[3], 0); t2 = lp_build_interleave2_half(gallivm, single_type_lp, src[0], src[1], 1); t3 = lp_build_interleave2_half(gallivm, single_type_lp, src[2], src[3], 1); /* Cast to double width type for second interleave */ t0 = LLVMBuildBitCast(gallivm->builder, t0, double_type, "t0"); t1 = LLVMBuildBitCast(gallivm->builder, t1, double_type, "t1"); t2 = LLVMBuildBitCast(gallivm->builder, t2, double_type, "t2"); t3 = LLVMBuildBitCast(gallivm->builder, t3, double_type, "t3"); /* Interleave xy, zw -> xyzw */ dst[0] = lp_build_interleave2_half(gallivm, double_type_lp, t0, t1, 0); dst[1] = lp_build_interleave2_half(gallivm, double_type_lp, t0, t1, 1); dst[2] = lp_build_interleave2_half(gallivm, double_type_lp, t2, t3, 0); dst[3] = lp_build_interleave2_half(gallivm, double_type_lp, t2, t3, 1); /* Cast back to original single width type */ dst[0] = LLVMBuildBitCast(gallivm->builder, dst[0], single_type, "dst0"); dst[1] = LLVMBuildBitCast(gallivm->builder, dst[1], single_type, "dst1"); dst[2] = LLVMBuildBitCast(gallivm->builder, dst[2], single_type, "dst2"); dst[3] = LLVMBuildBitCast(gallivm->builder, dst[3], single_type, "dst3"); } /** * Transpose from AOS <-> SOA for num_srcs */ void lp_build_transpose_aos_n(struct gallivm_state *gallivm, struct lp_type type, const LLVMValueRef* src, unsigned num_srcs, LLVMValueRef* dst) { switch (num_srcs) { case 1: dst[0] = src[0]; break; case 2: { /* Note: we must use a temporary incase src == dst */ LLVMValueRef lo, hi; lo = lp_build_interleave2_half(gallivm, type, src[0], src[1], 0); hi = lp_build_interleave2_half(gallivm, type, src[0], src[1], 1); dst[0] = lo; dst[1] = hi; break; } case 4: lp_build_transpose_aos(gallivm, type, src, dst); break; default: assert(0); }; } /** * Pack n-th element of aos values, * pad out to destination size. * i.e. x1 y1 _ _ x2 y2 _ _ will become x1 x2 _ _ */ LLVMValueRef lp_build_pack_aos_scalars(struct gallivm_state *gallivm, struct lp_type src_type, struct lp_type dst_type, const LLVMValueRef src, unsigned channel) { LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); LLVMValueRef undef = LLVMGetUndef(i32t); LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; unsigned num_src = src_type.length / 4; unsigned num_dst = dst_type.length; unsigned i; assert(num_src <= num_dst); for (i = 0; i < num_src; i++) { shuffles[i] = LLVMConstInt(i32t, i * 4 + channel, 0); } for (i = num_src; i < num_dst; i++) { shuffles[i] = undef; } if (num_dst == 1) { return LLVMBuildExtractElement(gallivm->builder, src, shuffles[0], ""); } else { return LLVMBuildShuffleVector(gallivm->builder, src, src, LLVMConstVector(shuffles, num_dst), ""); } } /** * Unpack and broadcast packed aos values consisting of only the * first value, i.e. x1 x2 _ _ will become x1 x1 x1 x1 x2 x2 x2 x2 */ LLVMValueRef lp_build_unpack_broadcast_aos_scalars(struct gallivm_state *gallivm, struct lp_type src_type, struct lp_type dst_type, const LLVMValueRef src) { LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; unsigned num_dst = dst_type.length; unsigned num_src = dst_type.length / 4; unsigned i; assert(num_dst / 4 <= src_type.length); for (i = 0; i < num_src; i++) { shuffles[i*4] = LLVMConstInt(i32t, i, 0); shuffles[i*4+1] = LLVMConstInt(i32t, i, 0); shuffles[i*4+2] = LLVMConstInt(i32t, i, 0); shuffles[i*4+3] = LLVMConstInt(i32t, i, 0); } if (num_src == 1) { return lp_build_extract_broadcast(gallivm, src_type, dst_type, src, shuffles[0]); } else { return LLVMBuildShuffleVector(gallivm->builder, src, src, LLVMConstVector(shuffles, num_dst), ""); } }