gallivm/nir: add support for 16-bit operations

4 files changed, 80 insertions, 25 deletions

diff --git a/src/gallium/auxiliary/gallivm/lp_bld_logic.c b/src/gallium/auxiliary/gallivm/lp_bld_logic.c
index 315977ae745..a1d18619c4e 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_logic.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_logic.c
@@ -273,6 +273,8 @@ lp_build_select_bitwise(struct lp_build_context *bld,
 
    if (type.width > 32)
       mask = LLVMBuildSExt(builder, mask, int_vec_type, "");
+   else if (type.width < 32)
+      mask = LLVMBuildTrunc(builder, mask, int_vec_type, "");
    a = LLVMBuildAnd(builder, a, mask, "");
 
    /* This often gets translated to PANDN, but sometimes the NOT is
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir.c b/src/gallium/auxiliary/gallivm/lp_bld_nir.c
index 86f4638f4e1..ac8aaacf66c 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_nir.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_nir.c
@@ -46,6 +46,8 @@ static LLVMValueRef cast_type(struct lp_build_nir_context *bld_base, LLVMValueRe
    switch (alu_type) {
    case nir_type_float:
       switch (bit_size) {
+      case 16:
+         return LLVMBuildBitCast(builder, val, LLVMVectorType(LLVMHalfTypeInContext(bld_base->base.gallivm->context), bld_base->base.type.length), "");
       case 32:
          return LLVMBuildBitCast(builder, val, bld_base->base.vec_type, "");
       case 64:
@@ -489,9 +491,18 @@ static LLVMValueRef do_alu_action(struct lp_build_nir_context *bld_base,
    case nir_op_f2b32:
       result = flt_to_bool32(bld_base, src_bit_size[0], src[0]);
       break;
-   case nir_op_f2f32:
+   case nir_op_f2f16:
+   case nir_op_f2f16_rtz:
+   case nir_op_f2f16_rtne:
       result = LLVMBuildFPTrunc(builder, src[0],
-                                bld_base->base.vec_type, "");
+                                LLVMVectorType(LLVMHalfTypeInContext(gallivm->context), bld_base->base.type.length), "");
+      break;
+   case nir_op_f2f32:
+      if (src_bit_size[0] < 32)
+         result = LLVMBuildFPExt(builder, src[0], bld_base->base.vec_type, "");
+      else
+         result = LLVMBuildFPTrunc(builder, src[0],
+                                   bld_base->base.vec_type, "");
       break;
    case nir_op_f2f64:
       result = LLVMBuildFPExt(builder, src[0],
@@ -500,6 +511,14 @@ static LLVMValueRef do_alu_action(struct lp_build_nir_context *bld_base,
    case nir_op_f2i32:
       result = LLVMBuildFPToSI(builder, src[0], bld_base->base.int_vec_type, "");
       break;
+   case nir_op_f2i16:
+      result = LLVMBuildFPToSI(builder, src[0], bld_base->int16_bld.vec_type, "");
+      break;
+   case nir_op_f2u16:
+      result = LLVMBuildFPToUI(builder,
+                               src[0],
+                               bld_base->uint16_bld.vec_type, "");
+      break;
    case nir_op_f2u32:
       result = LLVMBuildFPToUI(builder,
                                src[0],
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir.h b/src/gallium/auxiliary/gallivm/lp_bld_nir.h
index 7f29575bad5..af35a831f92 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_nir.h
+++ b/src/gallium/auxiliary/gallivm/lp_bld_nir.h
@@ -50,6 +50,7 @@ struct lp_build_nir_context
    struct lp_build_context uint16_bld;
    struct lp_build_context int16_bld;
    struct lp_build_context dbl_bld;
+   struct lp_build_context half_bld;
    struct lp_build_context uint64_bld;
    struct lp_build_context int64_bld;
 
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c
index b83acd0b58f..2352746d8fd 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c
@@ -35,6 +35,21 @@
 #include "lp_bld_coro.h"
 #include "lp_bld_printf.h"
 #include "util/u_math.h"
+
+static unsigned bit_size_to_shift(unsigned bit_size)
+{
+  switch (bit_size) {
+  case 32:
+  default:
+    return 2;
+  case 64:
+    return 3;
+  case 16:
+    return 1;
+  case 8:
+    return 0;
+  }
+}
 /*
  * combine the execution mask if there is one with the current mask.
  */
@@ -818,6 +833,9 @@ static void emit_store_global(struct lp_build_nir_context *bld_base,
                                                       loop_state.counter, "");
       addr_ptr = global_addr_to_ptr(gallivm, addr_ptr, bit_size);
       switch (bit_size) {
+      case 16:
+         value_ptr = LLVMBuildBitCast(builder, value_ptr, LLVMInt16TypeInContext(gallivm->context), "");
+         break;
       case 32:
          value_ptr = LLVMBuildBitCast(builder, value_ptr, LLVMInt32TypeInContext(gallivm->context), "");
          break;
@@ -943,18 +961,17 @@ static void emit_load_ubo(struct lp_build_nir_context *bld_base,
    struct gallivm_state *gallivm = bld_base->base.gallivm;
    LLVMBuilderRef builder = gallivm->builder;
    struct lp_build_context *uint_bld = &bld_base->uint_bld;
-   struct lp_build_context *bld_broad = bit_size == 64 ? &bld_base->dbl_bld : &bld_base->base;
+   struct lp_build_context *bld_broad = bit_size == 16 ? &bld_base->half_bld : (bit_size == 64 ? &bld_base->dbl_bld : &bld_base->base);
    LLVMValueRef consts_ptr = lp_build_array_get(gallivm, bld->consts_ptr, index);
-   unsigned size_shift = 0;
-   if (bit_size == 32)
-      size_shift = 2;
-   else if (bit_size == 64)
-      size_shift = 3;
+   unsigned size_shift = bit_size_to_shift(bit_size);
    if (size_shift)
       offset = lp_build_shr(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, size_shift));
    if (bit_size == 64) {
       LLVMTypeRef dptr_type = LLVMPointerType(bld_base->dbl_bld.elem_type, 0);
       consts_ptr = LLVMBuildBitCast(builder, consts_ptr, dptr_type, "");
+   } else if (bit_size == 16) {
+      LLVMTypeRef hptr_type = LLVMPointerType(bld_base->half_bld.elem_type, 0);
+      consts_ptr = LLVMBuildBitCast(builder, consts_ptr, hptr_type, "");
    }
 
    if (offset_is_uniform) {
@@ -993,19 +1010,25 @@ static void emit_load_mem(struct lp_build_nir_context *bld_base,
    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
    LLVMValueRef ssbo_ptr = NULL;
    struct lp_build_context *uint_bld = &bld_base->uint_bld;
-   struct lp_build_context *uint64_bld = &bld_base->uint64_bld;
    LLVMValueRef ssbo_limit = NULL;
-
+   unsigned size_shift = bit_size_to_shift(bit_size);
    if (index) {
       LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
-      ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, bit_size == 64 ? 3 : 2), "");
+      ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, size_shift), "");
       ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
 
       ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
    } else
       ssbo_ptr = bld->shared_ptr;
 
-   offset = LLVMBuildAShr(gallivm->builder, offset, lp_build_const_int_vec(gallivm, uint_bld->type, bit_size == 64 ? 3 : 2), "");
+   struct lp_build_context *load_bld = &bld_base->uint_bld;
+   if (bit_size == 64)
+      load_bld = &bld_base->uint64_bld;
+   else if (bit_size == 16)
+      load_bld = &bld_base->uint16_bld;
+
+   ssbo_ptr = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(load_bld->elem_type, 0), "");
+   offset = LLVMBuildAShr(gallivm->builder, offset, lp_build_const_int_vec(gallivm, uint_bld->type, size_shift), "");
    for (unsigned c = 0; c < nc; c++) {
       LLVMValueRef loop_index = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
       LLVMValueRef exec_mask = mask_vec(bld_base);
@@ -1015,7 +1038,8 @@ static void emit_load_mem(struct lp_build_nir_context *bld_base,
          exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
       }
 
-      LLVMValueRef result = lp_build_alloca(gallivm, bit_size == 64 ? uint64_bld->vec_type : uint_bld->vec_type, "");
+
+      LLVMValueRef result = lp_build_alloca(gallivm, load_bld->vec_type, "");
       struct lp_build_loop_state loop_state;
       lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
 
@@ -1030,11 +1054,7 @@ static void emit_load_mem(struct lp_build_nir_context *bld_base,
 
       lp_build_if(&ifthen, gallivm, cond);
       LLVMValueRef scalar;
-      if (bit_size == 64) {
-         LLVMValueRef ssbo_ptr2 = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(uint64_bld->elem_type, 0), "");
-         scalar = lp_build_pointer_get(builder, ssbo_ptr2, loop_index);
-      } else
-         scalar = lp_build_pointer_get(builder, ssbo_ptr, loop_index);
+      scalar = lp_build_pointer_get(builder, ssbo_ptr, loop_index);
 
       temp_res = LLVMBuildLoad(builder, result, "");
       temp_res = LLVMBuildInsertElement(builder, temp_res, scalar, loop_state.counter, "");
@@ -1044,6 +1064,8 @@ static void emit_load_mem(struct lp_build_nir_context *bld_base,
       LLVMValueRef zero;
       if (bit_size == 64)
          zero = LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 0, 0);
+      else if (bit_size == 16)
+         zero = LLVMConstInt(LLVMInt16TypeInContext(gallivm->context), 0, 0);
       else
          zero = lp_build_const_int32(gallivm, 0);
       temp_res = LLVMBuildInsertElement(builder, temp_res, zero, loop_state.counter, "");
@@ -1069,16 +1091,23 @@ static void emit_store_mem(struct lp_build_nir_context *bld_base,
    LLVMValueRef ssbo_ptr;
    struct lp_build_context *uint_bld = &bld_base->uint_bld;
    LLVMValueRef ssbo_limit = NULL;
+   unsigned size_shift = bit_size_to_shift(bit_size);
 
    if (index) {
       LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
-      ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, bit_size == 64 ? 3 : 2), "");
+      ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, size_shift), "");
       ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
       ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
    } else
       ssbo_ptr = bld->shared_ptr;
 
-   offset = lp_build_shr_imm(uint_bld, offset, bit_size == 64 ? 3 : 2);
+   offset = lp_build_shr_imm(uint_bld, offset, size_shift);
+
+   if (bit_size == 64) {
+      ssbo_ptr = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(bld_base->uint64_bld.elem_type, 0), "");
+   } else if (bit_size == 16) {
+      ssbo_ptr = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(bld_base->uint16_bld.elem_type, 0), "");
+   }
    for (unsigned c = 0; c < nc; c++) {
       if (!(writemask & (1u << c)))
          continue;
@@ -1097,6 +1126,8 @@ static void emit_store_mem(struct lp_build_nir_context *bld_base,
                                                        loop_state.counter, "");
       if (bit_size == 64)
          value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, bld_base->uint64_bld.elem_type, "");
+      else if (bit_size == 16)
+         value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, bld_base->uint16_bld.elem_type, "");
       else
          value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, uint_bld->elem_type, "");
       struct lp_build_if_state ifthen;
@@ -1107,11 +1138,7 @@ static void emit_store_mem(struct lp_build_nir_context *bld_base,
       cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
       cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
       lp_build_if(&ifthen, gallivm, cond);
-      if (bit_size == 64) {
-         LLVMValueRef ssbo_ptr2 = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(bld_base->uint64_bld.elem_type, 0), "");
-         lp_build_pointer_set(builder, ssbo_ptr2, loop_index, value_ptr);
-      } else
-         lp_build_pointer_set(builder, ssbo_ptr, loop_index, value_ptr);
+      lp_build_pointer_set(builder, ssbo_ptr, loop_index, value_ptr);
       lp_build_endif(&ifthen);
       lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
                              NULL, LLVMIntUGE);
@@ -1830,6 +1857,12 @@ void lp_build_nir_soa(struct gallivm_state *gallivm,
       lp_build_context_init(&bld.bld_base.dbl_bld, gallivm, dbl_type);
    }
    {
+      struct lp_type half_type;
+      half_type = type;
+      half_type.width /= 2;
+      lp_build_context_init(&bld.bld_base.half_bld, gallivm, half_type);
+   }
+   {
       struct lp_type uint64_type;
       uint64_type = lp_uint_type(type);
       uint64_type.width *= 2;