|
|
|
@@ -134,7 +134,12 @@ def type_base_type(type_):
|
|
|
|
|
assert m is not None, 'Invalid NIR type string: "{}"'.format(type_)
|
|
|
|
|
return m.group('type')
|
|
|
|
|
|
|
|
|
|
commutative = "commutative "
|
|
|
|
|
# Operation where the first two sources are commutative.
|
|
|
|
|
#
|
|
|
|
|
# For 2-source operations, this just mathematical commutativity. Some
|
|
|
|
|
# 3-source operations, like ffma, are only commutative in the first two
|
|
|
|
|
# sources.
|
|
|
|
|
_2src_commutative = "2src_commutative "
|
|
|
|
|
associative = "associative "
|
|
|
|
|
|
|
|
|
|
# global dictionary of opcodes
|
|
|
|
@@ -471,23 +476,23 @@ def binop_reduce(name, output_size, output_type, src_type, prereduce_expr,
|
|
|
|
|
src2 = prereduce("src0.z", "src1.z")
|
|
|
|
|
src3 = prereduce("src0.w", "src1.w")
|
|
|
|
|
opcode(name + "2", output_size, output_type,
|
|
|
|
|
[2, 2], [src_type, src_type], False, commutative,
|
|
|
|
|
[2, 2], [src_type, src_type], False, _2src_commutative,
|
|
|
|
|
final(reduce_(src0, src1)))
|
|
|
|
|
opcode(name + "3", output_size, output_type,
|
|
|
|
|
[3, 3], [src_type, src_type], False, commutative,
|
|
|
|
|
[3, 3], [src_type, src_type], False, _2src_commutative,
|
|
|
|
|
final(reduce_(reduce_(src0, src1), src2)))
|
|
|
|
|
opcode(name + "4", output_size, output_type,
|
|
|
|
|
[4, 4], [src_type, src_type], False, commutative,
|
|
|
|
|
[4, 4], [src_type, src_type], False, _2src_commutative,
|
|
|
|
|
final(reduce_(reduce_(src0, src1), reduce_(src2, src3))))
|
|
|
|
|
|
|
|
|
|
binop("fadd", tfloat, commutative + associative, "src0 + src1")
|
|
|
|
|
binop("iadd", tint, commutative + associative, "src0 + src1")
|
|
|
|
|
binop("iadd_sat", tint, commutative, """
|
|
|
|
|
binop("fadd", tfloat, _2src_commutative + associative, "src0 + src1")
|
|
|
|
|
binop("iadd", tint, _2src_commutative + associative, "src0 + src1")
|
|
|
|
|
binop("iadd_sat", tint, _2src_commutative, """
|
|
|
|
|
src1 > 0 ?
|
|
|
|
|
(src0 + src1 < src0 ? (1ull << (bit_size - 1)) - 1 : src0 + src1) :
|
|
|
|
|
(src0 < src0 + src1 ? (1ull << (bit_size - 1)) : src0 + src1)
|
|
|
|
|
""")
|
|
|
|
|
binop("uadd_sat", tuint, commutative,
|
|
|
|
|
binop("uadd_sat", tuint, _2src_commutative,
|
|
|
|
|
"(src0 + src1) < src0 ? MAX_UINT_FOR_SIZE(sizeof(src0) * 8) : (src0 + src1)")
|
|
|
|
|
binop("isub_sat", tint, "", """
|
|
|
|
|
src1 < 0 ?
|
|
|
|
@@ -499,18 +504,18 @@ binop("usub_sat", tuint, "", "src0 < src1 ? 0 : src0 - src1")
|
|
|
|
|
binop("fsub", tfloat, "", "src0 - src1")
|
|
|
|
|
binop("isub", tint, "", "src0 - src1")
|
|
|
|
|
|
|
|
|
|
binop("fmul", tfloat, commutative + associative, "src0 * src1")
|
|
|
|
|
binop("fmul", tfloat, _2src_commutative + associative, "src0 * src1")
|
|
|
|
|
# low 32-bits of signed/unsigned integer multiply
|
|
|
|
|
binop("imul", tint, commutative + associative, "src0 * src1")
|
|
|
|
|
binop("imul", tint, _2src_commutative + associative, "src0 * src1")
|
|
|
|
|
|
|
|
|
|
# Generate 64 bit result from 2 32 bits quantity
|
|
|
|
|
binop_convert("imul_2x32_64", tint64, tint32, commutative,
|
|
|
|
|
binop_convert("imul_2x32_64", tint64, tint32, _2src_commutative,
|
|
|
|
|
"(int64_t)src0 * (int64_t)src1")
|
|
|
|
|
binop_convert("umul_2x32_64", tuint64, tuint32, commutative,
|
|
|
|
|
binop_convert("umul_2x32_64", tuint64, tuint32, _2src_commutative,
|
|
|
|
|
"(uint64_t)src0 * (uint64_t)src1")
|
|
|
|
|
|
|
|
|
|
# high 32-bits of signed integer multiply
|
|
|
|
|
binop("imul_high", tint, commutative, """
|
|
|
|
|
binop("imul_high", tint, _2src_commutative, """
|
|
|
|
|
if (bit_size == 64) {
|
|
|
|
|
/* We need to do a full 128-bit x 128-bit multiply in order for the sign
|
|
|
|
|
* extension to work properly. The casts are kind-of annoying but needed
|
|
|
|
@@ -537,7 +542,7 @@ if (bit_size == 64) {
|
|
|
|
|
""")
|
|
|
|
|
|
|
|
|
|
# high 32-bits of unsigned integer multiply
|
|
|
|
|
binop("umul_high", tuint, commutative, """
|
|
|
|
|
binop("umul_high", tuint, _2src_commutative, """
|
|
|
|
|
if (bit_size == 64) {
|
|
|
|
|
/* The casts are kind-of annoying but needed to prevent compiler warnings. */
|
|
|
|
|
uint32_t src0_u32[2] = { src0, (uint64_t)src0 >> 32 };
|
|
|
|
@@ -557,7 +562,7 @@ binop("udiv", tuint, "", "src1 == 0 ? 0 : (src0 / src1)")
|
|
|
|
|
# returns a boolean representing the carry resulting from the addition of
|
|
|
|
|
# the two unsigned arguments.
|
|
|
|
|
|
|
|
|
|
binop_convert("uadd_carry", tuint, tuint, commutative, "src0 + src1 < src0")
|
|
|
|
|
binop_convert("uadd_carry", tuint, tuint, _2src_commutative, "src0 + src1 < src0")
|
|
|
|
|
|
|
|
|
|
# returns a boolean representing the borrow resulting from the subtraction
|
|
|
|
|
# of the two unsigned arguments.
|
|
|
|
@@ -574,8 +579,8 @@ binop_convert("usub_borrow", tuint, tuint, "", "src0 < src1")
|
|
|
|
|
#
|
|
|
|
|
# (x + y) >> 1 = (((x & y) << 1) + (x ^ y)) >> 1
|
|
|
|
|
# = (x & y) + ((x ^ y) >> 1)
|
|
|
|
|
binop("ihadd", tint, commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)")
|
|
|
|
|
binop("uhadd", tuint, commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)")
|
|
|
|
|
binop("ihadd", tint, _2src_commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)")
|
|
|
|
|
binop("uhadd", tuint, _2src_commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)")
|
|
|
|
|
|
|
|
|
|
# rhadd: (a + b + 1) >> 1 (without overflow)
|
|
|
|
|
# x + y + 1 = x + (~x & y) - (~x & y) + y + (x & ~y) - (x & ~y) + 1
|
|
|
|
@@ -587,8 +592,8 @@ binop("uhadd", tuint, commutative, "(src0 & src1) + ((src0 ^ src1) >> 1)")
|
|
|
|
|
#
|
|
|
|
|
# (x + y + 1) >> 1 = (x | y) + (-(x ^ y) + 1) >> 1)
|
|
|
|
|
# = (x | y) - ((x ^ y) >> 1)
|
|
|
|
|
binop("irhadd", tint, commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)")
|
|
|
|
|
binop("urhadd", tuint, commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)")
|
|
|
|
|
binop("irhadd", tint, _2src_commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)")
|
|
|
|
|
binop("urhadd", tuint, _2src_commutative, "(src0 | src1) + ((src0 ^ src1) >> 1)")
|
|
|
|
|
|
|
|
|
|
binop("umod", tuint, "", "src1 == 0 ? 0 : src0 % src1")
|
|
|
|
|
|
|
|
|
@@ -616,22 +621,22 @@ binop("frem", tfloat, "", "src0 - src1 * truncf(src0 / src1)")
|
|
|
|
|
|
|
|
|
|
binop_compare("flt", tfloat, "", "src0 < src1")
|
|
|
|
|
binop_compare("fge", tfloat, "", "src0 >= src1")
|
|
|
|
|
binop_compare("feq", tfloat, commutative, "src0 == src1")
|
|
|
|
|
binop_compare("fne", tfloat, commutative, "src0 != src1")
|
|
|
|
|
binop_compare("feq", tfloat, _2src_commutative, "src0 == src1")
|
|
|
|
|
binop_compare("fne", tfloat, _2src_commutative, "src0 != src1")
|
|
|
|
|
binop_compare("ilt", tint, "", "src0 < src1")
|
|
|
|
|
binop_compare("ige", tint, "", "src0 >= src1")
|
|
|
|
|
binop_compare("ieq", tint, commutative, "src0 == src1")
|
|
|
|
|
binop_compare("ine", tint, commutative, "src0 != src1")
|
|
|
|
|
binop_compare("ieq", tint, _2src_commutative, "src0 == src1")
|
|
|
|
|
binop_compare("ine", tint, _2src_commutative, "src0 != src1")
|
|
|
|
|
binop_compare("ult", tuint, "", "src0 < src1")
|
|
|
|
|
binop_compare("uge", tuint, "", "src0 >= src1")
|
|
|
|
|
binop_compare32("flt32", tfloat, "", "src0 < src1")
|
|
|
|
|
binop_compare32("fge32", tfloat, "", "src0 >= src1")
|
|
|
|
|
binop_compare32("feq32", tfloat, commutative, "src0 == src1")
|
|
|
|
|
binop_compare32("fne32", tfloat, commutative, "src0 != src1")
|
|
|
|
|
binop_compare32("feq32", tfloat, _2src_commutative, "src0 == src1")
|
|
|
|
|
binop_compare32("fne32", tfloat, _2src_commutative, "src0 != src1")
|
|
|
|
|
binop_compare32("ilt32", tint, "", "src0 < src1")
|
|
|
|
|
binop_compare32("ige32", tint, "", "src0 >= src1")
|
|
|
|
|
binop_compare32("ieq32", tint, commutative, "src0 == src1")
|
|
|
|
|
binop_compare32("ine32", tint, commutative, "src0 != src1")
|
|
|
|
|
binop_compare32("ieq32", tint, _2src_commutative, "src0 == src1")
|
|
|
|
|
binop_compare32("ine32", tint, _2src_commutative, "src0 != src1")
|
|
|
|
|
binop_compare32("ult32", tuint, "", "src0 < src1")
|
|
|
|
|
binop_compare32("uge32", tuint, "", "src0 >= src1")
|
|
|
|
|
|
|
|
|
@@ -667,8 +672,8 @@ binop_reduce("fany_nequal", 1, tfloat32, tfloat32, "{src0} != {src1}",
|
|
|
|
|
|
|
|
|
|
binop("slt", tfloat32, "", "(src0 < src1) ? 1.0f : 0.0f") # Set on Less Than
|
|
|
|
|
binop("sge", tfloat, "", "(src0 >= src1) ? 1.0f : 0.0f") # Set on Greater or Equal
|
|
|
|
|
binop("seq", tfloat32, commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal
|
|
|
|
|
binop("sne", tfloat32, commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal
|
|
|
|
|
binop("seq", tfloat32, _2src_commutative, "(src0 == src1) ? 1.0f : 0.0f") # Set on Equal
|
|
|
|
|
binop("sne", tfloat32, _2src_commutative, "(src0 != src1) ? 1.0f : 0.0f") # Set on Not Equal
|
|
|
|
|
|
|
|
|
|
# SPIRV shifts are undefined for shift-operands >= bitsize,
|
|
|
|
|
# but SM5 shifts are defined to use the least significant bits, only
|
|
|
|
@@ -686,9 +691,9 @@ opcode("ushr", 0, tuint, [0, 0], [tuint, tuint32], False, "",
|
|
|
|
|
# integers.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
binop("iand", tuint, commutative + associative, "src0 & src1")
|
|
|
|
|
binop("ior", tuint, commutative + associative, "src0 | src1")
|
|
|
|
|
binop("ixor", tuint, commutative + associative, "src0 ^ src1")
|
|
|
|
|
binop("iand", tuint, _2src_commutative + associative, "src0 & src1")
|
|
|
|
|
binop("ior", tuint, _2src_commutative + associative, "src0 | src1")
|
|
|
|
|
binop("ixor", tuint, _2src_commutative + associative, "src0 ^ src1")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# floating point logic operators
|
|
|
|
@@ -696,11 +701,11 @@ binop("ixor", tuint, commutative + associative, "src0 ^ src1")
|
|
|
|
|
# These use (src != 0.0) for testing the truth of the input, and output 1.0
|
|
|
|
|
# for true and 0.0 for false
|
|
|
|
|
|
|
|
|
|
binop("fand", tfloat32, commutative,
|
|
|
|
|
binop("fand", tfloat32, _2src_commutative,
|
|
|
|
|
"((src0 != 0.0f) && (src1 != 0.0f)) ? 1.0f : 0.0f")
|
|
|
|
|
binop("for", tfloat32, commutative,
|
|
|
|
|
binop("for", tfloat32, _2src_commutative,
|
|
|
|
|
"((src0 != 0.0f) || (src1 != 0.0f)) ? 1.0f : 0.0f")
|
|
|
|
|
binop("fxor", tfloat32, commutative,
|
|
|
|
|
binop("fxor", tfloat32, _2src_commutative,
|
|
|
|
|
"(src0 != 0.0f && src1 == 0.0f) || (src0 == 0.0f && src1 != 0.0f) ? 1.0f : 0.0f")
|
|
|
|
|
|
|
|
|
|
binop_reduce("fdot", 1, tfloat, tfloat, "{src0} * {src1}", "{src0} + {src1}",
|
|
|
|
@@ -715,14 +720,14 @@ opcode("fdph_replicated", 4, tfloat, [3, 4], [tfloat, tfloat], False, "",
|
|
|
|
|
"src0.x * src1.x + src0.y * src1.y + src0.z * src1.z + src1.w")
|
|
|
|
|
|
|
|
|
|
binop("fmin", tfloat, "", "fminf(src0, src1)")
|
|
|
|
|
binop("imin", tint, commutative + associative, "src1 > src0 ? src0 : src1")
|
|
|
|
|
binop("umin", tuint, commutative + associative, "src1 > src0 ? src0 : src1")
|
|
|
|
|
binop("imin", tint, _2src_commutative + associative, "src1 > src0 ? src0 : src1")
|
|
|
|
|
binop("umin", tuint, _2src_commutative + associative, "src1 > src0 ? src0 : src1")
|
|
|
|
|
binop("fmax", tfloat, "", "fmaxf(src0, src1)")
|
|
|
|
|
binop("imax", tint, commutative + associative, "src1 > src0 ? src1 : src0")
|
|
|
|
|
binop("umax", tuint, commutative + associative, "src1 > src0 ? src1 : src0")
|
|
|
|
|
binop("imax", tint, _2src_commutative + associative, "src1 > src0 ? src1 : src0")
|
|
|
|
|
binop("umax", tuint, _2src_commutative + associative, "src1 > src0 ? src1 : src0")
|
|
|
|
|
|
|
|
|
|
# Saturated vector add for 4 8bit ints.
|
|
|
|
|
binop("usadd_4x8", tint32, commutative + associative, """
|
|
|
|
|
binop("usadd_4x8", tint32, _2src_commutative + associative, """
|
|
|
|
|
dst = 0;
|
|
|
|
|
for (int i = 0; i < 32; i += 8) {
|
|
|
|
|
dst |= MIN2(((src0 >> i) & 0xff) + ((src1 >> i) & 0xff), 0xff) << i;
|
|
|
|
@@ -741,7 +746,7 @@ for (int i = 0; i < 32; i += 8) {
|
|
|
|
|
""")
|
|
|
|
|
|
|
|
|
|
# vector min for 4 8bit ints.
|
|
|
|
|
binop("umin_4x8", tint32, commutative + associative, """
|
|
|
|
|
binop("umin_4x8", tint32, _2src_commutative + associative, """
|
|
|
|
|
dst = 0;
|
|
|
|
|
for (int i = 0; i < 32; i += 8) {
|
|
|
|
|
dst |= MIN2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
|
|
|
|
@@ -749,7 +754,7 @@ for (int i = 0; i < 32; i += 8) {
|
|
|
|
|
""")
|
|
|
|
|
|
|
|
|
|
# vector max for 4 8bit ints.
|
|
|
|
|
binop("umax_4x8", tint32, commutative + associative, """
|
|
|
|
|
binop("umax_4x8", tint32, _2src_commutative + associative, """
|
|
|
|
|
dst = 0;
|
|
|
|
|
for (int i = 0; i < 32; i += 8) {
|
|
|
|
|
dst |= MAX2((src0 >> i) & 0xff, (src1 >> i) & 0xff) << i;
|
|
|
|
@@ -757,7 +762,7 @@ for (int i = 0; i < 32; i += 8) {
|
|
|
|
|
""")
|
|
|
|
|
|
|
|
|
|
# unorm multiply: (a * b) / 255.
|
|
|
|
|
binop("umul_unorm_4x8", tint32, commutative + associative, """
|
|
|
|
|
binop("umul_unorm_4x8", tint32, _2src_commutative + associative, """
|
|
|
|
|
dst = 0;
|
|
|
|
|
for (int i = 0; i < 32; i += 8) {
|
|
|
|
|
int src0_chan = (src0 >> i) & 0xff;
|
|
|
|
|