7e6a73959a
Increase fp16 support to allow clspv to continue to be OpenCL compliant following the update of the OpenCL-CTS adding more testing on math functions and conversions with half. Math functions are implemented by upscaling to fp32 and using the fp32 implementation. It garantees the accuracy required for half-precision float-point by the CTS.
283 lines
8.2 KiB
Common Lisp
283 lines
8.2 KiB
Common Lisp
/*
|
|
* Copyright (c) 2014 Advanced Micro Devices, Inc.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
* THE SOFTWARE.
|
|
*/
|
|
|
|
// This version is derived from the generic fma software implementation
|
|
// (__clc_sw_fma), but avoids the use of ulong in favor of uint2. The logic has
|
|
// been updated as appropriate.
|
|
|
|
#include <clc/clc.h>
|
|
#include "../../../generic/lib/clcmacro.h"
|
|
#include "../../../generic/lib/math/math.h"
|
|
|
|
struct fp {
|
|
uint2 mantissa;
|
|
int exponent;
|
|
uint sign;
|
|
};
|
|
|
|
static uint2 u2_set(uint hi, uint lo) {
|
|
uint2 res;
|
|
res.lo = lo;
|
|
res.hi = hi;
|
|
return res;
|
|
}
|
|
|
|
static uint2 u2_set_u(uint val) { return u2_set(0, val); }
|
|
|
|
static uint2 u2_mul(uint a, uint b) {
|
|
uint2 res;
|
|
res.hi = mul_hi(a, b);
|
|
res.lo = a * b;
|
|
return res;
|
|
}
|
|
|
|
static uint2 u2_sll(uint2 val, uint shift) {
|
|
if (shift == 0)
|
|
return val;
|
|
if (shift < 32) {
|
|
val.hi <<= shift;
|
|
val.hi |= val.lo >> (32 - shift);
|
|
val.lo <<= shift;
|
|
} else {
|
|
val.hi = val.lo << (shift - 32);
|
|
val.lo = 0;
|
|
}
|
|
return val;
|
|
}
|
|
|
|
static uint2 u2_srl(uint2 val, uint shift) {
|
|
if (shift == 0)
|
|
return val;
|
|
if (shift < 32) {
|
|
val.lo >>= shift;
|
|
val.lo |= val.hi << (32 - shift);
|
|
val.hi >>= shift;
|
|
} else {
|
|
val.lo = val.hi >> (shift - 32);
|
|
val.hi = 0;
|
|
}
|
|
return val;
|
|
}
|
|
|
|
static uint2 u2_or(uint2 a, uint b) {
|
|
a.lo |= b;
|
|
return a;
|
|
}
|
|
|
|
static uint2 u2_and(uint2 a, uint2 b) {
|
|
a.lo &= b.lo;
|
|
a.hi &= b.hi;
|
|
return a;
|
|
}
|
|
|
|
static uint2 u2_add(uint2 a, uint2 b) {
|
|
uint carry = (hadd(a.lo, b.lo) >> 31) & 0x1;
|
|
a.lo += b.lo;
|
|
a.hi += b.hi + carry;
|
|
return a;
|
|
}
|
|
|
|
static uint2 u2_add_u(uint2 a, uint b) { return u2_add(a, u2_set_u(b)); }
|
|
|
|
static uint2 u2_inv(uint2 a) {
|
|
a.lo = ~a.lo;
|
|
a.hi = ~a.hi;
|
|
return u2_add_u(a, 1);
|
|
}
|
|
|
|
static uint u2_clz(uint2 a) {
|
|
uint leading_zeroes = clz(a.hi);
|
|
if (leading_zeroes == 32) {
|
|
leading_zeroes += clz(a.lo);
|
|
}
|
|
return leading_zeroes;
|
|
}
|
|
|
|
static bool u2_eq(uint2 a, uint2 b) { return a.lo == b.lo && a.hi == b.hi; }
|
|
|
|
static bool u2_zero(uint2 a) { return u2_eq(a, u2_set_u(0)); }
|
|
|
|
static bool u2_gt(uint2 a, uint2 b) {
|
|
return a.hi > b.hi || (a.hi == b.hi && a.lo > b.lo);
|
|
}
|
|
|
|
_CLC_DEF _CLC_OVERLOAD float fma(float a, float b, float c) {
|
|
/* special cases */
|
|
if (isnan(a) || isnan(b) || isnan(c) || isinf(a) || isinf(b)) {
|
|
return mad(a, b, c);
|
|
}
|
|
|
|
/* If only c is inf, and both a,b are regular numbers, the result is c*/
|
|
if (isinf(c)) {
|
|
return c;
|
|
}
|
|
|
|
a = __clc_flush_denormal_if_not_supported(a);
|
|
b = __clc_flush_denormal_if_not_supported(b);
|
|
c = __clc_flush_denormal_if_not_supported(c);
|
|
|
|
if (a == 0.0f || b == 0.0f) {
|
|
return c;
|
|
}
|
|
|
|
if (c == 0) {
|
|
return a * b;
|
|
}
|
|
|
|
struct fp st_a, st_b, st_c;
|
|
|
|
st_a.exponent = a == .0f ? 0 : ((as_uint(a) & 0x7f800000) >> 23) - 127;
|
|
st_b.exponent = b == .0f ? 0 : ((as_uint(b) & 0x7f800000) >> 23) - 127;
|
|
st_c.exponent = c == .0f ? 0 : ((as_uint(c) & 0x7f800000) >> 23) - 127;
|
|
|
|
st_a.mantissa = u2_set_u(a == .0f ? 0 : (as_uint(a) & 0x7fffff) | 0x800000);
|
|
st_b.mantissa = u2_set_u(b == .0f ? 0 : (as_uint(b) & 0x7fffff) | 0x800000);
|
|
st_c.mantissa = u2_set_u(c == .0f ? 0 : (as_uint(c) & 0x7fffff) | 0x800000);
|
|
|
|
st_a.sign = as_uint(a) & 0x80000000;
|
|
st_b.sign = as_uint(b) & 0x80000000;
|
|
st_c.sign = as_uint(c) & 0x80000000;
|
|
|
|
// Multiplication.
|
|
// Move the product to the highest bits to maximize precision
|
|
// mantissa is 24 bits => product is 48 bits, 2bits non-fraction.
|
|
// Add one bit for future addition overflow,
|
|
// add another bit to detect subtraction underflow
|
|
struct fp st_mul;
|
|
st_mul.sign = st_a.sign ^ st_b.sign;
|
|
st_mul.mantissa = u2_sll(u2_mul(st_a.mantissa.lo, st_b.mantissa.lo), 14);
|
|
st_mul.exponent =
|
|
!u2_zero(st_mul.mantissa) ? st_a.exponent + st_b.exponent : 0;
|
|
|
|
// FIXME: Detecting a == 0 || b == 0 above crashed GCN isel
|
|
if (st_mul.exponent == 0 && u2_zero(st_mul.mantissa))
|
|
return c;
|
|
|
|
// Mantissa is 23 fractional bits, shift it the same way as product mantissa
|
|
#define C_ADJUST 37ul
|
|
|
|
// both exponents are bias adjusted
|
|
int exp_diff = st_mul.exponent - st_c.exponent;
|
|
|
|
st_c.mantissa = u2_sll(st_c.mantissa, C_ADJUST);
|
|
uint2 cutoff_bits = u2_set_u(0);
|
|
uint2 cutoff_mask = u2_add(u2_sll(u2_set_u(1), abs(exp_diff)),
|
|
u2_set(0xffffffff, 0xffffffff));
|
|
if (exp_diff > 0) {
|
|
cutoff_bits =
|
|
exp_diff >= 64 ? st_c.mantissa : u2_and(st_c.mantissa, cutoff_mask);
|
|
st_c.mantissa =
|
|
exp_diff >= 64 ? u2_set_u(0) : u2_srl(st_c.mantissa, exp_diff);
|
|
} else {
|
|
cutoff_bits = -exp_diff >= 64 ? st_mul.mantissa
|
|
: u2_and(st_mul.mantissa, cutoff_mask);
|
|
st_mul.mantissa =
|
|
-exp_diff >= 64 ? u2_set_u(0) : u2_srl(st_mul.mantissa, -exp_diff);
|
|
}
|
|
|
|
struct fp st_fma;
|
|
st_fma.sign = st_mul.sign;
|
|
st_fma.exponent = max(st_mul.exponent, st_c.exponent);
|
|
if (st_c.sign == st_mul.sign) {
|
|
st_fma.mantissa = u2_add(st_mul.mantissa, st_c.mantissa);
|
|
} else {
|
|
// cutoff bits borrow one
|
|
st_fma.mantissa =
|
|
u2_add(u2_add(st_mul.mantissa, u2_inv(st_c.mantissa)),
|
|
(!u2_zero(cutoff_bits) && (st_mul.exponent > st_c.exponent)
|
|
? u2_set(0xffffffff, 0xffffffff)
|
|
: u2_set_u(0)));
|
|
}
|
|
|
|
// underflow: st_c.sign != st_mul.sign, and magnitude switches the sign
|
|
if (u2_gt(st_fma.mantissa, u2_set(0x7fffffff, 0xffffffff))) {
|
|
st_fma.mantissa = u2_inv(st_fma.mantissa);
|
|
st_fma.sign = st_mul.sign ^ 0x80000000;
|
|
}
|
|
|
|
// detect overflow/underflow
|
|
int overflow_bits = 3 - u2_clz(st_fma.mantissa);
|
|
|
|
// adjust exponent
|
|
st_fma.exponent += overflow_bits;
|
|
|
|
// handle underflow
|
|
if (overflow_bits < 0) {
|
|
st_fma.mantissa = u2_sll(st_fma.mantissa, -overflow_bits);
|
|
overflow_bits = 0;
|
|
}
|
|
|
|
// rounding
|
|
uint2 trunc_mask = u2_add(u2_sll(u2_set_u(1), C_ADJUST + overflow_bits),
|
|
u2_set(0xffffffff, 0xffffffff));
|
|
uint2 trunc_bits =
|
|
u2_or(u2_and(st_fma.mantissa, trunc_mask), !u2_zero(cutoff_bits));
|
|
uint2 last_bit =
|
|
u2_and(st_fma.mantissa, u2_sll(u2_set_u(1), C_ADJUST + overflow_bits));
|
|
uint2 grs_bits = u2_sll(u2_set_u(4), C_ADJUST - 3 + overflow_bits);
|
|
|
|
// round to nearest even
|
|
if (u2_gt(trunc_bits, grs_bits) ||
|
|
(u2_eq(trunc_bits, grs_bits) && !u2_zero(last_bit))) {
|
|
st_fma.mantissa =
|
|
u2_add(st_fma.mantissa, u2_sll(u2_set_u(1), C_ADJUST + overflow_bits));
|
|
}
|
|
|
|
// Shift mantissa back to bit 23
|
|
st_fma.mantissa = u2_srl(st_fma.mantissa, C_ADJUST + overflow_bits);
|
|
|
|
// Detect rounding overflow
|
|
if (u2_gt(st_fma.mantissa, u2_set_u(0xffffff))) {
|
|
++st_fma.exponent;
|
|
st_fma.mantissa = u2_srl(st_fma.mantissa, 1);
|
|
}
|
|
|
|
if (u2_zero(st_fma.mantissa)) {
|
|
return 0.0f;
|
|
}
|
|
|
|
// Flating point range limit
|
|
if (st_fma.exponent > 127) {
|
|
return as_float(as_uint(INFINITY) | st_fma.sign);
|
|
}
|
|
|
|
// Flush denormals
|
|
if (st_fma.exponent <= -127) {
|
|
return as_float(st_fma.sign);
|
|
}
|
|
|
|
return as_float(st_fma.sign | ((st_fma.exponent + 127) << 23) |
|
|
((uint)st_fma.mantissa.lo & 0x7fffff));
|
|
}
|
|
_CLC_TERNARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, fma, float, float, float)
|
|
|
|
#ifdef cl_khr_fp16
|
|
|
|
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
|
|
|
_CLC_DEF _CLC_OVERLOAD half fma(half a, half b, half c) {
|
|
return (half)mad((float)a, (float)b, (float)c);
|
|
}
|
|
_CLC_TERNARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, half, fma, half, half, half)
|
|
|
|
#endif
|