From 5382a22e0eea878339c504b2a9a3b36bcd839fcc Mon Sep 17 00:00:00 2001 From: Roberto Ierusalimschy Date: Tue, 28 Aug 2018 12:36:58 -0300 Subject: Corrections in the implementation of '%' for floats. The multiplication (m*b) used to test whether 'm' is non-zero and 'm' and 'b' have different signs can underflow for very small numbers, giving a wrong result. The use of explicit comparisons solves this problem. This commit also adds several new tests for '%' (both for floats and for integers) to exercise more corner cases, such as very large and very small values. --- llimits.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'llimits.h') diff --git a/llimits.h b/llimits.h index 6afa8997..e91310a0 100644 --- a/llimits.h +++ b/llimits.h @@ -293,15 +293,17 @@ typedef unsigned long Instruction; #endif /* -** modulo: defined as 'a - floor(a/b)*b'; this definition gives NaN when -** 'b' is huge, but the result should be 'a'. 'fmod' gives the result of -** 'a - trunc(a/b)*b', and therefore must be corrected when 'trunc(a/b) -** ~= floor(a/b)'. That happens when the division has a non-integer -** negative result, which is equivalent to the test below. +** modulo: defined as 'a - floor(a/b)*b'; the direct computation +** using this definition has several problems with rounding errors, +** so it is better to use 'fmod'. 'fmod' gives the result of +** 'a - trunc(a/b)*b', and therefore must be corrected when +** 'trunc(a/b) ~= floor(a/b)'. That happens when the division has a +** non-integer negative result, which is equivalent to the tests below. */ #if !defined(luai_nummod) #define luai_nummod(L,a,b,m) \ - { (m) = l_mathop(fmod)(a,b); if ((m)*(b) < 0) (m) += (b); } + { (void)L; (m) = l_mathop(fmod)(a,b); \ + if (((m) > 0) ? (b) < 0 : ((m) < 0 && (b) > 0)) (m) += (b); } #endif /* exponentiation */ -- cgit v1.2.3-55-g6feb