From f4f52b6653f2a5639a5178d29a9a7d5d978b2fd5 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Mon, 19 Dec 2022 10:51:25 +0000 Subject: [PATCH] fix: sumx => sumx2 --- --- a/tests/src/calibrate.ts +++ b/tests/src/calibrate.ts @@ -126,14 +126,9 @@ const nb = new Fract(BigInt(n)); - // This is a workaround to beat the lack of precision of the `Number` type. - // We divide `BigInt`s. But since it is an integer division, we should take care of the precision on our own. - // After the division we can convert the result back to the `Number` and then we set the correct precision. - // It is crucial to have the correct slope for the regression line. - const a = rpn( [nb, sumxy, '*', sumx, sumy, '*', '-'], - [nb, sumx, '*', sumx, sumx, '*', '-'], + [nb, sumx2, '*', sumx, sumx, '*', '-'], '/', ); const b = rpn( -- gitstuff