diff options
| author | Igor Ryzhov <iryzhov@nfware.com> | 2024-01-20 03:43:18 +0200 |
|---|---|---|
| committer | Igor Ryzhov <iryzhov@nfware.com> | 2024-01-28 23:28:39 +0200 |
| commit | 20fe60f40d80634dd22dd2b85b4b14c86b7dc927 (patch) | |
| tree | 8224950788e5e9803c19b26b4aeb62c415d364ba /lib/yang_wrappers.c | |
| parent | 0a5b944aefdcc7da75bf383ea3f82bed562f1c12 (diff) | |
lib: fix conversion of yang decimal64 to double
Current denominators are not integers and some of them lose precision
because of that, for example, 1e-6 is actually stored as
9.9999999999999995e-07 and 1-e12 is stored as 9.9999999999999998e-13.
When multiplying by such denominators, we receive incorrect values.
Changing denominators to integers and using division instead of
multiplication improves precision and solves the problem.
Signed-off-by: Igor Ryzhov <iryzhov@nfware.com>
Diffstat (limited to 'lib/yang_wrappers.c')
| -rw-r--r-- | lib/yang_wrappers.c | 9 |
1 files changed, 4 insertions, 5 deletions
diff --git a/lib/yang_wrappers.c b/lib/yang_wrappers.c index 53f40e89b7..237913880a 100644 --- a/lib/yang_wrappers.c +++ b/lib/yang_wrappers.c @@ -168,10 +168,9 @@ struct yang_data *yang_data_new_dec64(const char *xpath, double value) double yang_dnode_get_dec64(const struct lyd_node *dnode, const char *xpath_fmt, ...) { - const double denom[19] = {1e0, 1e-1, 1e-2, 1e-3, 1e-4, - 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, - 1e-10, 1e-11, 1e-12, 1e-13, 1e-14, - 1e-15, 1e-16, 1e-17, 1e-18}; + const double denom[19] = { 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, + 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, + 1e14, 1e15, 1e16, 1e17, 1e18 }; const struct lysc_type_dec *dectype; const struct lyd_value *dvalue; @@ -179,7 +178,7 @@ double yang_dnode_get_dec64(const struct lyd_node *dnode, const char *xpath_fmt, dectype = (const struct lysc_type_dec *)dvalue->realtype; assert(dectype->basetype == LY_TYPE_DEC64); assert(dectype->fraction_digits < sizeof(denom) / sizeof(*denom)); - return (double)dvalue->dec64 * denom[dectype->fraction_digits]; + return (double)dvalue->dec64 / denom[dectype->fraction_digits]; } double yang_get_default_dec64(const char *xpath_fmt, ...) |