From 4b5bca9f7d3d6efb25fa001e456d8a3c829be020 Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Fri, 13 Nov 2026 19:59:43 +0000 Subject: [PATCH 1/2] [FIX] Leave zero sigmas out of the log-linear sigma fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit loglinear_interpolation replaced a zero sigma with 1e-320 and fitted its log (-736.8) at full weight, so a single zero-variation period could push the extrapolated sigmas to 1.8e308. Only positive sigmas now enter the fit, as in R's ChainLadder estimate.sigma, and zero sigmas are not filled. Where fewer than three positive sigmas remain, the fill falls back to Mack's approximation, which R's Mack.S.E also does for one point and for two points (undefined slope p-value). Closes #1609. Developed with an AI assistant (Árbol, running Muse). --- .../development/tests/test_development.py | 19 ++++++++++++++ chainladder/utils/weighted_regression.py | 25 +++++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index c6e8b78b9..605697ae7 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -1132,3 +1132,22 @@ def test_valid_params_pass_validation(raa): cl.Development(average=lambda age: "geometric", n_periods=[-1] * 9).fit(raa) dev = cl.Development(n_periods=4, average="simple") assert cl.read_json(dev.to_json()).get_params() == dev.get_params() + + +def test_loglinear_sigma_skips_zero_sigmas(clrd): + # GH 1609: zero sigmas used to enter the log-linear fit as log(1e-320). + tri = clrd["CumPaidLoss"] + + # One positive sigma: the fill falls back to Mack's approximation. + brotherhood = cl.Development().fit(tri.loc["Brotherhood Mut Ins Co", "prodliab"]) + assert np.nanmax(brotherhood.sigma_.set_backend("numpy").values) < 1e3 + assert np.isfinite(brotherhood.std_err_.set_backend("numpy").values).all() + + # Five positive sigmas (24-36 to 72-84) and two zeros (84-96, 96-108): + # the 12-24 and 108-120 fills come from the five positive sigmas only. + buckeye = cl.Development().fit(tri.loc["Buckeye Ins Grp", "othliab"]) + sigma = buckeye.sigma_.set_backend("numpy").values.ravel() + period = np.arange(1, sigma.size + 1) + slope, intercept = np.polyfit(period[1:6], np.log(sigma[1:6]), 1) + assert np.allclose(sigma[[0, 8]], np.exp(intercept + slope * period[[0, 8]])) + assert np.isclose(sigma[0], 7.1819, atol=1e-4) diff --git a/chainladder/utils/weighted_regression.py b/chainladder/utils/weighted_regression.py index 5e0a51a4b..6fd9959f5 100644 --- a/chainladder/utils/weighted_regression.py +++ b/chainladder/utils/weighted_regression.py @@ -262,17 +262,32 @@ def std_err_fill(self): return self def loglinear_interpolation(self, y): - """Use Cases: generally for filling in last element of sigma_""" + """ + Fill undefined sigmas by regressing log(sigma) on development period. + + As in R's ChainLadder (``estimate.sigma``), only positive sigmas enter + the regression: a zero sigma has no logarithm and would dominate the + fit. Zero sigmas are not filled. Where fewer than three positive sigmas + remain, the fill falls back to Mack's approximation. R's ``Mack.S.E`` + does the same for a single point, and for two points, where the + slope's p-value is undefined. + """ from chainladder.utils.utility_functions import num_to_nan xp = self.xp - ly = y.copy() - ly = xp.log(xp.where(ly == 0, 1e-320, ly)) + ly = xp.log(xp.where(y > 0, y, xp.nan)) w = xp.nan_to_num(ly * 0 + 1) + estimated = xp.nan_to_num(y * 0 + 1) reg = WeightedRegression(self.axis, False, xp=xp).fit(None, ly, w) slope, intercept = reg.slope_, reg.intercept_ - fill_ = xp.exp(reg.x * slope + intercept) * (1 - w) - out = xp.nan_to_num(y) + xp.nan_to_num(fill_) + fill_ = xp.nan_to_num(xp.exp(reg.x * slope + intercept) * (1 - estimated)) + use_mack = xp.sum(w, axis=self.axis, keepdims=True) < 3 + if use_mack.any(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + mack_fill = xp.nan_to_num(self.mack_interpolation(y)) * (1 - estimated) + fill_ = xp.where(use_mack, mack_fill, fill_) + out = xp.nan_to_num(y) + fill_ return num_to_nan(out) def mack_interpolation(self, y): From eb713311cad25108bd14702f142459b9aa27edb8 Mon Sep 17 00:00:00 2001 From: Simon Koch-Sultan Date: Mon, 16 Nov 2026 00:15:39 +0000 Subject: [PATCH 2/2] [FIX] Explain the sigma fill without R and test the Mack fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give the log-linear fill's reasons in the docstring: a zero sigma has no logarithm, and a line through two points leaves no residual to judge it by. Scope the fallback's error suppression to numpy's divide and invalid flags, which mack_interpolation sets where an earlier sigma is zero. Add a case where the fallback fills a sigma and compare it with sigma_interpolation="mack", and correct the comment on the case where Mack has nothing to fill from. Developed with an AI assistant (Árbol, running Muse). --- .../development/tests/test_development.py | 14 ++++++++++++- chainladder/utils/weighted_regression.py | 21 ++++++++++++------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index 605697ae7..fe9fddc04 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -1138,11 +1138,23 @@ def test_loglinear_sigma_skips_zero_sigmas(clrd): # GH 1609: zero sigmas used to enter the log-linear fit as log(1e-320). tri = clrd["CumPaidLoss"] - # One positive sigma: the fill falls back to Mack's approximation. + # One positive sigma (36-48) and a zero (48-60): no line is fitted, and + # Mack's approximation has no two positive sigmas in a row, so the + # 12-24 and 24-36 sigmas stay NaN instead of 1.8e308. brotherhood = cl.Development().fit(tri.loc["Brotherhood Mut Ins Co", "prodliab"]) assert np.nanmax(brotherhood.sigma_.set_backend("numpy").values) < 1e3 assert np.isfinite(brotherhood.std_err_.set_backend("numpy").values).all() + # Two positive sigmas (12-24, 24-36), then none at 36-48 and zeros after: + # the 36-48 fill comes from Mack's approximation, the same value as + # under sigma_interpolation="mack". + great_amer = tri.loc["Great Amer Grp", "medmal"] + loglinear = cl.Development().fit(great_amer).sigma_.set_backend("numpy") + mack = cl.Development(sigma_interpolation="mack").fit(great_amer) + fill = loglinear.values.ravel()[2] + assert np.isclose(fill, mack.sigma_.set_backend("numpy").values.ravel()[2]) + assert np.isclose(fill, 14.9781, atol=1e-4) + # Five positive sigmas (24-36 to 72-84) and two zeros (84-96, 96-108): # the 12-24 and 108-120 fills come from the five positive sigmas only. buckeye = cl.Development().fit(tri.loc["Buckeye Ins Grp", "othliab"]) diff --git a/chainladder/utils/weighted_regression.py b/chainladder/utils/weighted_regression.py index 6fd9959f5..c2992d7b3 100644 --- a/chainladder/utils/weighted_regression.py +++ b/chainladder/utils/weighted_regression.py @@ -265,12 +265,15 @@ def loglinear_interpolation(self, y): """ Fill undefined sigmas by regressing log(sigma) on development period. - As in R's ChainLadder (``estimate.sigma``), only positive sigmas enter - the regression: a zero sigma has no logarithm and would dominate the - fit. Zero sigmas are not filled. Where fewer than three positive sigmas - remain, the fill falls back to Mack's approximation. R's ``Mack.S.E`` - does the same for a single point, and for two points, where the - slope's p-value is undefined. + Only positive sigmas enter the regression. A zero sigma has no + logarithm, and a stand-in such as 1e-320 puts a point near -737 on + the log scale that outweighs every other point in the least-squares + fit. Zero sigmas are left as they are. + + With fewer than three positive sigmas, the fill comes from + ``mack_interpolation`` instead. One point gives no slope, and a line + through two points fits both exactly, so no residual is left to judge + it by. """ from chainladder.utils.utility_functions import num_to_nan @@ -283,8 +286,10 @@ def loglinear_interpolation(self, y): fill_ = xp.nan_to_num(xp.exp(reg.x * slope + intercept) * (1 - estimated)) use_mack = xp.sum(w, axis=self.axis, keepdims=True) < 3 if use_mack.any(): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=RuntimeWarning) + # mack_interpolation divides by the square of the sigma two + # periods back. Where that sigma is zero, the quotient is inf or + # nan and the cell's fill comes out as 0. + with np.errstate(divide="ignore", invalid="ignore"): mack_fill = xp.nan_to_num(self.mack_interpolation(y)) * (1 - estimated) fill_ = xp.where(use_mack, mack_fill, fill_) out = xp.nan_to_num(y) + fill_