diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index c6e8b78b9..fe9fddc04 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -1132,3 +1132,34 @@ def test_valid_params_pass_validation(raa): cl.Development(average=lambda age: "geometric", n_periods=[-1] * 9).fit(raa) dev = cl.Development(n_periods=4, average="simple") assert cl.read_json(dev.to_json()).get_params() == dev.get_params() + + +def test_loglinear_sigma_skips_zero_sigmas(clrd): + # GH 1609: zero sigmas used to enter the log-linear fit as log(1e-320). + tri = clrd["CumPaidLoss"] + + # One positive sigma (36-48) and a zero (48-60): no line is fitted, and + # Mack's approximation has no two positive sigmas in a row, so the + # 12-24 and 24-36 sigmas stay NaN instead of 1.8e308. + brotherhood = cl.Development().fit(tri.loc["Brotherhood Mut Ins Co", "prodliab"]) + assert np.nanmax(brotherhood.sigma_.set_backend("numpy").values) < 1e3 + assert np.isfinite(brotherhood.std_err_.set_backend("numpy").values).all() + + # Two positive sigmas (12-24, 24-36), then none at 36-48 and zeros after: + # the 36-48 fill comes from Mack's approximation, the same value as + # under sigma_interpolation="mack". + great_amer = tri.loc["Great Amer Grp", "medmal"] + loglinear = cl.Development().fit(great_amer).sigma_.set_backend("numpy") + mack = cl.Development(sigma_interpolation="mack").fit(great_amer) + fill = loglinear.values.ravel()[2] + assert np.isclose(fill, mack.sigma_.set_backend("numpy").values.ravel()[2]) + assert np.isclose(fill, 14.9781, atol=1e-4) + + # Five positive sigmas (24-36 to 72-84) and two zeros (84-96, 96-108): + # the 12-24 and 108-120 fills come from the five positive sigmas only. + buckeye = cl.Development().fit(tri.loc["Buckeye Ins Grp", "othliab"]) + sigma = buckeye.sigma_.set_backend("numpy").values.ravel() + period = np.arange(1, sigma.size + 1) + slope, intercept = np.polyfit(period[1:6], np.log(sigma[1:6]), 1) + assert np.allclose(sigma[[0, 8]], np.exp(intercept + slope * period[[0, 8]])) + assert np.isclose(sigma[0], 7.1819, atol=1e-4) diff --git a/chainladder/utils/weighted_regression.py b/chainladder/utils/weighted_regression.py index 5e0a51a4b..c2992d7b3 100644 --- a/chainladder/utils/weighted_regression.py +++ b/chainladder/utils/weighted_regression.py @@ -262,17 +262,37 @@ def std_err_fill(self): return self def loglinear_interpolation(self, y): - """Use Cases: generally for filling in last element of sigma_""" + """ + Fill undefined sigmas by regressing log(sigma) on development period. + + Only positive sigmas enter the regression. A zero sigma has no + logarithm, and a stand-in such as 1e-320 puts a point near -737 on + the log scale that outweighs every other point in the least-squares + fit. Zero sigmas are left as they are. + + With fewer than three positive sigmas, the fill comes from + ``mack_interpolation`` instead. One point gives no slope, and a line + through two points fits both exactly, so no residual is left to judge + it by. + """ from chainladder.utils.utility_functions import num_to_nan xp = self.xp - ly = y.copy() - ly = xp.log(xp.where(ly == 0, 1e-320, ly)) + ly = xp.log(xp.where(y > 0, y, xp.nan)) w = xp.nan_to_num(ly * 0 + 1) + estimated = xp.nan_to_num(y * 0 + 1) reg = WeightedRegression(self.axis, False, xp=xp).fit(None, ly, w) slope, intercept = reg.slope_, reg.intercept_ - fill_ = xp.exp(reg.x * slope + intercept) * (1 - w) - out = xp.nan_to_num(y) + xp.nan_to_num(fill_) + fill_ = xp.nan_to_num(xp.exp(reg.x * slope + intercept) * (1 - estimated)) + use_mack = xp.sum(w, axis=self.axis, keepdims=True) < 3 + if use_mack.any(): + # mack_interpolation divides by the square of the sigma two + # periods back. Where that sigma is zero, the quotient is inf or + # nan and the cell's fill comes out as 0. + with np.errstate(divide="ignore", invalid="ignore"): + mack_fill = xp.nan_to_num(self.mack_interpolation(y)) * (1 - estimated) + fill_ = xp.where(use_mack, mack_fill, fill_) + out = xp.nan_to_num(y) + fill_ return num_to_nan(out) def mack_interpolation(self, y):