Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions chainladder/development/tests/test_development.py
Original file line number Diff line number Diff line change
Expand Up @@ -1132,3 +1132,34 @@ def test_valid_params_pass_validation(raa):
cl.Development(average=lambda age: "geometric", n_periods=[-1] * 9).fit(raa)
dev = cl.Development(n_periods=4, average="simple")
assert cl.read_json(dev.to_json()).get_params() == dev.get_params()


def test_loglinear_sigma_skips_zero_sigmas(clrd):
# GH 1609: zero sigmas used to enter the log-linear fit as log(1e-320).
tri = clrd["CumPaidLoss"]

# One positive sigma (36-48) and a zero (48-60): no line is fitted, and
# Mack's approximation has no two positive sigmas in a row, so the
# 12-24 and 24-36 sigmas stay NaN instead of 1.8e308.
brotherhood = cl.Development().fit(tri.loc["Brotherhood Mut Ins Co", "prodliab"])
assert np.nanmax(brotherhood.sigma_.set_backend("numpy").values) < 1e3

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please double check my logic flow here.

we are here to test mack's approximation for sigma_ when loglinear regression doesn't have enough valid data. have we tested mack's approximatio itself somewhere? i.e. are any of our hardcoded mack tests covering the approximation for sigma?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your logic is right, and the Brotherhood case didn't test it. Its raw sigmas are NaN, NaN, 7.67, 0, then NaN. Mack's approximation needs two positive sigmas just before a gap, so it fills nothing there; the case only checks that the 1.8e308 fills are gone. I fixed its comment and added Great Amer Grp / medmal, whose sigmas are 14.98, 52.90, then a gap: the fallback fills 36-48 with 14.978, and the test checks that against sigma_interpolation="mack" on the same triangle. On main that fill is 1.2e-142. The approximation itself is covered by printed values: test_mack1999_hardcode checks mortgage's filled last sigma (16.887 against 16.89 in Mack 1999), and test_cdr_matches_merz2008 checks mw2008's filled last sigma^2 (0.040 against 0.04). test_mack1994_hardcode only reaches it through the oldest origin's standard error.

(This comment was written with Árbol, an AI assistant, under my direction.)

assert np.isfinite(brotherhood.std_err_.set_backend("numpy").values).all()

# Two positive sigmas (12-24, 24-36), then none at 36-48 and zeros after:
# the 36-48 fill comes from Mack's approximation, the same value as
# under sigma_interpolation="mack".
great_amer = tri.loc["Great Amer Grp", "medmal"]
loglinear = cl.Development().fit(great_amer).sigma_.set_backend("numpy")
mack = cl.Development(sigma_interpolation="mack").fit(great_amer)
fill = loglinear.values.ravel()[2]
assert np.isclose(fill, mack.sigma_.set_backend("numpy").values.ravel()[2])
assert np.isclose(fill, 14.9781, atol=1e-4)

# Five positive sigmas (24-36 to 72-84) and two zeros (84-96, 96-108):
# the 12-24 and 108-120 fills come from the five positive sigmas only.
buckeye = cl.Development().fit(tri.loc["Buckeye Ins Grp", "othliab"])
sigma = buckeye.sigma_.set_backend("numpy").values.ravel()
period = np.arange(1, sigma.size + 1)
slope, intercept = np.polyfit(period[1:6], np.log(sigma[1:6]), 1)
assert np.allclose(sigma[[0, 8]], np.exp(intercept + slope * period[[0, 8]]))
assert np.isclose(sigma[0], 7.1819, atol=1e-4)
30 changes: 25 additions & 5 deletions chainladder/utils/weighted_regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,17 +262,37 @@ def std_err_fill(self):
return self

def loglinear_interpolation(self, y):
"""Use Cases: generally for filling in last element of sigma_"""
"""
Fill undefined sigmas by regressing log(sigma) on development period.

Only positive sigmas enter the regression. A zero sigma has no
logarithm, and a stand-in such as 1e-320 puts a point near -737 on
the log scale that outweighs every other point in the least-squares
fit. Zero sigmas are left as they are.

With fewer than three positive sigmas, the fill comes from
``mack_interpolation`` instead. One point gives no slope, and a line
through two points fits both exactly, so no residual is left to judge
it by.
"""
from chainladder.utils.utility_functions import num_to_nan

xp = self.xp
ly = y.copy()
ly = xp.log(xp.where(ly == 0, 1e-320, ly))
ly = xp.log(xp.where(y > 0, y, xp.nan))
w = xp.nan_to_num(ly * 0 + 1)
estimated = xp.nan_to_num(y * 0 + 1)
reg = WeightedRegression(self.axis, False, xp=xp).fit(None, ly, w)
slope, intercept = reg.slope_, reg.intercept_
fill_ = xp.exp(reg.x * slope + intercept) * (1 - w)
out = xp.nan_to_num(y) + xp.nan_to_num(fill_)
fill_ = xp.nan_to_num(xp.exp(reg.x * slope + intercept) * (1 - estimated))
use_mack = xp.sum(w, axis=self.axis, keepdims=True) < 3
if use_mack.any():
# mack_interpolation divides by the square of the sigma two
# periods back. Where that sigma is zero, the quotient is inf or
# nan and the cell's fill comes out as 0.
with np.errstate(divide="ignore", invalid="ignore"):
mack_fill = xp.nan_to_num(self.mack_interpolation(y)) * (1 - estimated)
fill_ = xp.where(use_mack, mack_fill, fill_)
out = xp.nan_to_num(y) + fill_
return num_to_nan(out)

def mack_interpolation(self, y):
Expand Down
Loading