One year semantics for the FRS build: survey year for observed amounts, policy year for modelled seeds

changelog.d/frs-year-semantics.md

@@ -0,0 +1,3 @@
+- Threshold FRS disability categories against the survey fiscal year's DWP rates, the same fiscal-converted parameter tree the disability flags read, and convert reported amounts back to weekly with the survey's 365.25/7 factor in both paths (uk-data#475, uk-data#476).
+- Validate `create_frs`'s `year` against the FRS release folder being read, and add a `policy_year` argument (the release's calibration year in the build) so Disabled Students' Allowance expenses are seeded for the FRS 2024-25 dataset (uk-data#477, uk-data#478).
+- Document that the saved base-year dataset is not uprate-invertible on the post-calibration rail, bus and road-fuel scalings (uk-data#479).

policyengine_uk_data/datasets/create_datasets.py

@@ -134,6 +134,7 @@ def main():
             frs = create_frs(
                 raw_frs_folder=STORAGE_FOLDER / frs_release.name,
                 year=frs_release.survey_year,
+                policy_year=frs_release.calibration_year,
                 include_internal_disability_reported_amounts=True,
             )
             strip_internal_disability_reported_amounts(frs).save(
@@ -310,6 +311,29 @@ def main():
                 )
                 update_dataset(materialize_step, "completed")
 
+            # The four aggregate scalers below run after the base-year
+            # materialisation on purpose. Rail and bus (subsidy and fares) are
+            # fitted by simulating the saved base-year file at the calibration
+            # year, the configuration consumers actually run: policyengine-uk
+            # re-uprates rail_usage (gov.dft.rail.ridership_index) and
+            # bus_fare_spending (CPI) at load, so a factor fitted on a
+            # calibration-year file and then down-rated would be uprated a
+            # second time. rail_usage, rail_subsidy_spending,
+            # bus_subsidy_spending and bus_fare_spending are not in
+            # uprating_factors.csv, so `uprate_dataset` leaves them untouched
+            # in either direction: uprating the saved file back to the
+            # calibration year reproduces the calibrated weights and monetary
+            # levels but keeps these post-calibration scalings, which the
+            # weight solve never saw. The saved file is therefore not
+            # uprate-invertible on those columns (uk-data#479).
+            #
+            # Road fuel is fitted differently: the litre-proxy scaler reads the
+            # saved file's own time_period (the base year), so it reconciles
+            # base-year litres at base-year pump prices to the base-year HMRC
+            # clearances. petrol_spending and diesel_spending are in
+            # uprating_factors.csv and carry their own load-time litre-proxy
+            # indices, so whether calibration-year litres hit the
+            # calibration-year target depends on those indices, not on this fit.
             update_dataset("Calibrate public service aggregates", "processing")
             from policyengine_uk_data.datasets.imputations.services.services import (
                 calibrate_rail_subsidy_spending,

policyengine_uk_data/datasets/disability_benefits.py

@@ -4,6 +4,15 @@
 inputs. The FRS observes reported amounts, so the data pipeline keeps those
 amounts as internal build intermediates and converts them to model inputs
 before datasets are published.
+
+Conventions shared by the category and flag derivations:
+
+- ``year`` is the survey year (fiscal year ``year``/``year + 1``, the
+  dataset's ``time_period``). Reported amounts are thresholded against the
+  DWP rates in force during that fiscal year, read from the fiscal-converted
+  ``gov`` parameter tree.
+- Reported amounts are weekly survey responses annualised in ``frs.py`` with
+  ``365.25 / 7``; both derivations convert back with the same factor.
 """
 
 from __future__ import annotations
@@ -14,7 +23,6 @@
 import pandas as pd
 from policyengine_uk import CountryTaxBenefitSystem
 from policyengine_uk.data import UKSingleYearDataset
-from policyengine_uk.model_api import WEEKS_IN_YEAR as MODEL_WEEKS_IN_YEAR
 
 
 DISABILITY_REPORTED_AMOUNT_COLUMNS = (
@@ -48,19 +56,22 @@
 )
 
 CATEGORY_THRESHOLD_WEEKLY_TOLERANCE = 1.0
+# The factor `frs.py` annualises weekly FRS amounts with. Converting back
+# with the model's 52-week constant would inflate weekly amounts by 0.34% and
+# make the GBP 1/week tolerance below mean GBP 1.37 (uk-data#476).
 SURVEY_REPORTED_AMOUNT_WEEKS_IN_YEAR = 365.25 / 7
 
 
 @lru_cache(maxsize=None)
-def _dwp_category_threshold_parameters(year: int):
-    # Match the category formulas removed from policyengine-uk. Those formulas
-    # thresholded reported amounts against the baseline DWP rates.
-    return CountryTaxBenefitSystem().parameters(year).baseline.gov.dwp
-
-
-@lru_cache(maxsize=None)
-def _dwp_flag_parameters(year: int):
-    # Match the FRS disability flag derivation that already lived in uk-data.
+def _dwp_rate_parameters(year: int):
+    """DWP weekly rates in force during the survey's fiscal year.
+
+    policyengine-uk rewrites ``parameters.gov`` onto fiscal years at load, so
+    ``gov`` at ``year`` carries the rates paid from April of that year. The
+    ``baseline`` clone is taken before that rewrite and stays on calendar
+    instants, so ``baseline`` at ``year`` is the previous fiscal year's
+    table; categories and flags must read the same tree (uk-data#475).
+    """
     return CountryTaxBenefitSystem().parameters(year).gov.dwp
 
 
@@ -85,7 +96,9 @@ def _category_from_reported_amount(
     thresholds: tuple[tuple[str, float], ...],
 ) -> np.ndarray:
     weekly_amount = pd.to_numeric(reported_amount, errors="coerce").fillna(0)
-    weekly_amount = weekly_amount.to_numpy(dtype=float) / MODEL_WEEKS_IN_YEAR
+    weekly_amount = (
+        weekly_amount.to_numpy(dtype=float) / SURVEY_REPORTED_AMOUNT_WEEKS_IN_YEAR
+    )
     category = np.full(len(weekly_amount), "NONE", dtype=object)
     for category_name, weekly_rate in thresholds:
         # FRS benefit amounts are weekly survey responses annualised upstream;
@@ -110,7 +123,7 @@ def add_disability_benefit_categories_from_reported_amounts(
     if not inplace:
         person = person.copy()
 
-    dwp = _dwp_category_threshold_parameters(int(year))
+    dwp = _dwp_rate_parameters(int(year))
     mappings = (
         (
             "attendance_allowance_reported",
@@ -176,7 +189,7 @@ def add_disability_benefit_flags_from_reported_amounts(
     if not inplace:
         person = person.copy()
 
-    dwp = _dwp_flag_parameters(int(year))
+    dwp = _dwp_rate_parameters(int(year))
     attendance_allowance = _reported_amount(person, "attendance_allowance_reported")
     dla_sc = _reported_amount(person, "dla_sc_reported")
     pip_dl = _reported_amount(person, "pip_dl_reported")

policyengine_uk_data/datasets/frs.py

@@ -7,6 +7,8 @@
 modelling and policy analysis.
 """
 
+import re
+import warnings
 from functools import lru_cache
 from pathlib import Path
 
@@ -440,33 +442,47 @@ def allocate_reported_education_grants(
 
 
 def calculate_disabled_students_allowance_reported_grant_capacity(
-    sim, year: int, maximum: float
+    sim, policy_year: int, maximum: float
 ) -> np.ndarray:
-    if year < DISABLED_STUDENTS_ALLOWANCE_FIRST_MODELED_YEAR:
+    """DSA capacity for the first policy year the dataset will be simulated at.
+
+    ``DISABLED_STUDENTS_ALLOWANCE_FIRST_MODELED_YEAR`` is the first policy
+    year policyengine-uk models DSA, so the gate compares against the policy
+    year rather than the survey year: an FRS 2024-25 build simulated from
+    2025 must seed DSA expenses even though its survey year is 2024
+    (uk-data#478).
+    """
+    if policy_year < DISABLED_STUDENTS_ALLOWANCE_FIRST_MODELED_YEAR:
         return np.zeros_like(
             np.asarray(
                 sim.calculate(
-                    DISABLED_STUDENTS_ALLOWANCE_ELIGIBILITY_VARIABLES[0], year
+                    DISABLED_STUDENTS_ALLOWANCE_ELIGIBILITY_VARIABLES[0], policy_year
                 )
             ),
             dtype=float,
         )
 
     eligible = None
     for variable in DISABLED_STUDENTS_ALLOWANCE_ELIGIBILITY_VARIABLES:
-        variable_eligible = np.asarray(sim.calculate(variable, year), dtype=bool)
+        variable_eligible = np.asarray(sim.calculate(variable, policy_year), dtype=bool)
         eligible = (
             variable_eligible if eligible is None else eligible & variable_eligible
         )
     equivalent_support = np.asarray(
-        sim.calculate("disabled_students_allowance_receives_equivalent_support", year),
+        sim.calculate(
+            "disabled_students_allowance_receives_equivalent_support", policy_year
+        ),
         dtype=bool,
     )
     return np.where(eligible & ~equivalent_support, float(maximum), 0.0)
 
 
 def split_reported_education_grants(
-    pe_person: pd.DataFrame, sim, year: int, dsa_maximum: float
+    pe_person: pd.DataFrame,
+    sim,
+    year: int,
+    dsa_maximum: float,
+    policy_year: int | None = None,
 ) -> pd.DataFrame:
     """Move specific modelled grants out of the generic education-grant residual.
 
@@ -475,15 +491,23 @@ def split_reported_education_grants(
     counting the same reported FRS grant amount in the generic residual.
     DSA lacks a modelled amount signal, so its allocation seeds eligible
     expenses directly where the DSA parameter is available.
+
+    ``year`` is the survey year the grant capacities are evaluated at.
+    ``policy_year`` (default ``year``) is the first policy year the dataset
+    will be simulated at; it gates the DSA seed, which policyengine-uk only
+    models from ``DISABLED_STUDENTS_ALLOWANCE_FIRST_MODELED_YEAR``.
     """
 
+    if policy_year is None:
+        policy_year = year
+
     grant_capacities = {
         variable: sim.calculate(variable, year)
         for variable in FORMULA_MODELED_EDUCATION_GRANT_VARIABLES
     }
     grant_capacities[DISABLED_STUDENTS_ALLOWANCE_EXPENSE_INPUT] = (
         calculate_disabled_students_allowance_reported_grant_capacity(
-            sim, year, dsa_maximum
+            sim, policy_year, dsa_maximum
         )
     )
     allocations = allocate_reported_education_grants(
@@ -498,10 +522,59 @@ def split_reported_education_grants(
     return pe_person
 
 
+FRS_RELEASE_FOLDER_PATTERN = re.compile(r"^frs_(\d{4})_(\d{2})$")
+
+
+def survey_year_from_frs_folder_name(raw_frs_folder) -> int | None:
+    """Survey year encoded in an FRS release folder name (``frs_2024_25`` -> 2024).
+
+    Returns ``None`` for folders outside the release naming convention, such
+    as synthetic fixtures in tests.
+    """
+    match = FRS_RELEASE_FOLDER_PATTERN.match(Path(raw_frs_folder).name)
+    if match is None:
+        return None
+    return int(match.group(1))
+
+
+def validate_frs_survey_year(raw_frs_folder, year: int) -> None:
+    """Refuse a ``year`` that does not match the FRS release being read.
+
+    ``year`` stamps the dataset's ``time_period``, selects vintage-dependent
+    columns, and thresholds reported benefit amounts against that fiscal
+    year's rates. Nothing in ``create_frs`` uprates, so passing the survey
+    year plus one asserts the survey's amounts as the next year's and
+    mis-thresholds every disability category and flag (uk-data#477). The
+    folder name is the release identity available here: ``frs_2024_25`` is
+    survey year 2024. A folder outside that convention (a synthetic fixture,
+    an ad hoc extraction) cannot be checked, so the function warns and
+    returns rather than guessing.
+    """
+    folder_survey_year = survey_year_from_frs_folder_name(raw_frs_folder)
+    if folder_survey_year is None:
+        warnings.warn(
+            f"FRS folder {Path(raw_frs_folder).name!r} does not follow the "
+            f"frs_YYYY_YY release naming, so year={year} cannot be checked "
+            "against the release being read.",
+            stacklevel=2,
+        )
+        return
+    if int(year) != folder_survey_year:
+        raise ValueError(
+            f"FRS folder {Path(raw_frs_folder).name!r} is survey year "
+            f"{folder_survey_year} (fiscal year {folder_survey_year}/"
+            f"{(folder_survey_year + 1) % 100:02d}) but year={year} was passed. "
+            "`year` is the survey year and stamps time_period; it does not "
+            "uprate. Build with the survey year and uprate the saved dataset "
+            "with `uprate_dataset` instead."
+        )
+
+
 def create_frs(
     raw_frs_folder: str,
     year: int,
     include_internal_disability_reported_amounts: bool = False,
+    policy_year: int | None = None,
 ) -> UKSingleYearDataset:
     """
     Process raw FRS data into PolicyEngine UK dataset format.
@@ -513,17 +586,34 @@ def create_frs(
 
     Args:
         raw_frs_folder: Path to folder containing raw FRS .tab files.
-        year: Survey year for the dataset.
+        year: Survey year for the dataset: the fiscal year the fieldwork
+            covers (2024 for FRS 2024-25). It stamps ``time_period``, selects
+            vintage-dependent survey columns, and thresholds reported benefit
+            amounts against that fiscal year's rates. It must match the
+            release folder being read; nothing here uprates.
         include_internal_disability_reported_amounts: Keep raw disability
             benefit amount intermediates for downstream imputation. Public
             saved datasets should leave this as ``False``.
+        policy_year: First policy year the dataset will be simulated at (the
+            release's calibration year in the build). Gates seeds for
+            programmes policyengine-uk models from a later year than the
+            survey, currently Disabled Students' Allowance. Defaults to
+            ``year``.
 
     Returns:
         UKSingleYearDataset with processed FRS data ready for policy simulation.
     """
     raw_folder = Path(raw_frs_folder)
     if not raw_folder.exists():
         raise FileNotFoundError(f"Raw folder {raw_folder} does not exist.")
+    validate_frs_survey_year(raw_folder, year)
+    if policy_year is None:
+        policy_year = year
+    if int(policy_year) < int(year):
+        raise ValueError(
+            f"policy_year={policy_year} precedes survey year={year}; the dataset "
+            "cannot be simulated at a policy year before its survey year."
+        )
 
     frs = {}
     # Store SALSAC values before numeric conversion (for salary sacrifice
@@ -1379,10 +1469,14 @@ def determine_education_level(fted_val, typeed2_val, age_val):
         )
         student_support_sim = Microsimulation(dataset=student_support_dataset)
         dsa_maximum = student_support_sim.tax_benefit_system.parameters(
-            year
+            policy_year
         ).gov.dfe.disabled_students_allowance.maximum
         pe_person = split_reported_education_grants(
-            pe_person, student_support_sim, year, dsa_maximum
+            pe_person,
+            student_support_sim,
+            year,
+            dsa_maximum,
+            policy_year=policy_year,
         )
 
     # Generate stochastic take-up decisions
@@ -1580,5 +1674,6 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray:
     frs = create_frs(
         raw_frs_folder=STORAGE_FOLDER / CURRENT_FRS_RELEASE.name,
         year=CURRENT_FRS_RELEASE.survey_year,
+        policy_year=CURRENT_FRS_RELEASE.calibration_year,
     )
     frs.save(STORAGE_FOLDER / CURRENT_FRS_RELEASE.base_dataset_file)