From e12be048600b27fb3d47548411f6f8a04744c7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:01:55 +0200 Subject: [PATCH 01/10] Port the LCFS consumption, ETB VAT, and ETB services/NHS layers as declarative source stages Three stages inserted between regional_property_uprating and the SPI spine trio (incumbent order: consumption before SPI row-stacking): lcfs_consumption (18-output seeded weighted QRF chain in the incumbent's conditioning order, WAS-bridged has_fuel flag, NEED energy raking via the new generic iterative_proportional_fit operation, donor-support clipping, non-ICE petrol/diesel zeroing), etb_vat (single-output QRF; VAT rates from a cited anchors resource with parameter_path lockstep - the incumbent's silent 0.03-vs-0.025 fallback dies), and etb_services (3-output chain, rail_usage ratio, deterministic NHS age-gender person allocation with the signed 85+ fold-in / full-denominator / half-open-band fixes). Supporting: three new generic operation kinds (iterative_proportional_fit, bridge_donor_column_via_qrf, allocate_per_capita_from_cell_table) plus a condition parameter on assign_binary_from_rate; five cited build/uk resources (NEED 2023 kWh targets + Ofgem Q2 2026 rates, LCFS/ETB/services policy anchors, the NHS age-gender table) flagged chronicle_candidate; sha-bound SDC-rounded support bounds for all three donors with licensed round-trip tests; the uk_aggregate_admin gate (NEED/Ofgem + NHS anchors); gate-battery digests re-cut; coverage-manifest family entries; driver wiring with caller-supplied LCFS/ETB tabs and input-artifact pins; the carried #717 review fixes (fail-closed SPI support weight kind, last_result resume comments, the seed+1 convention note). full_rate_vat_expenditure_rate is deliberately absent from nonnegative_outputs (negative donor-realized support, 4 of 4,199 rows - the net_financial_wealth precedent). Donor uprating is identity at this vintage (LCFS survey year == 2023 build year) and is deferred to the FRS 2024-25 refresh (#687) rather than shipped as a dead declared op. Implements microcosm#682 (workstream E6 of #665). Codex implemented under /codex-implement-plan; Claude review adjudicated the deviations and added the engine-tree anchor lockstep test. Co-Authored-By: Claude Fable 5 --- changelog.d/682-uk-lcfs-consumption.added.md | 6 + .../src/microcosm/build/raking.py | 96 + .../src/microcosm/build/source_manifest.py | 3 + .../microcosm/build/uk/country_package.json | 40 + .../build/uk/etb_policy_anchors.json | 23 + .../build/uk/etb_services_anchors.json | 20 + .../build/uk/etb_services_support_bounds.json | 28 + .../build/uk/etb_vat_support_bounds.json | 20 + .../src/microcosm/build/uk/gates.json | 54 +- .../build/uk/lcfs_consumption_anchors.json | 27 + .../uk/lcfs_consumption_support_bounds.json | 77 + .../build/uk/need_energy_targets.json | 75 + .../uk/nhs_consumption_by_age_gender.json | 1778 +++++++++++++++++ .../uk/release_input_coverage_manifest.json | 88 +- .../src/microcosm/build/uk/source_stages.json | 438 ++++ .../build/uk_runtime/battery_bindings.py | 87 +- .../build/uk_runtime/etb_services.py | 376 ++++ .../src/microcosm/build/uk_runtime/etb_vat.py | 196 ++ .../build/uk_runtime/lcfs_consumption.py | 680 +++++++ .../build/uk_runtime/source_runtime.py | 6 + .../microcosm/build/uk_runtime/spi_income.py | 6 +- .../microcosm/build/uk_runtime/spi_spine.py | 39 +- .../build/uk_runtime/terminal_gates.py | 3 + .../tests/test_country_spec.py | 42 +- .../tests/test_uk_battery_bindings.py | 80 +- .../tests/test_uk_consumption_resources.py | 152 ++ .../tests/test_uk_etb_services.py | 217 ++ .../microcosm-build/tests/test_uk_etb_vat.py | 106 + .../tests/test_uk_frs_spine.py | 38 +- .../tests/test_uk_lcfs_consumption.py | 158 ++ .../tests/test_uk_national_build.py | 1 + .../tests/test_uk_nhs_allocation.py | 83 + .../microcosm-build/tests/test_uk_raking.py | 134 ++ .../tests/test_uk_release_input_coverage.py | 3 + .../tests/test_uk_source_runtime.py | 6 + .../tests/test_uk_source_stages.py | 150 +- .../tests/test_uk_spi_spine.py | 37 +- .../src/microcosm/data/contract.py | 9 +- .../microcosm-data/tests/test_contract.py | 12 +- tools/build_uk_e6_support_bounds.py | 174 ++ tools/build_uk_frs_spine.py | 77 +- ...uild_uk_release_input_coverage_manifest.py | 12 + 42 files changed, 5581 insertions(+), 76 deletions(-) create mode 100644 changelog.d/682-uk-lcfs-consumption.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/raking.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk/etb_policy_anchors.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/etb_services_anchors.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/etb_services_support_bounds.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/etb_vat_support_bounds.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_anchors.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/need_energy_targets.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk/nhs_consumption_by_age_gender.json create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py create mode 100644 packages/microcosm-build/tests/test_uk_consumption_resources.py create mode 100644 packages/microcosm-build/tests/test_uk_etb_services.py create mode 100644 packages/microcosm-build/tests/test_uk_etb_vat.py create mode 100644 packages/microcosm-build/tests/test_uk_lcfs_consumption.py create mode 100644 packages/microcosm-build/tests/test_uk_nhs_allocation.py create mode 100644 packages/microcosm-build/tests/test_uk_raking.py create mode 100644 tools/build_uk_e6_support_bounds.py diff --git a/changelog.d/682-uk-lcfs-consumption.added.md b/changelog.d/682-uk-lcfs-consumption.added.md new file mode 100644 index 00000000..878e81a7 --- /dev/null +++ b/changelog.d/682-uk-lcfs-consumption.added.md @@ -0,0 +1,6 @@ +UK E6 source stages: `lcfs_consumption` (18-output LCFS QRF chain with the WAS +has-fuel bridge and NEED energy raking as a generic `iterative_proportional_fit` +operation), `etb_vat`, and `etb_services` (public-services QRF, NHS age-gender +person allocation, `rail_usage`), with cited NEED/Ofgem/VAT/NHS anchor +resources, sha-bound SDC-rounded support bounds, the `uk_aggregate_admin` gate, +and the carried #717 review fixes (microcosm#682). diff --git a/packages/microcosm-build/src/microcosm/build/raking.py b/packages/microcosm-build/src/microcosm/build/raking.py new file mode 100644 index 00000000..88fa9953 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/raking.py @@ -0,0 +1,96 @@ +"""Generic iterative proportional fitting helpers for source stages.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +__all__ = ["MarginSpec", "iterative_proportional_fit"] + + +@dataclass(frozen=True) +class MarginSpec: + """Mean targets for one categorical margin. + + ``targets`` maps category value -> output column -> target mean. The helper + deliberately works in means because the UK LCFS/NEED application receives + published mean kWh/spend cells, but the utility is country-neutral. + """ + + column: str + targets: Mapping[object, Mapping[str, float]] + + +def iterative_proportional_fit( + frame: pd.DataFrame, + *, + columns: Sequence[str], + margins: Sequence[MarginSpec], + iterations: int, + weight_column: str | None = None, +) -> pd.DataFrame: + """Scale columns in-place-by-copy to match declared cell means. + + Empty cells, zero-current-mean cells, and categories absent from the + declared targets are skipped. That preserves support zeros and lets a + country-specific caller intentionally leave unmapped enum values alone. + """ + + if iterations < 1: + raise ValueError("iterations must be positive") + if not columns: + raise ValueError("at least one column is required") + + result = frame.copy() + for column in columns: + if column not in result: + raise KeyError(f"frame is missing raked column {column!r}") + result[column] = pd.to_numeric(result[column], errors="coerce").fillna(0.0) + if weight_column is not None and weight_column not in result: + raise KeyError(f"frame is missing weight column {weight_column!r}") + + weights = None + if weight_column is not None: + weights = pd.to_numeric(result[weight_column], errors="coerce").fillna(0.0) + if (weights < 0).any(): + raise ValueError("raking weights must be nonnegative") + + for _ in range(iterations): + for margin in margins: + if margin.column not in result: + raise KeyError(f"frame is missing margin column {margin.column!r}") + for category, target_by_column in margin.targets.items(): + mask = result[margin.column] == category + if not bool(mask.any()): + continue + for column in columns: + if column not in target_by_column: + continue + current = _cell_mean( + result.loc[mask, column], + None if weights is None else weights.loc[mask], + ) + if current <= 0 or not np.isfinite(current): + continue + target = float(target_by_column[column]) + if not np.isfinite(target) or target < 0: + raise ValueError( + f"target for {margin.column!r}={category!r}, " + f"{column!r} must be finite and nonnegative" + ) + result.loc[mask, column] *= target / current + return result + + +def _cell_mean(values: pd.Series, weights: pd.Series | None) -> float: + data = pd.to_numeric(values, errors="coerce").fillna(0.0).to_numpy(dtype=float) + if weights is None: + return float(data.mean()) if len(data) else 0.0 + w = weights.to_numpy(dtype=float) + total = float(w.sum()) + if total <= 0: + return 0.0 + return float(np.dot(data, w) / total) diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index 6d0c0441..c4dafc79 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -49,11 +49,13 @@ "assign_clipped_normal", "assign_uniform_draw", "aggregate_person_to_benunit", + "allocate_per_capita_from_cell_table", "allocate_within_group_waterfall", "allocate_zero_weight_prior_mass", "annualize_periodic_amounts", "assemble_group_entities", "attribute_self_employed_health_premiums", + "bridge_donor_column_via_qrf", "calibrate_binary_assignment", "calibrate_binary_assignment_joint_targets", "classify_hmrc_income_facts_with_reviewed_fences", @@ -112,6 +114,7 @@ "impute_retirement_distributions_to_puf_support", "impute_workers_compensation_to_puf_support", "impute_weeks_unemployed_to_puf_support", + "iterative_proportional_fit", "map_columns", "map_coded_amounts", "materialize_hmrc_income_bands_fail_closed", diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index 299baae7..5a161337 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -77,6 +77,46 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "need_energy_targets.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "lcfs_consumption_anchors.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "etb_policy_anchors.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "etb_services_anchors.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "nhs_consumption_by_age_gender.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "lcfs_consumption_support_bounds.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "etb_vat_support_bounds.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, + { + "path": "etb_services_support_bounds.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "regional_land_values.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/etb_policy_anchors.json b/packages/microcosm-build/src/microcosm/build/uk/etb_policy_anchors.json new file mode 100644 index 00000000..8979de14 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/etb_policy_anchors.json @@ -0,0 +1,23 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "PolicyEngine UK VAT parameters for 2023; incumbent fallback removed.", + "chronicle_candidate": true + }, + "vat": { + "standard_rate": { + "value": 0.2, + "period": 2023, + "parameter_path": "gov.hmrc.vat.standard_rate" + }, + "reduced_rate_share": { + "value": 0.025, + "period": 2023, + "parameter_path": "gov.hmrc.vat.reduced_rate_share" + } + }, + "chronicle": [ + "E6 committed VAT anchors for ETB VAT imputation." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/etb_services_anchors.json b/packages/microcosm-build/src/microcosm/build/uk/etb_services_anchors.json new file mode 100644 index 00000000..2d9bd2b1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/etb_services_anchors.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "DfT rail fare index and NHS 2025/26 budget anchor.", + "chronicle_candidate": true + }, + "rail_fare_index_2023": { + "value": 1.11, + "period": 2023, + "parameter_path": "gov.dft.rail.fare_index" + }, + "nhs_budget_2025_26": { + "value": 202000000000, + "citation": "NHS 2025/26 budget used by incumbent services allocation." + }, + "chronicle": [ + "E6 committed ETB services and NHS anchors." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/etb_services_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/etb_services_support_bounds.json new file mode 100644 index 00000000..e0df8861 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/etb_services_support_bounds.json @@ -0,0 +1,28 @@ +{ + "version": 1, + "country": "uk", + "policy": "Disclosure-safe outward-rounded ETB services support bounds generated from pinned licensed donor tabs. Values are rounded outward to one significant figure; exact donor min/max values are not committed.", + "source": { + "ukds_study_number": 8856, + "doi": "10.5255/UKDA-SN-8856-4", + "tab_sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "sdc_treatment": "Exact donor min/max values are rounded outward to one significant figure before commit." + }, + "bounds": { + "bus_subsidy_spending": [ + 0.0, + 20000 + ], + "dfe_education_spending": [ + 0.0, + 90000 + ], + "rail_subsidy_spending": [ + 0.0, + 20000 + ] + }, + "chronicle": [ + "Support bounds generated for ETB services from source SHA pins." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/etb_vat_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/etb_vat_support_bounds.json new file mode 100644 index 00000000..a12cb90f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/etb_vat_support_bounds.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "country": "uk", + "policy": "Disclosure-safe outward-rounded ETB VAT support bounds generated from pinned licensed donor tabs. Values are rounded outward to one significant figure; exact donor min/max values are not committed.", + "source": { + "ukds_study_number": 8856, + "doi": "10.5255/UKDA-SN-8856-4", + "tab_sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "sdc_treatment": "Exact donor min/max values are rounded outward to one significant figure before commit." + }, + "bounds": { + "full_rate_vat_expenditure_rate": [ + -4, + 60 + ] + }, + "chronicle": [ + "Support bounds generated for ETB VAT from source SHA pins." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index e8f57686..d397abbf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -115,9 +115,56 @@ "phase": "terminal", "criticality": "release_blocking", "parameters": { - "support_bounds_resource": "was_wealth_support_bounds.json" + "support_bounds_resources": [ + "was_wealth_support_bounds.json", + "lcfs_consumption_support_bounds.json", + "etb_vat_support_bounds.json", + "etb_services_support_bounds.json" + ] + }, + "notes": "Every WAS/LCFS/ETB imputed output covered by support clipping must remain inside committed disclosure-safe donor support bounds. Exact donor ranges are used in-run for clipping; this release gate uses outward-rounded reviewed bounds so unit-record donor minima and maxima are never committed." + }, + { + "id": "uk_aggregate_admin", + "gate": "aggregate_admin", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": { + "default_rtol": 0.15, + "anchors": [ + { + "name": "need_electricity_mean_spending", + "entity": "household", + "measure": "electricity_consumption", + "value": 882.91463, + "period": "2023", + "source": "Unweighted mean of NEED 2023 income-band electricity kWh anchors converted with Ofgem Q2 2026 unit rates.", + "family": "need_energy", + "tolerance": 132.4371945 + }, + { + "name": "need_gas_mean_spending", + "entity": "household", + "measure": "gas_consumption", + "value": 700.3661, + "period": "2023", + "source": "Unweighted mean of NEED 2023 income-band gas kWh anchors converted with Ofgem Q2 2026 unit rates.", + "family": "need_energy", + "tolerance": 105.054915 + }, + { + "name": "nhs_spending_total", + "entity": "person", + "measure": "nhs_spending", + "value": 202000000000, + "period": "2025_26", + "source": "NHS 2025/26 budget anchor in etb_services_anchors.json.", + "family": "nhs", + "tolerance": 30300000000 + } + ] }, - "notes": "Every WAS-imputed wealth output must remain inside the committed disclosure-safe donor support bounds. Exact donor ranges are used in-run for clipping; this release gate uses outward-rounded reviewed bounds so unit-record WAS minima and maxima are never committed." + "notes": "Release-blocking aggregate-admin gate for E6 NEED energy anchors and the NHS budget. The build supplies SDC-safe measured aggregates as evidence; the gate checks signs and tolerances." }, { "id": "uk_export_surface", @@ -149,6 +196,8 @@ "household.region_code_oa", "household.stocks_and_shares_isa", "person.aa_category", + "person.a_and_e_visits", + "person.admitted_patient_visits", "person.age_started_or_accepted_current_education_or_training", "person.attends_private_school_random_draw", "person.charitable_investment_gifts", @@ -163,6 +212,7 @@ "person.is_in_non_advanced_education", "person.is_parent", "person.legacy_jobseeker_proxy", + "person.outpatient_visits", "person.pension_contributions_via_salary_sacrifice", "person.pip_dl_category", "person.pip_m_category", diff --git a/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_anchors.json b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_anchors.json new file mode 100644 index 00000000..5bddd1b7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_anchors.json @@ -0,0 +1,27 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "NTS 2024 ICE vehicle share, DESNZ pump prices, road-fuel volume/population indices, and PolicyEngine UK CPI parameter paths.", + "chronicle_candidate": true + }, + "nts_ice_share": { + "value": 0.9, + "period": 2024, + "citation": "National Travel Survey 2024 vehicle fuel-type share." + }, + "fuel_prices_gbp_per_litre": { + "2023": { + "petrol": 1.4615903846153844, + "diesel": 1.5348538461538461 + } + }, + "cpi": { + "parameter_path": "gov.economic_assumptions.indices.obr.consumer_price_index", + "start_period": 2023, + "target_period": 2023 + }, + "chronicle": [ + "E6 committed LCFS consumption anchors with parameter paths for lockstep tests." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json new file mode 100644 index 00000000..dca38b49 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json @@ -0,0 +1,77 @@ +{ + "version": 1, + "country": "uk", + "policy": "Disclosure-safe outward-rounded LCFS consumption support bounds generated from pinned licensed donor tabs. Values are rounded outward to one significant figure; exact donor min/max values are not committed.", + "source": { + "ukds_study_number": 9468, + "doi": "10.5255/UKDA-SN-9468-3", + "household_tab_sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", + "person_tab_sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", + "sdc_treatment": "Exact donor min/max values are rounded outward to one significant figure before commit." + }, + "bounds": { + "alcohol_and_tobacco_consumption": [ + 0.0, + 20000 + ], + "bus_fare_spending": [ + 0.0, + 9000 + ], + "clothing_and_footwear_consumption": [ + 0.0, + 40000 + ], + "communication_consumption": [ + 0.0, + 40000 + ], + "diesel_spending": [ + 0.0, + 30000 + ], + "education_consumption": [ + 0.0, + 200000 + ], + "food_and_non_alcoholic_beverages_consumption": [ + 0.0, + 20000 + ], + "health_consumption": [ + 0.0, + 100000 + ], + "household_furnishings_consumption": [ + 0.0, + 200000 + ], + "housing_water_and_electricity_consumption": [ + -20000, + 300000 + ], + "miscellaneous_consumption": [ + 0.0, + 70000 + ], + "petrol_spending": [ + 0.0, + 20000 + ], + "recreation_consumption": [ + 0.0, + 300000 + ], + "restaurants_and_hotels_consumption": [ + 0.0, + 60000 + ], + "transport_consumption": [ + 0.0, + 200000 + ] + }, + "chronicle": [ + "Support bounds generated for LCFS consumption from source SHA pins." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/need_energy_targets.json b/packages/microcosm-build/src/microcosm/build/uk/need_energy_targets.json new file mode 100644 index 00000000..fb50e995 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/need_energy_targets.json @@ -0,0 +1,75 @@ +{ + "version": 1, + "country": "uk", + "source": { + "citation": "NEED 2023 headline tables 5b/6b, 9b/10b, 11b/12b, 15b/16b; Ofgem Q2 2026 unit rates.", + "ofgem_q2_2026": { + "electricity_gbp_per_kwh": 0.2467, + "gas_gbp_per_kwh": 0.0574 + }, + "chronicle_candidate": true + }, + "income_bands": [ + {"label": "under_15k", "lower": 0, "upper": 15000, "gas_kwh": 7755, "electricity_kwh": 2412}, + {"label": "15k_20k", "lower": 15000, "upper": 20000, "gas_kwh": 9196, "electricity_kwh": 2700}, + {"label": "20k_30k", "lower": 20000, "upper": 30000, "gas_kwh": 9886, "electricity_kwh": 2915}, + {"label": "30k_40k", "lower": 30000, "upper": 40000, "gas_kwh": 10697, "electricity_kwh": 3114}, + {"label": "40k_50k", "lower": 40000, "upper": 50000, "gas_kwh": 11230, "electricity_kwh": 3276}, + {"label": "50k_60k", "lower": 50000, "upper": 60000, "gas_kwh": 11721, "electricity_kwh": 3410}, + {"label": "60k_70k", "lower": 60000, "upper": 70000, "gas_kwh": 12200, "electricity_kwh": 3548}, + {"label": "70k_100k", "lower": 70000, "upper": 100000, "gas_kwh": 13244, "electricity_kwh": 3872}, + {"label": "100k_150k", "lower": 100000, "upper": 150000, "gas_kwh": 15727, "electricity_kwh": 4598}, + {"label": "over_150k", "lower": 150000, "upper": null, "gas_kwh": 20359, "electricity_kwh": 5944} + ], + "tenure": { + "map": { + "OWNED_OUTRIGHT": "owner", + "OWNED_WITH_MORTGAGE": "owner", + "RENT_PRIVATELY": "private_rent", + "RENT_FROM_COUNCIL": "social", + "RENT_FROM_HA": "social" + }, + "gas_kwh": {"owner": 12339, "private_rent": 10183, "social": 8357}, + "electricity_kwh": {"owner": 3465, "private_rent": 3261, "social": 2896} + }, + "accommodation": { + "map": { + "HOUSE_DETACHED": "detached", + "HOUSE_SEMI_DETACHED": "semi", + "HOUSE_TERRACED": "terraced", + "FLAT": "flat", + "MOBILE": "other" + }, + "gas_kwh": {"detached": 15518, "semi": 11715, "terraced": 10365, "flat": 7058, "other": 11303}, + "electricity_kwh": {"detached": 4346, "semi": 3338, "terraced": 3096, "flat": 2896, "other": 3327} + }, + "region": { + "gas_kwh": { + "NORTH_EAST": 11278, + "NORTH_WEST": 11111, + "YORKSHIRE": 11552, + "EAST_MIDLANDS": 11234, + "WEST_MIDLANDS": 11485, + "EAST_OF_ENGLAND": 11334, + "LONDON": 12335, + "SOUTH_EAST": 11555, + "SOUTH_WEST": 9811, + "WALES": 10558 + }, + "electricity_kwh": { + "NORTH_EAST": 2822, + "NORTH_WEST": 3211, + "YORKSHIRE": 3114, + "EAST_MIDLANDS": 3266, + "WEST_MIDLANDS": 3332, + "EAST_OF_ENGLAND": 3543, + "LONDON": 3275, + "SOUTH_EAST": 3568, + "SOUTH_WEST": 3537, + "WALES": 3151 + } + }, + "chronicle": [ + "E6 committed NEED/Ofgem anchors for LCFS energy raking." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/nhs_consumption_by_age_gender.json b/packages/microcosm-build/src/microcosm/build/uk/nhs_consumption_by_age_gender.json new file mode 100644 index 00000000..cdb0ef1e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/nhs_consumption_by_age_gender.json @@ -0,0 +1,1778 @@ +{ + "version": 1, + "country": "uk", + "source": { + "artifact": "nhs_consumption_by_age_gender.csv", + "citation": "Incumbent public NHS age-gender activity/cost table carried into microcosm#682.", + "sdc_treatment": "Public aggregate table; no licensed donor microdata." + }, + "rows": [ + { + "Service": "AE", + "Gender": "Female", + "Age group": "0 years", + "Metric": "Activity Count", + "Total": 212079.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "01-04 years", + "Metric": "Activity Count", + "Total": 505885.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "05-09 years", + "Metric": "Activity Count", + "Total": 347081.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "10-14 years", + "Metric": "Activity Count", + "Total": 371563.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "15-19 years", + "Metric": "Activity Count", + "Total": 483626.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "20-24 years", + "Metric": "Activity Count", + "Total": 617087.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "25-29 years", + "Metric": "Activity Count", + "Total": 611997.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "30-34 years", + "Metric": "Activity Count", + "Total": 560654.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "35-39 years", + "Metric": "Activity Count", + "Total": 480036.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "40-44 years", + "Metric": "Activity Count", + "Total": 402489.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "45-49 years", + "Metric": "Activity Count", + "Total": 421889.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "50-54 years", + "Metric": "Activity Count", + "Total": 448613.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "55-59 years", + "Metric": "Activity Count", + "Total": 418718.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "60-64 years", + "Metric": "Activity Count", + "Total": 357017.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "65-69 years", + "Metric": "Activity Count", + "Total": 331175.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "70-74 years", + "Metric": "Activity Count", + "Total": 384744.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "75-79 years", + "Metric": "Activity Count", + "Total": 369570.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "80-84 years", + "Metric": "Activity Count", + "Total": 378626.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "85-89 years", + "Metric": "Activity Count", + "Total": 327708.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "90-94 years", + "Metric": "Activity Count", + "Total": 195096.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "95 years or older", + "Metric": "Activity Count", + "Total": 73381.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "0 years", + "Metric": "Activity Count", + "Total": 266383.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "01-04 years", + "Metric": "Activity Count", + "Total": 651982.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "05-09 years", + "Metric": "Activity Count", + "Total": 411892.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "10-14 years", + "Metric": "Activity Count", + "Total": 443253.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "15-19 years", + "Metric": "Activity Count", + "Total": 409286.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "20-24 years", + "Metric": "Activity Count", + "Total": 499314.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "25-29 years", + "Metric": "Activity Count", + "Total": 520689.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "30-34 years", + "Metric": "Activity Count", + "Total": 499270.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "35-39 years", + "Metric": "Activity Count", + "Total": 452672.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "40-44 years", + "Metric": "Activity Count", + "Total": 401018.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "45-49 years", + "Metric": "Activity Count", + "Total": 421480.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "50-54 years", + "Metric": "Activity Count", + "Total": 437104.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "55-59 years", + "Metric": "Activity Count", + "Total": 415672.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "60-64 years", + "Metric": "Activity Count", + "Total": 363507.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "65-69 years", + "Metric": "Activity Count", + "Total": 334916.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "70-74 years", + "Metric": "Activity Count", + "Total": 376324.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "75-79 years", + "Metric": "Activity Count", + "Total": 341874.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "80-84 years", + "Metric": "Activity Count", + "Total": 323601.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "85-89 years", + "Metric": "Activity Count", + "Total": 242458.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "90-94 years", + "Metric": "Activity Count", + "Total": 114545.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "95 years or older", + "Metric": "Activity Count", + "Total": 31993.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "0 years", + "Metric": "Total Cost", + "Total": 30148770.7 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "01-04 years", + "Metric": "Total Cost", + "Total": 73303426.7 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "05-09 years", + "Metric": "Total Cost", + "Total": 49003152.2 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "10-14 years", + "Metric": "Total Cost", + "Total": 55261204.8 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "15-19 years", + "Metric": "Total Cost", + "Total": 79355311.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "20-24 years", + "Metric": "Total Cost", + "Total": 101677604.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "25-29 years", + "Metric": "Total Cost", + "Total": 103591677.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "30-34 years", + "Metric": "Total Cost", + "Total": 96833952.2 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "35-39 years", + "Metric": "Total Cost", + "Total": 85119455.3 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "40-44 years", + "Metric": "Total Cost", + "Total": 73111612.4 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "45-49 years", + "Metric": "Total Cost", + "Total": 79541655.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "50-54 years", + "Metric": "Total Cost", + "Total": 86739022.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "55-59 years", + "Metric": "Total Cost", + "Total": 83554003.1 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "60-64 years", + "Metric": "Total Cost", + "Total": 74921474.2 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "65-69 years", + "Metric": "Total Cost", + "Total": 74205741.3 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "70-74 years", + "Metric": "Total Cost", + "Total": 92209066.3 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "75-79 years", + "Metric": "Total Cost", + "Total": 94937123.5 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "80-84 years", + "Metric": "Total Cost", + "Total": 104581887.0 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "85-89 years", + "Metric": "Total Cost", + "Total": 96823888.9 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "90-94 years", + "Metric": "Total Cost", + "Total": 60486434.5 + }, + { + "Service": "AE", + "Gender": "Female", + "Age group": "95 years or older", + "Metric": "Total Cost", + "Total": 23345611.4 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "0 years", + "Metric": "Total Cost", + "Total": 38597370.1 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "01-04 years", + "Metric": "Total Cost", + "Total": 95191584.1 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "05-09 years", + "Metric": "Total Cost", + "Total": 58855565.3 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "10-14 years", + "Metric": "Total Cost", + "Total": 67069862.1 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "15-19 years", + "Metric": "Total Cost", + "Total": 68299159.1 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "20-24 years", + "Metric": "Total Cost", + "Total": 83483710.5 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "25-29 years", + "Metric": "Total Cost", + "Total": 88653812.2 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "30-34 years", + "Metric": "Total Cost", + "Total": 87093334.1 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "35-39 years", + "Metric": "Total Cost", + "Total": 80745741.5 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "40-44 years", + "Metric": "Total Cost", + "Total": 74135948.3 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "45-49 years", + "Metric": "Total Cost", + "Total": 80953801.4 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "50-54 years", + "Metric": "Total Cost", + "Total": 88215789.8 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "55-59 years", + "Metric": "Total Cost", + "Total": 86921818.8 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "60-64 years", + "Metric": "Total Cost", + "Total": 80659276.5 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "65-69 years", + "Metric": "Total Cost", + "Total": 79633763.7 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "70-74 years", + "Metric": "Total Cost", + "Total": 94195978.6 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "75-79 years", + "Metric": "Total Cost", + "Total": 90252429.9 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "80-84 years", + "Metric": "Total Cost", + "Total": 90053985.2 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "85-89 years", + "Metric": "Total Cost", + "Total": 70865034.3 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "90-94 years", + "Metric": "Total Cost", + "Total": 34791109.0 + }, + { + "Service": "AE", + "Gender": "Male", + "Age group": "95 years or older", + "Metric": "Total Cost", + "Total": 9951869.71 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "0 years", + "Metric": "Activity Count", + "Total": 179975.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "01-04 years", + "Metric": "Activity Count", + "Total": 166080.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "05-09 years", + "Metric": "Activity Count", + "Total": 119129.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "10-14 years", + "Metric": "Activity Count", + "Total": 116854.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "15-19 years", + "Metric": "Activity Count", + "Total": 220700.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "20-24 years", + "Metric": "Activity Count", + "Total": 417022.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "25-29 years", + "Metric": "Activity Count", + "Total": 604916.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "30-34 years", + "Metric": "Activity Count", + "Total": 659361.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "35-39 years", + "Metric": "Activity Count", + "Total": 521586.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "40-44 years", + "Metric": "Activity Count", + "Total": 373350.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "45-49 years", + "Metric": "Activity Count", + "Total": 418570.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "50-54 years", + "Metric": "Activity Count", + "Total": 504566.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "55-59 years", + "Metric": "Activity Count", + "Total": 591516.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "60-64 years", + "Metric": "Activity Count", + "Total": 546883.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "65-69 years", + "Metric": "Activity Count", + "Total": 593173.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "70-74 years", + "Metric": "Activity Count", + "Total": 745088.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "75-79 years", + "Metric": "Activity Count", + "Total": 716489.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "80-84 years", + "Metric": "Activity Count", + "Total": 701215.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "85-89 years", + "Metric": "Activity Count", + "Total": 575906.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "90-94 years", + "Metric": "Activity Count", + "Total": 326043.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "95 years or older", + "Metric": "Activity Count", + "Total": 116671.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "0 years", + "Metric": "Activity Count", + "Total": 231010.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "01-04 years", + "Metric": "Activity Count", + "Total": 227129.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "05-09 years", + "Metric": "Activity Count", + "Total": 150286.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "10-14 years", + "Metric": "Activity Count", + "Total": 124152.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "15-19 years", + "Metric": "Activity Count", + "Total": 138695.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "20-24 years", + "Metric": "Activity Count", + "Total": 165402.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "25-29 years", + "Metric": "Activity Count", + "Total": 201294.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "30-34 years", + "Metric": "Activity Count", + "Total": 228088.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "35-39 years", + "Metric": "Activity Count", + "Total": 250474.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "40-44 years", + "Metric": "Activity Count", + "Total": 265449.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "45-49 years", + "Metric": "Activity Count", + "Total": 345386.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "50-54 years", + "Metric": "Activity Count", + "Total": 449294.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "55-59 years", + "Metric": "Activity Count", + "Total": 597189.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "60-64 years", + "Metric": "Activity Count", + "Total": 591735.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "65-69 years", + "Metric": "Activity Count", + "Total": 671518.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "70-74 years", + "Metric": "Activity Count", + "Total": 833479.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "75-79 years", + "Metric": "Activity Count", + "Total": 762975.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "80-84 years", + "Metric": "Activity Count", + "Total": 679676.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "85-89 years", + "Metric": "Activity Count", + "Total": 476440.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "90-94 years", + "Metric": "Activity Count", + "Total": 211073.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "95 years or older", + "Metric": "Activity Count", + "Total": 52974.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "0 years", + "Metric": "Total Cost", + "Total": 249777365.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "01-04 years", + "Metric": "Total Cost", + "Total": 217088968.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "05-09 years", + "Metric": "Total Cost", + "Total": 177517727.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "10-14 years", + "Metric": "Total Cost", + "Total": 216733472.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "15-19 years", + "Metric": "Total Cost", + "Total": 320270378.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "20-24 years", + "Metric": "Total Cost", + "Total": 601289475.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "25-29 years", + "Metric": "Total Cost", + "Total": 972425428.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "30-34 years", + "Metric": "Total Cost", + "Total": 1135682570.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "35-39 years", + "Metric": "Total Cost", + "Total": 867881313.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "40-44 years", + "Metric": "Total Cost", + "Total": 553715118.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "45-49 years", + "Metric": "Total Cost", + "Total": 603048703.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "50-54 years", + "Metric": "Total Cost", + "Total": 754837523.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "55-59 years", + "Metric": "Total Cost", + "Total": 847924810.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "60-64 years", + "Metric": "Total Cost", + "Total": 877315571.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "65-69 years", + "Metric": "Total Cost", + "Total": 991517578.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "70-74 years", + "Metric": "Total Cost", + "Total": 1281142417.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "75-79 years", + "Metric": "Total Cost", + "Total": 1289140495.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "80-84 years", + "Metric": "Total Cost", + "Total": 1349578327.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "85-89 years", + "Metric": "Total Cost", + "Total": 1185018283.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "90-94 years", + "Metric": "Total Cost", + "Total": 704541248.0 + }, + { + "Service": "APC", + "Gender": "Female", + "Age group": "95 years or older", + "Metric": "Total Cost", + "Total": 255392422.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "0 years", + "Metric": "Total Cost", + "Total": 324223190.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "01-04 years", + "Metric": "Total Cost", + "Total": 294910749.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "05-09 years", + "Metric": "Total Cost", + "Total": 227643007.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "10-14 years", + "Metric": "Total Cost", + "Total": 227920739.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "15-19 years", + "Metric": "Total Cost", + "Total": 236394221.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "20-24 years", + "Metric": "Total Cost", + "Total": 244330615.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "25-29 years", + "Metric": "Total Cost", + "Total": 288740865.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "30-34 years", + "Metric": "Total Cost", + "Total": 316268020.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "35-39 years", + "Metric": "Total Cost", + "Total": 354154798.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "40-44 years", + "Metric": "Total Cost", + "Total": 387154627.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "45-49 years", + "Metric": "Total Cost", + "Total": 524862509.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "50-54 years", + "Metric": "Total Cost", + "Total": 723066778.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "55-59 years", + "Metric": "Total Cost", + "Total": 932563200.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "60-64 years", + "Metric": "Total Cost", + "Total": 1031083117.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "65-69 years", + "Metric": "Total Cost", + "Total": 1188794886.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "70-74 years", + "Metric": "Total Cost", + "Total": 1482408081.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "75-79 years", + "Metric": "Total Cost", + "Total": 1378990921.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "80-84 years", + "Metric": "Total Cost", + "Total": 1274704997.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "85-89 years", + "Metric": "Total Cost", + "Total": 940697209.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "90-94 years", + "Metric": "Total Cost", + "Total": 432169323.0 + }, + { + "Service": "APC", + "Gender": "Male", + "Age group": "95 years or older", + "Metric": "Total Cost", + "Total": 109644790.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "0 years", + "Metric": "Activity Count", + "Total": 325441.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "01-04 years", + "Metric": "Activity Count", + "Total": 719275.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "05-09 years", + "Metric": "Activity Count", + "Total": 897665.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "10-14 years", + "Metric": "Activity Count", + "Total": 969630.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "15-19 years", + "Metric": "Activity Count", + "Total": 1156991.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "20-24 years", + "Metric": "Activity Count", + "Total": 1755334.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "25-29 years", + "Metric": "Activity Count", + "Total": 2950592.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "30-34 years", + "Metric": "Activity Count", + "Total": 3530495.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "35-39 years", + "Metric": "Activity Count", + "Total": 2899647.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "40-44 years", + "Metric": "Activity Count", + "Total": 2043206.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "45-49 years", + "Metric": "Activity Count", + "Total": 2228921.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "50-54 years", + "Metric": "Activity Count", + "Total": 2686270.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "55-59 years", + "Metric": "Activity Count", + "Total": 2823953.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "60-64 years", + "Metric": "Activity Count", + "Total": 2693294.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "65-69 years", + "Metric": "Activity Count", + "Total": 2751613.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "70-74 years", + "Metric": "Activity Count", + "Total": 3163861.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "75-79 years", + "Metric": "Activity Count", + "Total": 2672250.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "80-84 years", + "Metric": "Activity Count", + "Total": 2162918.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "85-89 years", + "Metric": "Activity Count", + "Total": 1348139.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "90-94 years", + "Metric": "Activity Count", + "Total": 532412.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "95 years or older", + "Metric": "Activity Count", + "Total": 137316.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "0 years", + "Metric": "Activity Count", + "Total": 401692.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "01-04 years", + "Metric": "Activity Count", + "Total": 947015.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "05-09 years", + "Metric": "Activity Count", + "Total": 1079069.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "10-14 years", + "Metric": "Activity Count", + "Total": 1033931.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "15-19 years", + "Metric": "Activity Count", + "Total": 896856.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "20-24 years", + "Metric": "Activity Count", + "Total": 709694.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "25-29 years", + "Metric": "Activity Count", + "Total": 851016.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "30-34 years", + "Metric": "Activity Count", + "Total": 956355.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "35-39 years", + "Metric": "Activity Count", + "Total": 1046825.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "40-44 years", + "Metric": "Activity Count", + "Total": 1121153.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "45-49 years", + "Metric": "Activity Count", + "Total": 1465178.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "50-54 years", + "Metric": "Activity Count", + "Total": 1886549.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "55-59 years", + "Metric": "Activity Count", + "Total": 2283433.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "60-64 years", + "Metric": "Activity Count", + "Total": 2456193.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "65-69 years", + "Metric": "Activity Count", + "Total": 2699217.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "70-74 years", + "Metric": "Activity Count", + "Total": 3186785.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "75-79 years", + "Metric": "Activity Count", + "Total": 2687074.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "80-84 years", + "Metric": "Activity Count", + "Total": 2079196.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "85-89 years", + "Metric": "Activity Count", + "Total": 1161187.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "90-94 years", + "Metric": "Activity Count", + "Total": 372921.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "95 years or older", + "Metric": "Activity Count", + "Total": 68484.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "0 years", + "Metric": "Total Cost", + "Total": 58872702.9 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "01-04 years", + "Metric": "Total Cost", + "Total": 121444400.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "05-09 years", + "Metric": "Total Cost", + "Total": 146984131.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "10-14 years", + "Metric": "Total Cost", + "Total": 164019001.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "15-19 years", + "Metric": "Total Cost", + "Total": 179659867.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "20-24 years", + "Metric": "Total Cost", + "Total": 242131906.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "25-29 years", + "Metric": "Total Cost", + "Total": 407477772.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "30-34 years", + "Metric": "Total Cost", + "Total": 488197443.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "35-39 years", + "Metric": "Total Cost", + "Total": 409246386.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "40-44 years", + "Metric": "Total Cost", + "Total": 298827477.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "45-49 years", + "Metric": "Total Cost", + "Total": 325925802.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "50-54 years", + "Metric": "Total Cost", + "Total": 388913382.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "55-59 years", + "Metric": "Total Cost", + "Total": 402521375.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "60-64 years", + "Metric": "Total Cost", + "Total": 380833755.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "65-69 years", + "Metric": "Total Cost", + "Total": 386525748.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "70-74 years", + "Metric": "Total Cost", + "Total": 440767051.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "75-79 years", + "Metric": "Total Cost", + "Total": 367629876.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "80-84 years", + "Metric": "Total Cost", + "Total": 293296393.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "85-89 years", + "Metric": "Total Cost", + "Total": 177694021.0 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "90-94 years", + "Metric": "Total Cost", + "Total": 68202176.6 + }, + { + "Service": "OP", + "Gender": "Female", + "Age group": "95 years or older", + "Metric": "Total Cost", + "Total": 16859358.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "0 years", + "Metric": "Total Cost", + "Total": 74301400.9 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "01-04 years", + "Metric": "Total Cost", + "Total": 163252103.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "05-09 years", + "Metric": "Total Cost", + "Total": 183189795.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "10-14 years", + "Metric": "Total Cost", + "Total": 178808739.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "15-19 years", + "Metric": "Total Cost", + "Total": 141828786.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "20-24 years", + "Metric": "Total Cost", + "Total": 99340346.1 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "25-29 years", + "Metric": "Total Cost", + "Total": 118503476.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "30-34 years", + "Metric": "Total Cost", + "Total": 135535727.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "35-39 years", + "Metric": "Total Cost", + "Total": 149377324.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "40-44 years", + "Metric": "Total Cost", + "Total": 160115100.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "45-49 years", + "Metric": "Total Cost", + "Total": 209730217.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "50-54 years", + "Metric": "Total Cost", + "Total": 271532785.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "55-59 years", + "Metric": "Total Cost", + "Total": 325197136.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "60-64 years", + "Metric": "Total Cost", + "Total": 347698137.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "65-69 years", + "Metric": "Total Cost", + "Total": 379726112.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "70-74 years", + "Metric": "Total Cost", + "Total": 442751873.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "75-79 years", + "Metric": "Total Cost", + "Total": 370982792.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "80-84 years", + "Metric": "Total Cost", + "Total": 282006082.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "85-89 years", + "Metric": "Total Cost", + "Total": 154559700.0 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "90-94 years", + "Metric": "Total Cost", + "Total": 48058882.9 + }, + { + "Service": "OP", + "Gender": "Male", + "Age group": "95 years or older", + "Metric": "Total Cost", + "Total": 8573041.95 + } + ], + "chronicle": [ + "Converted from the public incumbent CSV fixture for E6 NHS allocation." + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 6a12fa68..ffd194d0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -458,6 +458,53 @@ "weight_source": "household_weight" }, "family_coverage": { + "etb_services": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "output_weight_kind": "importance", + "outputs": [ + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending", + "rail_usage", + "a_and_e_visits", + "admitted_patient_visits", + "outpatient_visits", + "nhs_a_and_e_spending", + "nhs_admitted_patient_spending", + "nhs_outpatient_spending" + ], + "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", + "rewrites": [], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", + "source_vintages": { + "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", + "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" + }, + "stage": "etb_services", + "status": "required_at_build" + }, + "etb_vat": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "output_weight_kind": "importance", + "outputs": [ + "full_rate_vat_expenditure_rate" + ], + "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", + "rewrites": [], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", + "source_vintages": { + "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", + "survey": "Effects of Taxes and Benefits 1977-2024" + }, + "stage": "etb_vat", + "status": "required_at_build" + }, "hmrc_cgt_gains": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", @@ -563,6 +610,43 @@ "stage": "hmrc_spi_income", "status": "required_at_build" }, + "lcfs_consumption": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "output_weight_kind": "importance", + "outputs": [ + "food_and_non_alcoholic_beverages_consumption", + "alcohol_and_tobacco_consumption", + "clothing_and_footwear_consumption", + "housing_water_and_electricity_consumption", + "household_furnishings_consumption", + "health_consumption", + "transport_consumption", + "communication_consumption", + "recreation_consumption", + "education_consumption", + "restaurants_and_hotels_consumption", + "miscellaneous_consumption", + "petrol_spending", + "diesel_spending", + "bus_fare_spending", + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption", + "has_fuel_consumption" + ], + "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", + "rewrites": [], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", + "source_vintages": { + "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", + "survey": "Living Costs and Food Survey 2023-24" + }, + "stage": "lcfs_consumption", + "status": "required_at_build" + }, "regional_property_uprating": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", @@ -575,7 +659,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c8eba47691c3e3b4abe833b2c2816acfc091c7039df5131e3768eb4fce393eb3", + "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -606,7 +690,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c8eba47691c3e3b4abe833b2c2816acfc091c7039df5131e3768eb4fce393eb3", + "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 31f8d092..92932406 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -1009,6 +1009,444 @@ ], "notes": "Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior." }, + { + "stage": "lcfs_consumption", + "survey": "Living Costs and Food Survey 2023-24", + "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", + "grain": "household", + "base_candidate": { + "filename": "populace_uk_2023.h5", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "tier": "frs" + }, + "artifacts": [ + { + "role": "lcfs_household_tab", + "kind": "private_microdata", + "format": "tab", + "vintage": "2023_24", + "locator": "dvhh_ukanon_v2_2023.tab", + "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", + "size_bytes": 22812887, + "runtime_sha256_required": true + }, + { + "role": "lcfs_person_tab", + "kind": "private_microdata", + "format": "tab", + "vintage": "2023_24", + "locator": "dvper_ukanon_202324_2023.tab", + "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", + "size_bytes": 6545146, + "runtime_sha256_required": true + }, + { + "role": "was_bridge_donor", + "kind": "private_microdata", + "format": "tab", + "vintage": "2018_20", + "locator": "was_round_8_hhold_eul_may_2025_230525.tab", + "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", + "size_bytes": 39073613, + "runtime_sha256_required": true + }, + { + "role": "need_energy_targets", + "resource": "need_energy_targets.json", + "kind": "public_aggregate_reference", + "format": "json" + }, + { + "role": "lcfs_consumption_anchors", + "resource": "lcfs_consumption_anchors.json", + "kind": "public_aggregate_reference", + "format": "json" + } + ], + "operations": [ + { + "kind": "derive", + "lowercase_columns": true, + "annualization_weeks": 52.17857142857143, + "donor_weight": "weighta * 1000", + "lossy_mappings": [ + "LCFS tenure 4 and 8 -> RENT_PRIVATELY", + "LCFS accommodation 4 and 5 -> FLAT" + ], + "logged_dropna_row_count": true + }, + { + "kind": "iterative_proportional_fit", + "columns": [ + "electricity_consumption", + "gas_consumption" + ], + "margins": [ + "gross_income_band" + ], + "iterations": 1, + "weighted": false + }, + { + "kind": "bridge_donor_column_via_qrf", + "source": "was_wealth", + "target": "has_fuel_consumption", + "predictors": [ + "household_net_income", + "num_adults", + "num_children", + "private_pension_income", + "employment_income", + "self_employment_income", + "region" + ], + "weights": "explicit", + "seed": 0, + "n_estimators": 100 + }, + { + "kind": "assign_binary_from_rate", + "target": "has_fuel_consumption", + "rate_key": "nts_ice_share", + "condition": "num_vehicles > 0", + "seed": 0 + }, + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "is_adult", + "is_child", + "employment_income", + "self_employment_income", + "private_pension_income", + "hbai_household_net_income" + ] + }, + { + "kind": "fit_weighted_qrf_chain", + "predictors": [ + "is_adult", + "is_child", + "region", + "employment_income", + "self_employment_income", + "private_pension_income", + "hbai_household_net_income", + "tenure_type", + "accommodation_type", + "has_fuel_consumption" + ], + "targets": [ + "food_and_non_alcoholic_beverages_consumption", + "alcohol_and_tobacco_consumption", + "clothing_and_footwear_consumption", + "housing_water_and_electricity_consumption", + "household_furnishings_consumption", + "health_consumption", + "transport_consumption", + "communication_consumption", + "recreation_consumption", + "education_consumption", + "restaurants_and_hotels_consumption", + "miscellaneous_consumption", + "petrol_spending", + "diesel_spending", + "bus_fare_spending", + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption" + ], + "categorical_predictors": [ + "region", + "tenure_type", + "accommodation_type" + ], + "weights": "explicit", + "seed": 0, + "n_estimators": 100 + }, + { + "kind": "support_clip", + "range": "donor_realized", + "exempt": [ + "electricity_consumption", + "gas_consumption", + "domestic_energy_consumption" + ] + }, + { + "kind": "iterative_proportional_fit", + "columns": [ + "electricity_consumption", + "gas_consumption" + ], + "margins": [ + "income", + "tenure", + "accommodation", + "region" + ], + "iterations": 50, + "weighted": true + }, + { + "kind": "fold_into", + "output": "domestic_energy_consumption", + "inputs": [ + "electricity_consumption", + "gas_consumption" + ], + "drop_inputs": false + }, + { + "kind": "zero_when_false", + "columns": [ + "petrol_spending", + "diesel_spending" + ], + "condition": "has_fuel_consumption == false" + } + ], + "outputs": [ + "food_and_non_alcoholic_beverages_consumption", + "alcohol_and_tobacco_consumption", + "clothing_and_footwear_consumption", + "housing_water_and_electricity_consumption", + "household_furnishings_consumption", + "health_consumption", + "transport_consumption", + "communication_consumption", + "recreation_consumption", + "education_consumption", + "restaurants_and_hotels_consumption", + "miscellaneous_consumption", + "petrol_spending", + "diesel_spending", + "bus_fare_spending", + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption", + "has_fuel_consumption" + ], + "nonnegative_outputs": [ + "food_and_non_alcoholic_beverages_consumption", + "alcohol_and_tobacco_consumption", + "clothing_and_footwear_consumption", + "housing_water_and_electricity_consumption", + "household_furnishings_consumption", + "health_consumption", + "transport_consumption", + "communication_consumption", + "recreation_consumption", + "education_consumption", + "restaurants_and_hotels_consumption", + "miscellaneous_consumption", + "petrol_spending", + "diesel_spending", + "bus_fare_spending", + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption", + "has_fuel_consumption" + ], + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and identity-keyed seed-0 fuel flags. Energy support clipping is exempt because NEED raking governs those columns. Donor uprating is identity at this vintage (LCFS survey year equals the 2023 build year), so the incumbent's CPI and fuel litre-proxy donor uprating is deliberately not declared here; the machinery lands with the FRS 2024-25 refresh (microcosm#687), where a donor/build year gap first exists. The DESNZ pump-price anchors stay committed as the cited litre-proxy denominators for that refresh." + }, + { + "stage": "etb_vat", + "survey": "Effects of Taxes and Benefits 1977-2024", + "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", + "grain": "household", + "base_candidate": { + "filename": "populace_uk_2023.h5", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "tier": "frs" + }, + "artifacts": [ + { + "role": "etb_household_tab", + "kind": "private_microdata", + "format": "tab", + "vintage": "1977_24", + "locator": "householdv2_1977-2024.tab", + "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "size_bytes": 216967663, + "runtime_sha256_required": true + }, + { + "role": "etb_policy_anchors", + "resource": "etb_policy_anchors.json", + "kind": "public_parameter_reference", + "format": "json" + } + ], + "operations": [ + { + "kind": "derive", + "year": 2023, + "annualization_weeks": 52, + "standard_rate": 0.2, + "reduced_rate_share": 0.025, + "fail_loud_on_missing_rate": true + }, + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "is_adult", + "is_child", + "is_SP_age", + "household_net_income" + ] + }, + { + "kind": "fit_weighted_qrf", + "predictors": [ + "is_adult", + "is_child", + "is_SP_age", + "household_net_income" + ], + "targets": [ + "full_rate_vat_expenditure_rate" + ], + "weights": "explicit", + "seed": 0, + "n_estimators": 100 + }, + { + "kind": "support_clip", + "range": "donor_realized" + } + ], + "outputs": [ + "full_rate_vat_expenditure_rate" + ], + "nonnegative_outputs": [], + "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the support gate is the guard - the net_financial_wealth precedent from was_wealth." + }, + { + "stage": "etb_services", + "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table", + "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", + "grain": "household+person", + "base_candidate": { + "filename": "populace_uk_2023.h5", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "tier": "frs" + }, + "artifacts": [ + { + "role": "etb_household_tab", + "kind": "private_microdata", + "format": "tab", + "vintage": "1977_24", + "locator": "householdv2_1977-2024.tab", + "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "size_bytes": 216967663, + "runtime_sha256_required": true + }, + { + "role": "nhs_consumption_by_age_gender", + "resource": "nhs_consumption_by_age_gender.json", + "kind": "public_aggregate_reference", + "format": "json" + }, + { + "role": "etb_services_anchors", + "resource": "etb_services_anchors.json", + "kind": "public_parameter_reference", + "format": "json" + } + ], + "operations": [ + { + "kind": "derive", + "year": "max", + "annualization_weeks": 52 + }, + { + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "is_adult", + "is_child", + "is_SP_age", + "count_primary_education", + "count_secondary_education", + "count_further_education", + "dla", + "pip", + "hbai_household_net_income" + ] + }, + { + "kind": "fit_weighted_qrf_chain", + "predictors": [ + "is_adult", + "is_child", + "is_SP_age", + "count_primary_education", + "count_secondary_education", + "count_further_education", + "dla", + "pip", + "hbai_household_net_income" + ], + "targets": [ + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending" + ], + "weights": "explicit", + "seed": 0, + "n_estimators": 100 + }, + { + "kind": "support_clip", + "range": "donor_realized" + }, + { + "kind": "compute_ratio", + "output": "rail_usage", + "numerator": "rail_subsidy_spending", + "denominator_resource": "etb_services_anchors.json", + "denominator_key": "rail_fare_index_2023" + }, + { + "kind": "allocate_per_capita_from_cell_table", + "resource": "nhs_consumption_by_age_gender.json", + "budget_resource": "etb_services_anchors.json", + "age_bands": "half_open", + "top_band_fold_in": "85+" + } + ], + "outputs": [ + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending", + "rail_usage", + "a_and_e_visits", + "admitted_patient_visits", + "outpatient_visits", + "nhs_a_and_e_spending", + "nhs_admitted_patient_spending", + "nhs_outpatient_spending" + ], + "nonnegative_outputs": [ + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending", + "rail_usage", + "a_and_e_visits", + "admitted_patient_visits", + "outpatient_visits", + "nhs_a_and_e_spending", + "nhs_admitted_patient_spending", + "nhs_outpatient_spending" + ], + "notes": "Ports ETB public-services QRF at household grain, computes rail_usage from the 2023 fare index, and allocates NHS visits/spending to persons using the signed half-open age-band and 85+ fold-in fixes. The year-max donor filter resolves to 2023 on the pinned tab (the file labels financial year ending 2024 as year 2023), so the services training year coincides with the VAT training year and the fare-index year - the incumbent's apparent three-way year mismatch is vacuous on this vintage." + }, { "stage": "frs_hmrc_spine_leaves", "survey": "Family Resources Survey 2023-24", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index 3833da3f..b4f0330d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -45,6 +45,7 @@ ) from microcosm.build.gates import ( GateResult, + aggregate_admin_gate, enum_domain_gate, nonnegative_columns_gate, support_gate, @@ -89,6 +90,7 @@ uk_qrf_tail_concentration_columns, uk_qrf_tail_concentration_gate, ) +from microcosm.calibrate.registry import TargetSpec __all__ = [ "UK_GATE_REGISTRY", @@ -268,22 +270,38 @@ def _evaluate_brma_enum_domain( def _evaluate_support( context: EvidenceContext, parameters: Mapping[str, Any] ) -> GateResult: - resource_name = parameters.get("support_bounds_resource") - if resource_name != "was_wealth_support_bounds.json": + resource_names = parameters.get("support_bounds_resources") + if resource_names is None: + single = parameters.get("support_bounds_resource") + resource_names = [single] if isinstance(single, str) else None + if not isinstance(resource_names, (list, tuple)) or not all( + isinstance(name, str) for name in resource_names + ): raise ValueError( - "uk_support must declare support_bounds_resource " - "'was_wealth_support_bounds.json'." + "uk_support must declare support_bounds_resources as a list of " + "support-bound resource filenames." ) - resource = json.loads( - files("microcosm.build.uk").joinpath(str(resource_name)).read_text() - ) - raw_bounds = resource.get("bounds") - if not isinstance(raw_bounds, Mapping): - raise ValueError("WAS wealth support-bounds resource is missing bounds.") - donor_ranges = { - str(column): (float(bounds[0]), float(bounds[1])) - for column, bounds in raw_bounds.items() + allowed = { + "was_wealth_support_bounds.json", + "lcfs_consumption_support_bounds.json", + "etb_vat_support_bounds.json", + "etb_services_support_bounds.json", } + if set(resource_names) - allowed: + raise ValueError( + "uk_support declared unknown support-bound resource(s): " + f"{sorted(set(resource_names) - allowed)}." + ) + donor_ranges: dict[str, tuple[float, float]] = {} + for resource_name in resource_names: + resource = json.loads( + files("microcosm.build.uk").joinpath(str(resource_name)).read_text() + ) + raw_bounds = resource.get("bounds") + if not isinstance(raw_bounds, Mapping): + raise ValueError(f"{resource_name} is missing bounds.") + for column, bounds in raw_bounds.items(): + donor_ranges[str(column)] = (float(bounds[0]), float(bounds[1])) values: dict[str, np.ndarray] = {} for entity in context.frame.entities: table = context.frame.table(entity) @@ -293,6 +311,37 @@ def _evaluate_support( return support_gate(values, donor_ranges) +def _evaluate_aggregate_admin( + context: EvidenceContext, parameters: Mapping[str, Any] +) -> GateResult: + aggregate_artifact = context.artifacts["aggregate_admin"] + if not isinstance(aggregate_artifact, Mapping): + raise ValueError("aggregate_admin artifact must be a mapping.") + anchors_payload = parameters.get("anchors") + if not isinstance(anchors_payload, (list, tuple)): + raise ValueError("aggregate_admin requires an anchors list.") + anchors = tuple( + TargetSpec( + name=str(anchor["name"]), + entity=str(anchor.get("entity", "household")), + value=float(anchor["value"]), + measure=str(anchor.get("measure", anchor["name"])), + period=str(anchor.get("period", "2023")), + source=str(anchor["source"]), + family=str(anchor.get("family", "uk_admin")), + tolerance=( + None if anchor.get("tolerance") is None else float(anchor["tolerance"]) + ), + ) + for anchor in anchors_payload + ) + return aggregate_admin_gate( + {str(key): float(value) for key, value in aggregate_artifact.items()}, + anchors, + default_rtol=float(parameters.get("default_rtol", 0.5)), + ) + + def _stage_names_evidence( context: EvidenceContext, parameters: Mapping[str, Any] ) -> object: @@ -698,7 +747,17 @@ def _evaluate_tail_concentration( "support": UKGateBinding( name="support", evaluator=_evaluate_support, - parameter_keys=frozenset({"support_bounds_resource"}), + parameter_keys=frozenset( + {"support_bounds_resource", "support_bounds_resources"} + ), + ), + "aggregate_admin": UKGateBinding( + name="aggregate_admin", + evaluator=_evaluate_aggregate_admin, + parameter_keys=frozenset({"anchors", "default_rtol"}), + artifact_keys=frozenset({"aggregate_admin"}), + needs_frame=False, + legacy_name="aggregate_vs_admin", ), "degenerate_release_surface": UKGateBinding( name="degenerate_release_surface", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py new file mode 100644 index 00000000..8b3256c3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -0,0 +1,376 @@ +"""UK ETB services and NHS allocation stage.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.gates import FitWeightRecord +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.frame import Frame +from microcosm.frame.rules import assert_rules_engine_country + +ETB_SERVICES_YEAR = 2024 +ETB_SERVICES_WEEKS_IN_YEAR = 52 +RAIL_FARE_INDEX_2023 = 1.110 +NHS_BUDGET_2025_26 = 202_000_000_000.0 +UK_ETB_SERVICES_PREDICTORS = ( + "is_adult", + "is_child", + "is_SP_age", + "count_primary_education", + "count_secondary_education", + "count_further_education", + "dla", + "pip", + "hbai_household_net_income", +) +UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS = ( + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending", + "rail_usage", +) +UK_NHS_OUTPUT_COLUMNS = ( + "a_and_e_visits", + "admitted_patient_visits", + "outpatient_visits", + "nhs_a_and_e_spending", + "nhs_admitted_patient_spending", + "nhs_outpatient_spending", +) +UK_ETB_SERVICES_OUTPUT_COLUMNS = ( + *UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS, + *UK_NHS_OUTPUT_COLUMNS, +) +UK_ETB_SERVICES_NONNEGATIVE_OUTPUT_COLUMNS = UK_ETB_SERVICES_OUTPUT_COLUMNS +UK_ETB_SERVICES_FIT_NAME = "uk_etb_2024_services" + + +@dataclass +class UKETBServicesStageTransform: + stage: SourceStageSpec + engine: object + etb_tab_path: str | Path | None = None + donor: pd.DataFrame | None = None + nhs_table: pd.DataFrame | None = None + last_fit_weight_records: tuple[FitWeightRecord, ...] | None = field( + default=None, + init=False, + repr=False, + ) + + @property + def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: + return ( + () if self.last_fit_weight_records is None else self.last_fit_weight_records + ) + + def __call__(self, frame: Frame) -> Frame: + assert_rules_engine_country(self.engine, "uk") + raw = ( + self.donor + if self.donor is not None + else read_pinned_tab( + _require_path(self.etb_tab_path), self.stage.artifacts[0] + ) + ) + donor = clean_etb_services_table(raw) + predictors = recipient_predictors(frame, self.engine) + draws, records = impute_etb_services( + donor, predictors, seed=_qrf_seed(self.stage) + ) + draws = support_clip_to_donor(draws, donor) + draws["rail_usage"] = draws["rail_subsidy_spending"] / RAIL_FARE_INDEX_2023 + household = frame.table("household").copy() + for column in UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS: + household[column] = draws[column].to_numpy() + person = frame.table("person").copy() + nhs = allocate_nhs_by_age_gender( + person, + household_weights=frame.weights_for("household").values, + household=household, + nhs_table=self.nhs_table, + ) + for column in UK_NHS_OUTPUT_COLUMNS: + person[column] = nhs[column].to_numpy() + result = uk_national_frame( + person=person, + benunit=frame.table("benunit").copy(), + household=household, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + validate_uk_national_frame(result) + self.last_fit_weight_records = records + return result + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_ETB_SERVICES_OUTPUT_COLUMNS + + +def clean_etb_services_table(raw: pd.DataFrame) -> pd.DataFrame: + data = raw.replace(r"^\s*$", np.nan, regex=True).copy() + if "year" not in data: + raise ValueError("ETB services donor is missing 'year'.") + data["year"] = pd.to_numeric(data["year"], errors="coerce") + data = data[data["year"] == data["year"].max()].copy() + required = [ + "adults", + "childs", + "disinc", + "educ", + "rail", + "bussub", + "hhold_adj_weight", + "noretd", + "primed", + "secoed", + "furted", + "disliv", + "pips", + ] + missing = [column for column in required if column not in data] + if missing: + raise ValueError( + f"ETB services donor is missing required column(s): {missing}." + ) + for column in required: + data[column] = pd.to_numeric(data[column], errors="coerce") + data = data.dropna(subset=required) + train = pd.DataFrame() + train["is_adult"] = data["adults"] + train["is_child"] = data["childs"] + train["hbai_household_net_income"] = data["disinc"] * ETB_SERVICES_WEEKS_IN_YEAR + train["is_SP_age"] = data["noretd"] + train["count_primary_education"] = data["primed"] + train["count_secondary_education"] = data["secoed"] + train["count_further_education"] = data["furted"] + train["dla"] = data["disliv"] + train["pip"] = data["pips"] + train["weight"] = data["hhold_adj_weight"] + train["dfe_education_spending"] = data["educ"] * ETB_SERVICES_WEEKS_IN_YEAR + train["rail_subsidy_spending"] = data["rail"] * ETB_SERVICES_WEEKS_IN_YEAR + train["bus_subsidy_spending"] = data["bussub"] * ETB_SERVICES_WEEKS_IN_YEAR + return train + + +def household_grain_services_predictors(person_level: pd.DataFrame) -> pd.DataFrame: + grouped = person_level.groupby("household_id", sort=False).sum(numeric_only=True) + return grouped.loc[:, list(UK_ETB_SERVICES_PREDICTORS)] + + +def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: + materialized = engine.materialize( + frame, UK_ETB_SERVICES_PREDICTORS, uk_time_period(frame) + ) + household = frame.table("household") + person = frame.table("person") + result = pd.DataFrame(index=household.index) + for predictor in UK_ETB_SERVICES_PREDICTORS: + values = np.asarray(materialized[predictor]) + entity = str(engine.variable_metadata(predictor).entity) + if entity == "household": + result[predictor] = values + elif entity == "person": + summed = ( + pd.Series(values.astype(float)) + .groupby(person["person_household_id"].to_numpy()) + .sum() + ) + result[predictor] = ( + summed.reindex(household["household_id"]).fillna(0.0).to_numpy() + ) + else: + raise ValueError(f"unsupported ETB services predictor entity {entity!r}.") + return result + + +def impute_etb_services( + donor: pd.DataFrame, recipient: pd.DataFrame, *, seed: int, n_estimators: int = 100 +) -> tuple[pd.DataFrame, tuple[FitWeightRecord, ...]]: + from microcosm.fit import RegimeGatedQRF + + targets = UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS[:3] + model = RegimeGatedQRF(n_estimators=n_estimators, seed=seed) + state = model.start_chain( + donor, + list(UK_ETB_SERVICES_PREDICTORS), + list(targets), + weights="weight", + ) + raw = pd.DataFrame(index=recipient.index) + records: list[FitWeightRecord] = [] + for target in targets: + result = model.fit_draw_next( + donor, + recipient.loc[:, list(UK_ETB_SERVICES_PREDICTORS)], + raw, + state=state, + weights="weight", + ) + raw[target] = result.raw_draw + records.append( + FitWeightRecord(f"{UK_ETB_SERVICES_FIT_NAME}:{target}", result.weight_kind) + ) + state = result.state + return raw, tuple(records) + + +def support_clip_to_donor(draws: pd.DataFrame, donor: pd.DataFrame) -> pd.DataFrame: + result = draws.copy() + for column in UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS[:3]: + values = donor[column] + finite = values[np.isfinite(values)] + if finite.empty: + continue + result[column] = result[column].clip(float(finite.min()), float(finite.max())) + return result + + +def donor_realized_ranges(donor: pd.DataFrame) -> dict[str, tuple[float, float]]: + ranges = {} + for column in UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS[:3]: + values = donor[column] + finite = values[np.isfinite(values)] + if not finite.empty: + ranges[column] = (float(finite.min()), float(finite.max())) + return ranges + + +def parse_nhs_age_bounds(age_group: str) -> tuple[int, int]: + if age_group == "0 years": + return 0, 1 + if age_group == "95 years or older": + return 95, 120 + if "-" in age_group: + lo, hi = age_group.split("-", maxsplit=1) + return int(lo.strip()), int(hi.strip()) + 1 + raise ValueError(f"unsupported NHS age group {age_group!r}") + + +def build_nhs_cell_table( + raw: pd.DataFrame, person: pd.DataFrame, household: pd.DataFrame +) -> pd.DataFrame: + nhs = raw.copy() + bounds = nhs["Age group"].map(parse_nhs_age_bounds) + nhs["Lower age"] = [lo for lo, _ in bounds] + nhs["Upper age"] = [hi for _, hi in bounds] + nhs["Gender"] = nhs["Gender"].str.upper() + pivot = nhs.pivot_table( + index=["Lower age", "Upper age", "Gender", "Service"], + columns="Metric", + values="Total", + aggfunc="sum", + ).reset_index() + top = ( + pivot[pivot["Lower age"] >= 85] + .groupby(["Gender", "Service"], as_index=False)[ + ["Activity Count", "Total Cost"] + ] + .sum() + ) + top["Lower age"] = 85 + top["Upper age"] = 120 + pivot = pd.concat([pivot[pivot["Lower age"] < 85], top], ignore_index=True) + counts = _weighted_person_counts(person, household) + pivot["Total people"] = [ + counts((row["Lower age"], row["Upper age"]), row["Gender"]) + for _, row in pivot.iterrows() + ] + pivot["Per-person average units"] = pivot["Activity Count"] / pivot["Total people"] + factor = NHS_BUDGET_2025_26 / pivot["Total Cost"].sum() + pivot["Per-person average spending"] = ( + pivot["Total Cost"] / pivot["Total people"] * factor + ) + return pivot + + +def allocate_nhs_by_age_gender( + person: pd.DataFrame, + *, + household_weights: np.ndarray, + household: pd.DataFrame, + nhs_table: pd.DataFrame | None, +) -> pd.DataFrame: + if nhs_table is None: + path = ( + Path(__file__).resolve().parents[1] + / "uk/nhs_consumption_by_age_gender.json" + ) + nhs_table = pd.DataFrame(json.loads(path.read_text(encoding="utf-8"))["rows"]) + cells = build_nhs_cell_table(nhs_table, person, household) + output = pd.DataFrame(0.0, index=person.index, columns=UK_NHS_OUTPUT_COLUMNS) + service_to_columns = { + "A&E": ("a_and_e_visits", "nhs_a_and_e_spending"), + "AE": ("a_and_e_visits", "nhs_a_and_e_spending"), + "Admitted Patient": ( + "admitted_patient_visits", + "nhs_admitted_patient_spending", + ), + "APC": ( + "admitted_patient_visits", + "nhs_admitted_patient_spending", + ), + "Outpatient": ("outpatient_visits", "nhs_outpatient_spending"), + "OP": ("outpatient_visits", "nhs_outpatient_spending"), + } + ages = pd.to_numeric(person["age"], errors="coerce").fillna(0) + genders = person["gender"].map(_enum_name).str.upper() + for _, row in cells.iterrows(): + visit_col, spending_col = service_to_columns[row["Service"]] + mask = ( + (ages >= row["Lower age"]) + & (ages < row["Upper age"]) + & (genders == row["Gender"]) + ) + output.loc[mask, visit_col] = row["Per-person average units"] + output.loc[mask, spending_col] = row["Per-person average spending"] + return output + + +def _weighted_person_counts(person: pd.DataFrame, household: pd.DataFrame): + weight_by_household = household.set_index("household_id")["household_weight"] + person_weights = person["person_household_id"].map(weight_by_household).fillna(0.0) + ages = pd.to_numeric(person["age"], errors="coerce").fillna(0) + genders = person["gender"].map(_enum_name).str.upper() + + def count(age_bounds: tuple[int, int], gender: str) -> float: + lo, hi = age_bounds + mask = (ages >= lo) & (ages < hi) & (genders == gender) + total = float(person_weights[mask].sum()) + return total if total > 0 else 1.0 + + return count + + +def _qrf_seed(stage: SourceStageSpec) -> int: + for operation in stage.operations: + if operation.kind == "fit_weighted_qrf_chain": + return int(operation.parameters.get("seed", 0)) + return 0 + + +def _require_path(path: str | Path | None) -> Path: + if path is None: + raise ValueError("ETB services stage requires a caller-supplied ETB tab path.") + return Path(path).expanduser().resolve() + + +def _enum_name(value: object) -> str: + name = getattr(value, "name", None) + return str(name if name is not None else value) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py new file mode 100644 index 00000000..56b138a7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py @@ -0,0 +1,196 @@ +"""UK ETB VAT imputation stage.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.gates import FitWeightRecord +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.frame import Frame +from microcosm.frame.rules import assert_rules_engine_country + +ETB_FILENAME = "householdv2_1977-2024.tab" +ETB_SHA256 = "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8" +ETB_SIZE_BYTES = 216_967_663 +DEFAULT_ETB_VAT_YEAR = 2023 +VAT_STANDARD_RATE_2023 = 0.20 +VAT_REDUCED_RATE_SHARE_2023 = 0.025 +UK_ETB_VAT_PREDICTORS = ( + "is_adult", + "is_child", + "is_SP_age", + "household_net_income", +) +UK_ETB_VAT_OUTPUT_COLUMNS = ("full_rate_vat_expenditure_rate",) +# The donor-realized support includes negative rates (totvat can exceed expdis +# in the raw ETB accounts), so the output stays out of the nonnegative gate and +# the support gate is the guard — the net_financial_wealth precedent. +UK_ETB_VAT_NONNEGATIVE_OUTPUT_COLUMNS: tuple[str, ...] = () +UK_ETB_VAT_FIT_NAME = "uk_etb_2023_vat:full_rate_vat_expenditure_rate" + + +@dataclass +class UKETBVATStageTransform: + stage: SourceStageSpec + engine: object + etb_tab_path: str | Path | None = None + donor: pd.DataFrame | None = None + last_fit_weight_records: tuple[FitWeightRecord, ...] | None = field( + default=None, + init=False, + repr=False, + ) + + @property + def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: + return ( + () if self.last_fit_weight_records is None else self.last_fit_weight_records + ) + + def __call__(self, frame: Frame) -> Frame: + assert_rules_engine_country(self.engine, "uk") + raw = ( + self.donor + if self.donor is not None + else read_pinned_tab( + _require_path(self.etb_tab_path), self.stage.artifacts[0] + ) + ) + donor = clean_etb_vat_table(raw) + predictors = recipient_predictors(frame, self.engine) + imputed, record = impute_etb_vat(donor, predictors, seed=_qrf_seed(self.stage)) + imputed = support_clip_to_donor(imputed, donor) + household = frame.table("household").copy() + household["full_rate_vat_expenditure_rate"] = imputed[ + "full_rate_vat_expenditure_rate" + ].to_numpy() + result = uk_national_frame( + person=frame.table("person").copy(), + benunit=frame.table("benunit").copy(), + household=household, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + validate_uk_national_frame(result) + self.last_fit_weight_records = (record,) + return result + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_ETB_VAT_OUTPUT_COLUMNS + + +def clean_etb_vat_table( + raw: pd.DataFrame, + *, + year: int = DEFAULT_ETB_VAT_YEAR, + standard_rate: float = VAT_STANDARD_RATE_2023, + reduced_rate_share: float = VAT_REDUCED_RATE_SHARE_2023, +) -> pd.DataFrame: + if not np.isfinite(standard_rate) or standard_rate <= 0: + raise ValueError("VAT standard_rate must be positive and finite.") + if not np.isfinite(reduced_rate_share): + raise ValueError("VAT reduced_rate_share must be finite.") + data = raw.replace(r"^\s*$", np.nan, regex=True) + required = [ + "year", + "adults", + "childs", + "noretd", + "disinc", + "totvat", + "expdis", + "hhold_adj_weight", + ] + missing = [column for column in required if column not in data] + if missing: + raise ValueError(f"ETB VAT donor is missing required column(s): {missing}.") + data["year"] = pd.to_numeric(data["year"], errors="coerce") + data = data[data["year"] == year].copy() + for column in required: + data[column] = pd.to_numeric(data[column], errors="coerce") + data = data.dropna(subset=required) + train = pd.DataFrame() + train["is_adult"] = data["adults"] + train["is_child"] = data["childs"] + train["is_SP_age"] = data["noretd"] + train["household_net_income"] = data["disinc"] * 52 + train["weight"] = data["hhold_adj_weight"] + train["full_rate_vat_expenditure_rate"] = ( + data["totvat"] * (1 - reduced_rate_share) / standard_rate + ) / (data["expdis"] - data["totvat"]) + return train.dropna() + + +def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: + materialized = engine.materialize( + frame, UK_ETB_VAT_PREDICTORS, uk_time_period(frame) + ) + household = frame.table("household") + result = pd.DataFrame(index=household.index) + for predictor in UK_ETB_VAT_PREDICTORS: + values = np.asarray(materialized[predictor]) + result[predictor] = values + return result + + +def impute_etb_vat( + donor: pd.DataFrame, recipient: pd.DataFrame, *, seed: int, n_estimators: int = 100 +) -> tuple[pd.DataFrame, FitWeightRecord]: + from microcosm.fit import RegimeGatedQRF + + model = RegimeGatedQRF(n_estimators=n_estimators, seed=seed) + fitted = model.fit( + donor, + list(UK_ETB_VAT_PREDICTORS), + list(UK_ETB_VAT_OUTPUT_COLUMNS), + weights="weight", + ) + return fitted.predict(recipient), FitWeightRecord(UK_ETB_VAT_FIT_NAME, "explicit") + + +def support_clip_to_donor(draws: pd.DataFrame, donor: pd.DataFrame) -> pd.DataFrame: + values = donor["full_rate_vat_expenditure_rate"] + finite = values[np.isfinite(values)] + result = draws.copy() + if not finite.empty: + result["full_rate_vat_expenditure_rate"] = result[ + "full_rate_vat_expenditure_rate" + ].clip(float(finite.min()), float(finite.max())) + return result + + +def donor_realized_ranges(donor: pd.DataFrame) -> dict[str, tuple[float, float]]: + values = donor["full_rate_vat_expenditure_rate"] + finite = values[np.isfinite(values)] + if finite.empty: + return {} + return { + "full_rate_vat_expenditure_rate": (float(finite.min()), float(finite.max())) + } + + +def _qrf_seed(stage: SourceStageSpec) -> int: + for operation in stage.operations: + if operation.kind == "fit_weighted_qrf": + return int(operation.parameters.get("seed", 0)) + return 0 + + +def _require_path(path: str | Path | None) -> Path: + if path is None: + raise ValueError("ETB VAT stage requires a caller-supplied ETB tab path.") + return Path(path).expanduser().resolve() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py new file mode 100644 index 00000000..69305208 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -0,0 +1,680 @@ +"""UK LCFS consumption imputation stage.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.gates import FitWeightRecord +from microcosm.build.raking import MarginSpec, iterative_proportional_fit +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import ( + assign_binary_from_rate, + stable_identity_uniforms, +) +from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.national_frame import ( + uk_household_weight_kind, + uk_national_frame, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.was_wealth import ( + clean_was_household_table, + encode_qrf_predictor_pair, +) +from microcosm.frame import Frame +from microcosm.frame.rules import assert_rules_engine_country + +WEEKS_IN_YEAR = 365.25 / 7 +LCFS_HOUSEHOLD_FILENAME = "dvhh_ukanon_v2_2023.tab" +LCFS_HOUSEHOLD_SHA256 = ( + "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72" +) +LCFS_HOUSEHOLD_SIZE_BYTES = 22_812_887 +LCFS_PERSON_FILENAME = "dvper_ukanon_202324_2023.tab" +LCFS_PERSON_SHA256 = "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50" +LCFS_PERSON_SIZE_BYTES = 6_545_146 + +NTS_ICE_SHARE = 0.90 +UK_LCFS_CONSUMPTION_DECLARED_SEEDS = {"lcfs_consumption": 0} + +LCFS_REGIONS: Mapping[int, str] = { + 1: "NORTH_EAST", + 2: "NORTH_WEST", + 3: "YORKSHIRE", + 4: "EAST_MIDLANDS", + 5: "WEST_MIDLANDS", + 6: "EAST_OF_ENGLAND", + 7: "LONDON", + 8: "SOUTH_EAST", + 9: "SOUTH_WEST", + 10: "WALES", + 11: "SCOTLAND", + 12: "NORTHERN_IRELAND", +} +LCFS_TENURE_MAP: Mapping[int, str] = { + 1: "RENT_FROM_COUNCIL", + 2: "RENT_FROM_HA", + 3: "RENT_PRIVATELY", + 4: "RENT_PRIVATELY", + 5: "OWNED_WITH_MORTGAGE", + 6: "OWNED_WITH_MORTGAGE", + 7: "OWNED_OUTRIGHT", + 8: "RENT_PRIVATELY", +} +LCFS_ACCOMM_MAP: Mapping[int, str] = { + 1: "HOUSE_DETACHED", + 2: "HOUSE_SEMI_DETACHED", + 3: "HOUSE_TERRACED", + 4: "FLAT", + 5: "FLAT", + 6: "MOBILE", + 7: "HOUSE_DETACHED", + 8: "OTHER", +} +HOUSEHOLD_LCFS_RENAMES = { + "g018": "is_adult", + "g019": "is_child", + "gorx": "region", + "p389p": "hbai_household_net_income", + "p344p": "household_gross_income", + "weighta": "household_weight", +} +PERSON_LCFS_RENAMES = { + "b303p": "employment_income", + "b3262p": "self_employment_income", + "p049p": "private_pension_income", +} +CONSUMPTION_VARIABLE_RENAMES = { + "p601": "food_and_non_alcoholic_beverages_consumption", + "p602": "alcohol_and_tobacco_consumption", + "p603": "clothing_and_footwear_consumption", + "p604": "housing_water_and_electricity_consumption", + "p605": "household_furnishings_consumption", + "p606": "health_consumption", + "p607": "transport_consumption", + "p608": "communication_consumption", + "p609": "recreation_consumption", + "p610": "education_consumption", + "p611": "restaurants_and_hotels_consumption", + "p612": "miscellaneous_consumption", + "c72211": "petrol_spending", + "c72212": "diesel_spending", + "p537": "domestic_energy_consumption", +} +BUS_FARE_LCFS_CODES = ("c73212", "c73213", "c73214") +UK_LCFS_HAS_FUEL_PREDICTORS = ( + "household_net_income", + "num_adults", + "num_children", + "private_pension_income", + "employment_income", + "self_employment_income", + "region", +) +UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS = ( + "is_adult", + "is_child", + "employment_income", + "self_employment_income", + "private_pension_income", + "hbai_household_net_income", +) +UK_LCFS_CONSUMPTION_PREDICTORS = ( + "is_adult", + "is_child", + "region", + "employment_income", + "self_employment_income", + "private_pension_income", + "hbai_household_net_income", + "tenure_type", + "accommodation_type", + "has_fuel_consumption", +) +UK_LCFS_CONSUMPTION_TARGET_COLUMNS = ( + "food_and_non_alcoholic_beverages_consumption", + "alcohol_and_tobacco_consumption", + "clothing_and_footwear_consumption", + "housing_water_and_electricity_consumption", + "household_furnishings_consumption", + "health_consumption", + "transport_consumption", + "communication_consumption", + "recreation_consumption", + "education_consumption", + "restaurants_and_hotels_consumption", + "miscellaneous_consumption", + "petrol_spending", + "diesel_spending", + "bus_fare_spending", + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption", +) +UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS = ( + *UK_LCFS_CONSUMPTION_TARGET_COLUMNS, + "has_fuel_consumption", +) +UK_LCFS_CONSUMPTION_NONNEGATIVE_OUTPUT_COLUMNS = UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS +UK_LCFS_CONSUMPTION_FIT_NAME = "uk_lcfs_2023_24_consumption" +UK_LCFS_HAS_FUEL_FIT_NAME = "uk_was_2018_20_has_fuel" + + +@dataclass +class UKLCFSConsumptionStageTransform: + """Whole-stage callable for LCFS-trained consumption imputation.""" + + stage: SourceStageSpec + engine: object + lcfs_hh_tab_path: str | Path | None = None + lcfs_person_tab_path: str | Path | None = None + was_tab_path: str | Path | None = None + lcfs_household: pd.DataFrame | None = None + lcfs_person: pd.DataFrame | None = None + was_donor: pd.DataFrame | None = None + last_fit_weight_records: tuple[FitWeightRecord, ...] | None = field( + default=None, + init=False, + repr=False, + ) + + @property + def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: + if self.last_fit_weight_records is None: + return () + return tuple(self.last_fit_weight_records) + + def __call__(self, frame: Frame) -> Frame: + assert_rules_engine_country(self.engine, "uk") + lcfs_household = ( + self.lcfs_household + if self.lcfs_household is not None + else read_pinned_tab( + _require_path(self.lcfs_hh_tab_path), + _artifact(self.stage, "lcfs_household_tab"), + ) + ) + lcfs_person = ( + self.lcfs_person + if self.lcfs_person is not None + else read_pinned_tab( + _require_path(self.lcfs_person_tab_path), + _artifact(self.stage, "lcfs_person_tab"), + ) + ) + was_raw = ( + self.was_donor + if self.was_donor is not None + else read_pinned_tab( + _require_path(self.was_tab_path), + _artifact(self.stage, "was_bridge_donor"), + ) + ) + was = clean_was_household_table(was_raw) + donor = clean_lcfs_consumption_table(lcfs_person, lcfs_household) + donor, bridge_record = bridge_has_fuel_to_lcfs( + donor, was, seed=_operation_seed(self.stage, "bridge_donor_column_via_qrf") + ) + recipient = recipient_predictors(frame, self.engine) + recipient["has_fuel_consumption"] = assign_recipient_has_fuel( + frame, + rate=NTS_ICE_SHARE, + seed=_operation_seed(self.stage, "assign_binary_from_rate"), + ) + imputation = impute_lcfs_consumption( + donor, + recipient, + seed=_operation_seed(self.stage, "fit_weighted_qrf_chain"), + n_estimators=_qrf_n_estimators(self.stage), + ) + household_draws = support_clip_to_donor( + imputation.draws, + donor, + exempt={ + "electricity_consumption", + "gas_consumption", + "domestic_energy_consumption", + }, + ) + household_draws = rake_energy_to_need( + household_draws.join(recipient[["household_gross_income"]]), + weights=frame.weights_for("household").values, + ) + household_draws["domestic_energy_consumption"] = ( + household_draws["electricity_consumption"] + + household_draws["gas_consumption"] + ) + household_draws.loc[ + ~recipient["has_fuel_consumption"].astype(bool), + ["petrol_spending", "diesel_spending"], + ] = 0.0 + household = frame.table("household").copy() + for column in UK_LCFS_CONSUMPTION_TARGET_COLUMNS: + household[column] = household_draws[column].to_numpy() + household["has_fuel_consumption"] = recipient["has_fuel_consumption"].to_numpy( + dtype=bool + ) + result = uk_national_frame( + person=frame.table("person").copy(), + benunit=frame.table("benunit").copy(), + household=household, + time_period=uk_time_period(frame), + weight_kind=uk_household_weight_kind(frame), + household_weights=frame.weights_for("household").values, + mass_log=frame.mass_log, + ) + validate_uk_national_frame(result) + self.last_fit_weight_records = (bridge_record, *imputation.fit_weight_records) + return result + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS + + +@dataclass(frozen=True) +class UKLCFSConsumptionImputationResult: + draws: pd.DataFrame + fit_weight_records: tuple[FitWeightRecord, ...] + + +def clean_lcfs_consumption_table( + lcfs_person: pd.DataFrame, lcfs_household: pd.DataFrame +) -> pd.DataFrame: + """Return the LCFS donor table with annualized consumption variables.""" + + person = _lowercase(lcfs_person).rename(columns=PERSON_LCFS_RENAMES) + household = _lowercase(lcfs_household).rename(columns=HOUSEHOLD_LCFS_RENAMES) + _require_columns(household, ("case", *HOUSEHOLD_LCFS_RENAMES.values())) + _require_columns(person, ("case", *PERSON_LCFS_RENAMES.values())) + household["region"] = _numeric(household["region"]).map(LCFS_REGIONS) + household["tenure_type"] = _numeric(_lowercase(lcfs_household)["a122"]).map( + LCFS_TENURE_MAP + ) + household["accommodation_type"] = _numeric(_lowercase(lcfs_household)["a121"]).map( + LCFS_ACCOMM_MAP + ) + household = derive_energy_from_lcfs(household) + household = household.rename(columns=CONSUMPTION_VARIABLE_RENAMES) + for code in BUS_FARE_LCFS_CODES: + if code not in household: + raise ValueError(f"LCFS household donor is missing {code!r}.") + household["bus_fare_spending"] = sum( + _numeric(household[code]) for code in BUS_FARE_LCFS_CODES + ) + annualize = [ + *CONSUMPTION_VARIABLE_RENAMES.values(), + "bus_fare_spending", + "hbai_household_net_income", + "household_gross_income", + "electricity_consumption", + "gas_consumption", + ] + for column in annualize: + household[column] = _numeric(household[column]) * WEEKS_IN_YEAR + for column in PERSON_LCFS_RENAMES.values(): + totals = person.groupby("case")[column].sum() + household[column] = household["case"].map(totals).fillna(0.0) * WEEKS_IN_YEAR + household["household_weight"] = _numeric(household["household_weight"]) * 1_000 + household = rake_energy_to_need(household, weights=None, iterations=1) + household["domestic_energy_consumption"] = ( + household["electricity_consumption"] + household["gas_consumption"] + ) + return household[ + [ + *UK_LCFS_CONSUMPTION_PREDICTORS[:-1], + *UK_LCFS_CONSUMPTION_TARGET_COLUMNS, + "household_gross_income", + "household_weight", + ] + ].dropna() + + +def derive_energy_from_lcfs(household: pd.DataFrame) -> pd.DataFrame: + """Split LCFS domestic energy into electricity and gas weekly amounts.""" + + for column in ("p537", "b226", "b489", "b490"): + if column not in household: + raise ValueError(f"LCFS household donor is missing {column!r}.") + p537 = _numeric(household["p537"]) + b226 = _numeric(household["b226"]) + b489 = _numeric(household["b489"]) + b490 = _numeric(household["b490"]) + dd_mask = (b226 > 0) & (p537 > 0) + mean_elec_share = (b226[dd_mask] / p537[dd_mask]).clip(0, 1).mean() + if np.isnan(mean_elec_share): + mean_elec_share = 0.52 + electricity = np.zeros(len(household)) + gas = np.zeros(len(household)) + mask1 = b226 > 0 + electricity[mask1] = b226[mask1] + gas[mask1] = np.maximum(p537[mask1] - b226[mask1], 0) + mask2 = (~mask1) & (b489 > 0) & (b490 > 0) + electricity[mask2] = np.maximum(b489[mask2] - b490[mask2], 0) + gas[mask2] = b490[mask2] + mask3 = (~mask1) & (b489 > 0) & (b490 == 0) + electricity[mask3] = b489[mask3] * mean_elec_share + gas[mask3] = b489[mask3] * (1 - mean_elec_share) + mask4 = (~mask1) & (b489 == 0) + electricity[mask4] = p537[mask4] * mean_elec_share + gas[mask4] = p537[mask4] * (1 - mean_elec_share) + result = household.copy() + result["electricity_consumption"] = np.maximum(electricity, 0.0) + result["gas_consumption"] = np.maximum(gas, 0.0) + return result + + +def bridge_has_fuel_to_lcfs( + lcfs: pd.DataFrame, + was: pd.DataFrame, + *, + seed: int, + n_estimators: int = 100, +) -> tuple[pd.DataFrame, FitWeightRecord]: + """Fit a WAS has-fuel bridge and predict a clipped rate onto LCFS.""" + + from microcosm.fit import RegimeGatedQRF + + donor = was.copy() + donor["has_fuel_consumption"] = ( + (_numeric(donor["num_vehicles"]) > 0) + & ( + stable_identity_uniforms( + donor.index.to_numpy(), seed=seed, salt="was_has_fuel" + ) + < NTS_ICE_SHARE + ) + ).astype(float) + donor_encoded, recipient_encoded, predictors = encode_qrf_predictor_pair( + donor[[*UK_LCFS_HAS_FUEL_PREDICTORS, "has_fuel_consumption", "weight"]], + lcfs[list(UK_LCFS_HAS_FUEL_PREDICTORS)], + ) + model = RegimeGatedQRF(n_estimators=n_estimators, seed=seed) + result = model.fit( + donor_encoded, + list(predictors), + ["has_fuel_consumption"], + weights="weight", + ).draw(recipient_encoded) + out = lcfs.copy() + out["has_fuel_consumption"] = np.clip( + np.asarray(result["has_fuel_consumption"], dtype=float), 0.0, 1.0 + ) + return out, FitWeightRecord(UK_LCFS_HAS_FUEL_FIT_NAME, "explicit") + + +def assign_recipient_has_fuel(frame: Frame, *, rate: float, seed: int) -> np.ndarray: + household = frame.table("household") + if "num_vehicles" not in household: + raise KeyError("recipient household table is missing 'num_vehicles'.") + draws = stable_identity_uniforms( + household["household_id"].to_numpy(), + seed=seed, + salt="lcfs_has_fuel_consumption", + ) + return (_numeric(household["num_vehicles"]) > 0) & assign_binary_from_rate( + draws, rate + ) + + +def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: + """Materialize LCFS recipient predictors at household grain.""" + + materialized = engine.materialize( + frame, UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS, uk_time_period(frame) + ) + household = frame.table("household") + person = frame.table("person") + result = pd.DataFrame(index=household.index) + for predictor in UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS: + declared = str(engine.variable_metadata(predictor).entity) + values = np.asarray(materialized[predictor]) + if declared == "household": + result[predictor] = values + elif declared == "person": + summed = ( + pd.Series(values.astype(float)) + .groupby(person["person_household_id"].to_numpy()) + .sum() + ) + result[predictor] = ( + summed.reindex(household["household_id"]).fillna(0.0).to_numpy() + ) + else: + raise ValueError(f"unsupported LCFS predictor entity {declared!r}.") + for predictor in ("region", "tenure_type", "accommodation_type", "num_vehicles"): + if predictor in household: + result[predictor] = household[predictor].map(_enum_name).to_numpy() + if "household_gross_income" in household: + result["household_gross_income"] = household[ + "household_gross_income" + ].to_numpy() + else: + result["household_gross_income"] = result["hbai_household_net_income"] + return result + + +def impute_lcfs_consumption( + donor: pd.DataFrame, + recipient_predictor_frame: pd.DataFrame, + *, + seed: int, + n_estimators: int, +) -> UKLCFSConsumptionImputationResult: + from microcosm.fit import RegimeGatedQRF + + donor_encoded, recipient_encoded, predictors = _encode_consumption_predictors( + donor, recipient_predictor_frame + ) + model = RegimeGatedQRF(n_estimators=n_estimators, seed=seed) + state = model.start_chain( + donor_encoded, + list(predictors), + list(UK_LCFS_CONSUMPTION_TARGET_COLUMNS), + weights="household_weight", + ) + raw = pd.DataFrame(index=recipient_encoded.index) + fit_records: list[FitWeightRecord] = [] + for target in UK_LCFS_CONSUMPTION_TARGET_COLUMNS: + result = model.fit_draw_next( + donor_encoded, + recipient_encoded.loc[:, list(predictors)], + raw, + state=state, + weights="household_weight", + ) + raw[target] = result.raw_draw + fit_records.append( + FitWeightRecord( + f"{UK_LCFS_CONSUMPTION_FIT_NAME}:{target}", result.weight_kind + ) + ) + state = result.state + return UKLCFSConsumptionImputationResult(raw, tuple(fit_records)) + + +def support_clip_to_donor( + draws: pd.DataFrame, + donor: pd.DataFrame, + *, + exempt: set[str] | None = None, +) -> pd.DataFrame: + clipped = draws.copy() + exempt = exempt or set() + for column in UK_LCFS_CONSUMPTION_TARGET_COLUMNS: + if column in exempt or column not in clipped or column not in donor: + continue + values = pd.to_numeric(donor[column], errors="coerce") + finite = values[np.isfinite(values)] + if finite.empty: + continue + clipped[column] = clipped[column].clip( + lower=float(finite.min()), upper=float(finite.max()) + ) + return clipped + + +def donor_realized_ranges(donor: pd.DataFrame) -> dict[str, tuple[float, float]]: + ranges: dict[str, tuple[float, float]] = {} + for column in UK_LCFS_CONSUMPTION_TARGET_COLUMNS: + if column in { + "electricity_consumption", + "gas_consumption", + "domestic_energy_consumption", + }: + continue + values = pd.to_numeric(donor[column], errors="coerce") + finite = values[np.isfinite(values)] + if not finite.empty: + ranges[column] = (float(finite.min()), float(finite.max())) + return ranges + + +def rake_energy_to_need( + household: pd.DataFrame, + *, + weights: Sequence[float] | None, + iterations: int = 50, +) -> pd.DataFrame: + frame = household.copy() + frame["_need_income_band"] = _income_band(frame["household_gross_income"]) + weight_column = None + if weights is not None: + frame["_weight"] = np.asarray(weights, dtype=float) + weight_column = "_weight" + raked = iterative_proportional_fit( + frame, + columns=("electricity_consumption", "gas_consumption"), + margins=(MarginSpec("_need_income_band", _NEED_INCOME_TARGETS),), + iterations=iterations, + weight_column=weight_column, + ) + return raked.drop( + columns=[c for c in ("_need_income_band", "_weight") if c in raked] + ) + + +_NEED_INCOME_BANDS = ( + (0, 15_000, "under_15k", 7_755, 2_412), + (15_000, 20_000, "15k_20k", 9_196, 2_700), + (20_000, 30_000, "20k_30k", 9_886, 2_915), + (30_000, 40_000, "30k_40k", 10_697, 3_114), + (40_000, 50_000, "40k_50k", 11_230, 3_276), + (50_000, 60_000, "50k_60k", 11_721, 3_410), + (60_000, 70_000, "60k_70k", 12_200, 3_548), + (70_000, 100_000, "70k_100k", 13_244, 3_872), + (100_000, 150_000, "100k_150k", 15_727, 4_598), + (150_000, np.inf, "over_150k", 20_359, 5_944), +) +_ELEC_RATE = 24.67 / 100 +_GAS_RATE = 5.74 / 100 +_NEED_INCOME_TARGETS = { + name: { + "gas_consumption": gas * _GAS_RATE, + "electricity_consumption": elec * _ELEC_RATE, + } + for _, _, name, gas, elec in _NEED_INCOME_BANDS +} + + +def _income_band(values: pd.Series) -> pd.Series: + income = _numeric(values) + result = pd.Series(index=income.index, dtype=object) + for lo, hi, name, _, _ in _NEED_INCOME_BANDS: + result[(income >= lo) & (income < hi)] = name + return result + + +def _encode_consumption_predictors( + donor: pd.DataFrame, recipient: pd.DataFrame +) -> tuple[pd.DataFrame, pd.DataFrame, tuple[str, ...]]: + categorical = ("region", "tenure_type", "accommodation_type") + base_predictors = tuple( + p for p in UK_LCFS_CONSUMPTION_PREDICTORS if p not in categorical + ) + donor_work = donor.copy() + recipient_work = recipient.copy() + combined = pd.concat( + [ + donor_work.loc[:, categorical].reset_index(drop=True), + recipient_work.loc[:, categorical].reset_index(drop=True), + ], + ignore_index=True, + ) + dummies = pd.get_dummies( + combined.astype(str), columns=list(categorical), dtype=float + ) + dummies = dummies.reindex(sorted(dummies.columns), axis=1) + + def encode(table: pd.DataFrame, block: pd.DataFrame) -> pd.DataFrame: + encoded = table.drop(columns=list(categorical), errors="ignore").copy() + for column in encoded.columns: + if column == "household_weight": + continue + encoded[column] = pd.to_numeric(encoded[column], errors="coerce").fillna( + 0.0 + ) + block = block.copy() + block.index = encoded.index + return pd.concat([encoded, block], axis=1) + + donor_encoded = encode(donor_work, dummies.iloc[: len(donor_work)]) + recipient_encoded = encode(recipient_work, dummies.iloc[len(donor_work) :]) + return donor_encoded, recipient_encoded, (*base_predictors, *tuple(dummies.columns)) + + +def _lowercase(data: pd.DataFrame) -> pd.DataFrame: + result = data.copy() + result.columns = [str(column).lower() for column in result.columns] + return result + + +def _numeric(values: pd.Series) -> pd.Series: + return pd.to_numeric(values, errors="coerce").fillna(0.0) + + +def _require_columns(data: pd.DataFrame, columns: Sequence[str]) -> None: + missing = [column for column in columns if column not in data] + if missing: + raise ValueError(f"LCFS donor is missing required column(s): {missing}.") + + +def _artifact(stage: SourceStageSpec, role: str) -> Mapping[str, Any]: + for artifact in stage.artifacts: + if artifact.get("role") == role: + return artifact + raise ValueError(f"{stage.stage} declares no {role!r} artifact.") + + +def _operation_seed(stage: SourceStageSpec, kind: str) -> int: + for operation in stage.operations: + if operation.kind == kind and isinstance(operation.parameters.get("seed"), int): + return int(operation.parameters["seed"]) + return 0 + + +def _qrf_n_estimators(stage: SourceStageSpec) -> int: + for operation in stage.operations: + if operation.kind == "fit_weighted_qrf_chain": + value = operation.parameters.get("n_estimators", 100) + if isinstance(value, int) and value > 0: + return value + return 100 + + +def _require_path(path: str | Path | None) -> Path: + if path is None: + raise ValueError("LCFS consumption stage requires caller-supplied donor paths.") + return Path(path).expanduser().resolve() + + +def _enum_name(value: object) -> str: + name = getattr(value, "name", None) + return str(name if name is not None else value) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py index c1b4548b..57323656 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/source_runtime.py @@ -63,6 +63,9 @@ def uk_stage_implementations( frs_brma_transform: Callable[[Frame], Frame] | None = None, was_wealth_transform: Callable[[Frame], Frame] | None = None, regional_property_uprating_transform: Callable[[Frame], Frame] | None = None, + lcfs_consumption_transform: Callable[[Frame], Frame] | None = None, + etb_vat_transform: Callable[[Frame], Frame] | None = None, + etb_services_transform: Callable[[Frame], Frame] | None = None, frs_hmrc_spine_leaves_transform: Callable[[Frame], Frame] | None = None, spi_support_channel_transform: Callable[[Frame], Frame] | None = None, hmrc_spi_income_spine_transform: Callable[[Frame], Frame] | None = None, @@ -87,6 +90,9 @@ def uk_stage_implementations( "frs_brma": frs_brma_transform, "was_wealth": was_wealth_transform, "regional_property_uprating": regional_property_uprating_transform, + "lcfs_consumption": lcfs_consumption_transform, + "etb_vat": etb_vat_transform, + "etb_services": etb_services_transform, "frs_hmrc_spine_leaves": frs_hmrc_spine_leaves_transform, "spi_support_channel": spi_support_channel_transform, "hmrc_spi_income_spine": hmrc_spi_income_spine_transform, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py index d0eefeab..4bd613f7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_income.py @@ -416,7 +416,9 @@ def impute_uk_spi_income_support( columns=initialize_frs_channel_columns, ) base_redraw_columns = tuple(stage1_base_redraw_columns) - unknown_redraw = sorted(set(base_redraw_columns) - set(SPI_INCOME_QRF_OUTPUT_COLUMNS)) + unknown_redraw = sorted( + set(base_redraw_columns) - set(SPI_INCOME_QRF_OUTPUT_COLUMNS) + ) if unknown_redraw: raise ValueError( "SPI stage-1 base redraw columns must be stage-1 QRF outputs; " @@ -508,6 +510,8 @@ def impute_uk_spi_income_support( weights=person_weights.loc[training_people].to_numpy(dtype=np.float64), weight_kind=WeightKind.IMPORTANCE, ) + # Stage 2 deliberately derives a distinct RNG stream from the single + # reviewed stage seed; changing this edits the derivation convention. stage2 = qrf_cls(n_estimators=n_estimators, seed=seed + 1).fit( stage2_frame, list(encoded_train.columns), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py index 68392488..57b79833 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spi_spine.py @@ -250,6 +250,7 @@ def evidence(self) -> dict[str, object]: class UKFRSHMRCSpineLeavesStageTransform: frs_raw_dir: Path stage: SourceStageSpec + # Populated only by a live run; resume paths must re-run or skip evidence. last_result: UKFRSHMRCSpineLeavesResult | None = field(default=None, init=False) def __init__(self, frs_raw_dir: str | Path, *, stage: SourceStageSpec) -> None: @@ -295,11 +296,10 @@ def __call__(self, frame: Frame) -> Frame: ) if employee.isna().any() or (employee < 0.0).any(): raise ValueError( - "employee_pension_contributions must be finite and " - "non-negative." + "employee_pension_contributions must be finite and non-negative." ) - person[EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN] = ( - 3.0 * employee.to_numpy(dtype=float) + person[EMPLOYER_PENSION_CONTRIBUTIONS_COLUMN] = 3.0 * employee.to_numpy( + dtype=float ) result_frame = uk_national_frame( person=person, @@ -319,9 +319,7 @@ def __call__(self, frame: Frame) -> Frame: frame=result_frame, source_signal_rows=source_signal_rows, structural_zero_columns=tuple( - column - for column, rows in source_signal_rows.items() - if rows == 0 + column for column, rows in source_signal_rows.items() if rows == 0 ), ) object.__setattr__(self, "last_result", result) @@ -341,6 +339,7 @@ def checkpoint_metadata(self) -> dict[str, object]: class UKSPISupportChannelStageTransform: stage: SourceStageSpec seed: int = 42 + # Populated only by a live run; resume paths must re-run or skip evidence. last_result: UKSPISupportResult | None = field(default=None, init=False) def __call__(self, frame: Frame) -> Frame: @@ -363,12 +362,22 @@ def __call__(self, frame: Frame) -> Frame: mass_log=frame.mass_log, zero_weight_declarations=declarations, ) + if result.household_weight_kind is not WeightKind.IMPORTANCE: + got = ( + None + if result.household_weight_kind is None + else result.household_weight_kind.value + ) + raise ValueError( + "SPI support channel builder must return importance household " + f"weights, got {got!r}." + ) result_frame = uk_national_frame( person=result.person, benunit=result.benunit, household=result.household, time_period=uk_time_period(frame), - weight_kind=result.household_weight_kind or WeightKind.IMPORTANCE, + weight_kind=result.household_weight_kind, household_weights=result.household["household_weight"].to_numpy( dtype=float ), @@ -407,6 +416,7 @@ class UKSPIIncomeSpineStageTransform: seed: int = 42 qrf_estimators: int = 100 donor_sample_size: int | None = DEFAULT_SPI_DONOR_SAMPLE_SIZE + # Populated only by a live run; resume paths must re-run or skip evidence. last_result: UKSPIIncomeSpineResult | None = field(default=None, init=False) def __init__( @@ -636,7 +646,9 @@ def _support_stage_parameters( if allocation.parameters.get("weight_kind_out") != WeightKind.IMPORTANCE.value: raise ValueError("SPI support allocation must advance to importance weights.") if allocation.parameters.get("conservation") != "exact_total": - raise ValueError("SPI support allocation must declare exact-total conservation.") + raise ValueError( + "SPI support allocation must declare exact-total conservation." + ) strata = tuple(allocation.parameters.get("strata", ())) if strata != ("region",): raise ValueError("SPI support spine allocation must use strata ['region'].") @@ -675,7 +687,10 @@ def _assert_income_stage_parameters( if stage1.parameters.get("seed") != seed: raise ValueError("SPI income stage-1 seed drifted from the reviewed value.") if stage2.parameters.get("seed") != seed + 1: - raise ValueError("SPI income stage-2 seed drifted from the reviewed value.") + raise ValueError( + "SPI income stage-2 seed drifted from the reviewed seed + 1 " + "derivation convention." + ) if stage1.parameters.get("n_estimators") not in (None, qrf_estimators): raise ValueError("SPI income stage-1 estimator count drifted.") if stage2.parameters.get("n_estimators") not in (None, qrf_estimators): @@ -744,7 +759,9 @@ def _artifact_by_table( } missing = sorted(set(expected) - set(artifacts)) if missing: - raise ValueError(f"Stage {stage.stage!r} is missing tab artifact(s): {missing}.") + raise ValueError( + f"Stage {stage.stage!r} is missing tab artifact(s): {missing}." + ) return artifacts diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py index 85d5cddb..8759a77f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/terminal_gates.py @@ -177,6 +177,8 @@ def __post_init__(self) -> None: "household.region_code_oa", "household.stocks_and_shares_isa", "person.aa_category", + "person.a_and_e_visits", + "person.admitted_patient_visits", "person.age_started_or_accepted_current_education_or_training", "person.attends_private_school_random_draw", "person.charitable_investment_gifts", @@ -191,6 +193,7 @@ def __post_init__(self) -> None: "person.is_in_non_advanced_education", "person.is_parent", "person.legacy_jobseeker_proxy", + "person.outpatient_visits", "person.pension_contributions_via_salary_sacrifice", "person.pip_dl_category", "person.pip_m_category", diff --git a/packages/microcosm-build/tests/test_country_spec.py b/packages/microcosm-build/tests/test_country_spec.py index 2253c179..de592a46 100644 --- a/packages/microcosm-build/tests/test_country_spec.py +++ b/packages/microcosm-build/tests/test_country_spec.py @@ -286,6 +286,14 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", "hmrc_income_source_stages.json", + "need_energy_targets.json", + "lcfs_consumption_anchors.json", + "etb_policy_anchors.json", + "etb_services_anchors.json", + "nhs_consumption_by_age_gender.json", + "lcfs_consumption_support_bounds.json", + "etb_vat_support_bounds.json", + "etb_services_support_bounds.json", "regional_land_values.json", "source_stages.json", "take_up_contract.json", @@ -299,11 +307,11 @@ def test_spi_spine_adds_no_country_package_resources(self) -> None: "target_references.json", ) - def test_uk_source_manifest_loads_eighteen_stages(self) -> None: + def test_uk_source_manifest_loads_twenty_one_stages(self) -> None: spec = load_country_spec("uk") assert spec.sources is not None - assert len(spec.sources.stages) == 18 + assert len(spec.sources.stages) == 21 class TestExistingPackagesGeneralize: @@ -338,6 +346,14 @@ def test_uk_package_loads(self) -> None: "hmrc_income_release_gate_report.json", "hmrc_income_replay_report.json", "hmrc_income_source_stages.json", + "need_energy_targets.json", + "lcfs_consumption_anchors.json", + "etb_policy_anchors.json", + "etb_services_anchors.json", + "nhs_consumption_by_age_gender.json", + "lcfs_consumption_support_bounds.json", + "etb_vat_support_bounds.json", + "etb_services_support_bounds.json", "regional_land_values.json", "source_stages.json", "take_up_contract.json", @@ -562,6 +578,7 @@ def test_declares_the_full_june_battery(self, manifest) -> None: "uk_weights_audit", "uk_nonnegative_columns", "uk_support", + "uk_aggregate_admin", "uk_export_surface", "uk_take_up_signal", "uk_brma_enum_domain", @@ -597,21 +614,30 @@ def test_thresholds_match_the_schema4_manifest(self, manifest) -> None: params["uk_weight_ratio"]["maximum_max_to_median_ratio"] == 1_151.2542195939373 ) - assert ( - params["uk_input_mass_parity"]["relative_tolerance"] - == 4.521811483823806 - ) + assert params["uk_input_mass_parity"]["relative_tolerance"] == 4.521811483823806 assert params["uk_input_mass_parity"]["minimum_reference_total"] == 0.0 assert params["uk_qrf_tail_concentration"]["top_k"] == 100 assert ( - params["uk_qrf_tail_concentration"]["max_top_share"] - == 0.9970712395200448 + params["uk_qrf_tail_concentration"]["max_top_share"] == 0.9970712395200448 ) assert params["uk_qrf_tail_concentration"]["min_nonzero_records"] == 274 assert ( params["uk_target_fit"]["max_abs_relative_error"] == terminal_gates.UK_MAX_TARGET_ABS_RELATIVE_ERROR ) + assert params["uk_support"]["support_bounds_resources"] == ( + "was_wealth_support_bounds.json", + "lcfs_consumption_support_bounds.json", + "etb_vat_support_bounds.json", + "etb_services_support_bounds.json", + ) + aggregate = params["uk_aggregate_admin"] + assert aggregate["default_rtol"] == 0.15 + assert [anchor["name"] for anchor in aggregate["anchors"]] == [ + "need_electricity_mean_spending", + "need_gas_mean_spending", + "nhs_spending_total", + ] def test_zero_weight_declarations_match_the_june_strata(self, manifest) -> None: params = {gate.id: gate.parameters for gate in manifest.gates} diff --git a/packages/microcosm-build/tests/test_uk_battery_bindings.py b/packages/microcosm-build/tests/test_uk_battery_bindings.py index 042cbfbf..0feec00f 100644 --- a/packages/microcosm-build/tests/test_uk_battery_bindings.py +++ b/packages/microcosm-build/tests/test_uk_battery_bindings.py @@ -176,6 +176,11 @@ def _run_battery(tables, *, parity=None, fit_records=None, armed=True, clock=CLO artifacts["parity_evidence"] = parity if armed: artifacts["input_mass_reference"] = _reference() + artifacts["aggregate_admin"] = { + "need_electricity_mean_spending": 882.91463, + "need_gas_mean_spending": 700.3661, + "nhs_spending_total": 202_000_000_000, + } # Small synthetic totals exercise battery behavior without disclosing # the licensed 131-column reference (same patch as the legacy tests); # the binding's declared-pin check compares spec to runtime constant and @@ -297,7 +302,14 @@ def test_support_binding_passes_in_range_was_outputs(self) -> None: result = binding.evaluate( EvidenceContext(frame=frame, artifacts={}), - {"support_bounds_resource": "was_wealth_support_bounds.json"}, + { + "support_bounds_resources": [ + "was_wealth_support_bounds.json", + "lcfs_consumption_support_bounds.json", + "etb_vat_support_bounds.json", + "etb_services_support_bounds.json", + ] + }, ) assert result.passed is True @@ -322,6 +334,66 @@ def test_support_binding_fails_out_of_range_was_outputs(self) -> None: assert result.passed is False assert "cash_isa" in result.failures[0] + def test_support_binding_checks_e6_support_resources(self) -> None: + person, benunit, household = _tables(n=1) + household["full_rate_vat_expenditure_rate"] = [999_999.0] + frame = uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2023", + ) + binding = UK_GATE_REGISTRY["support"] + + result = binding.evaluate( + EvidenceContext(frame=frame, artifacts={}), + {"support_bounds_resources": ["etb_vat_support_bounds.json"]}, + ) + + assert result.passed is False + assert "full_rate_vat_expenditure_rate" in result.failures[0] + + def test_aggregate_admin_binding_checks_declared_anchors(self) -> None: + binding = UK_GATE_REGISTRY["aggregate_admin"] + + result = binding.evaluate( + EvidenceContext( + frame=None, + artifacts={ + "aggregate_admin": { + "need_electricity_mean_spending": 882.91463, + "nhs_spending_total": 202_000_000_000, + } + }, + ), + { + "default_rtol": 0.15, + "anchors": [ + { + "name": "need_electricity_mean_spending", + "entity": "household", + "measure": "electricity_consumption", + "value": 882.91463, + "period": "2023", + "source": "test", + "family": "need_energy", + }, + { + "name": "nhs_spending_total", + "entity": "person", + "measure": "nhs_spending", + "value": 202_000_000_000, + "period": "2025_26", + "source": "test", + "family": "nhs", + }, + ], + }, + ) + + assert result.passed is True + assert result.details["anchors_checked"] == 2 + class TestUKCompatibility: """The BE plumbing test, run over the UK spec: an empty evidence context @@ -377,8 +449,9 @@ def test_fully_armed_battery_evaluates_gate_for_gate(self) -> None: ] # 11 as on main (uk_nonnegative_columns passes with zero required # columns — the scheduled stages declare none), the two E4 stochastic - # gates, and the E5 support gate; their evaluators have direct tests. - assert len(passed) == 14 + # gates, the E5 support gate, and the E6 aggregate-admin gate; their + # evaluators have direct tests. + assert len(passed) == 15 qrf = by_id["uk_qrf_tail_concentration"] assert qrf.status is GateStatus.FAILED assert "declared QRF output is absent" in qrf.result.failures[0] @@ -433,6 +506,7 @@ def test_battery_records_evidence_absent(self, uk_gates) -> None: "uk_target_surface", "uk_target_fit", "uk_input_mass_parity", + "uk_aggregate_admin", } for reason in absent.values(): assert reason.startswith("missing evidence: ") diff --git a/packages/microcosm-build/tests/test_uk_consumption_resources.py b/packages/microcosm-build/tests/test_uk_consumption_resources.py new file mode 100644 index 00000000..d1681862 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_consumption_resources.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +UK_PACKAGE = ROOT / "packages/microcosm-build/src/microcosm/build/uk" + + +def _load(name: str) -> dict: + return json.loads((UK_PACKAGE / name).read_text(encoding="utf-8")) + + +def test_need_energy_targets_shape_and_citations() -> None: + payload = _load("need_energy_targets.json") + + assert payload["version"] == 1 + assert payload["country"] == "uk" + assert payload["source"]["chronicle_candidate"] is True + assert "NEED 2023" in payload["source"]["citation"] + assert len(payload["income_bands"]) == 10 + assert payload["tenure"]["map"]["OWNED_OUTRIGHT"] == "owner" + assert payload["accommodation"]["map"]["FLAT"] == "flat" + assert "NORTHERN_IRELAND" not in payload["region"]["gas_kwh"] + + +def test_policy_anchor_resources_carry_parameter_paths() -> None: + lcfs = _load("lcfs_consumption_anchors.json") + vat = _load("etb_policy_anchors.json") + services = _load("etb_services_anchors.json") + + assert lcfs["source"]["chronicle_candidate"] is True + assert lcfs["cpi"]["parameter_path"] + assert vat["vat"]["standard_rate"]["parameter_path"] == ( + "gov.hmrc.vat.standard_rate" + ) + assert vat["vat"]["reduced_rate_share"]["value"] == 0.025 + assert services["rail_fare_index_2023"]["parameter_path"] == ( + "gov.dft.rail.fare_index" + ) + assert services["nhs_budget_2025_26"]["value"] == 202_000_000_000 + + +def test_policy_anchor_values_lockstep_with_engine_parameter_tree() -> None: + """Every anchor value with a parameter_path equals the installed tree's value. + + This is the Option A drift guard adjudicated on microcosm#682: an engine + bump that moves one of these historical values fails here and forces a + reviewed resource diff. Skips where policyengine-uk is absent — PR CI's + hermetic lanes never import the engine. + """ + system_module = pytest.importorskip("policyengine_uk.system") + + parameters = system_module.system.parameters + vat = _load("etb_policy_anchors.json")["vat"] + for name, anchor in vat.items(): + node = parameters + for part in anchor["parameter_path"].split("."): + node = getattr(node, part) + assert float(node(str(anchor["period"]))) == anchor["value"], name + + services = _load("etb_services_anchors.json")["rail_fare_index_2023"] + node = parameters + for part in services["parameter_path"].split("."): + node = getattr(node, part) + assert float(node(str(services["period"]))) == services["value"] + + cpi = _load("lcfs_consumption_anchors.json")["cpi"] + node = parameters + for part in cpi["parameter_path"].split("."): + node = getattr(node, part) + assert float(node(str(cpi["start_period"]))) > 0 + + +def test_nhs_consumption_resource_ports_public_csv_rows() -> None: + payload = _load("nhs_consumption_by_age_gender.json") + + assert payload["version"] == 1 + assert payload["source"]["sdc_treatment"].startswith("Public aggregate") + assert len(payload["rows"]) == 252 + first = payload["rows"][0] + assert set(first) == {"Service", "Gender", "Age group", "Metric", "Total"} + assert isinstance(first["Total"], float) + + +def test_e6_support_bounds_resources_are_sha_bound_and_non_placeholder() -> None: + lcfs = _load("lcfs_consumption_support_bounds.json") + vat = _load("etb_vat_support_bounds.json") + services = _load("etb_services_support_bounds.json") + + assert lcfs["source"]["household_tab_sha256"] == ( + "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72" + ) + assert lcfs["source"]["person_tab_sha256"] == ( + "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50" + ) + assert len(lcfs["bounds"]) == 15 + assert vat["source"]["tab_sha256"] == ( + "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8" + ) + assert set(vat["bounds"]) == {"full_rate_vat_expenditure_rate"} + assert set(services["bounds"]) == { + "bus_subsidy_spending", + "dfe_education_spending", + "rail_subsidy_spending", + } + for name in ( + "lcfs_consumption_support_bounds.json", + "etb_vat_support_bounds.json", + "etb_services_support_bounds.json", + ): + assert ( + "placeholder" not in (UK_PACKAGE / name).read_text(encoding="utf-8").lower() + ) + + +def test_e6_support_bounds_round_trip_against_licensed_tabs() -> None: + lcfs_hh = os.environ.get("POPULACE_UK_LCFS_HH_TAB") + lcfs_person = os.environ.get("POPULACE_UK_LCFS_PERSON_TAB") + etb = os.environ.get("POPULACE_UK_ETB_TAB") + if not all( + value and Path(value).is_file() for value in (lcfs_hh, lcfs_person, etb) + ): + pytest.skip( + "licensed E6 tabs not available " + "(set POPULACE_UK_LCFS_HH_TAB, POPULACE_UK_LCFS_PERSON_TAB, " + "POPULACE_UK_ETB_TAB)" + ) + import importlib.util + + path = ROOT / "tools/build_uk_e6_support_bounds.py" + spec = importlib.util.spec_from_file_location("build_uk_e6_support_bounds", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + + expected = { + "lcfs_consumption_support_bounds.json": module.build_lcfs_support_bounds( + Path(lcfs_hh), Path(lcfs_person) + ), + "etb_vat_support_bounds.json": module.build_etb_vat_support_bounds(Path(etb)), + "etb_services_support_bounds.json": module.build_etb_services_support_bounds( + Path(etb) + ), + } + for name, payload in expected.items(): + assert json.dumps(payload, indent=2, sort_keys=False) + "\n" == ( + UK_PACKAGE / name + ).read_text(encoding="utf-8") diff --git a/packages/microcosm-build/tests/test_uk_etb_services.py b/packages/microcosm-build/tests/test_uk_etb_services.py new file mode 100644 index 00000000..3b88e2d5 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_etb_services.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.etb_services import ( + NHS_BUDGET_2025_26, + RAIL_FARE_INDEX_2023, + UK_ETB_SERVICES_FIT_NAME, + build_nhs_cell_table, + clean_etb_services_table, + donor_realized_ranges, + household_grain_services_predictors, + impute_etb_services, + parse_nhs_age_bounds, + support_clip_to_donor, +) + + +def _raw_etb() -> pd.DataFrame: + return pd.DataFrame( + { + "year": [2023, 2024, 2024], + "adults": [9, 2, 1], + "childs": [9, 1, 0], + "disinc": [9, 100.0, 200.0], + "educ": [9, 10.0, 20.0], + "rail": [9, 2.0, 4.0], + "bussub": [9, 1.0, 3.0], + "hhold_adj_weight": [9, 5.0, 6.0], + "noretd": [9, 0, 1], + "primed": [9, 1, 0], + "secoed": [9, 2, 0], + "furted": [9, 0, 1], + "disliv": [9, 7.0, 8.0], + "pips": [9, 3.0, 4.0], + } + ) + + +def test_etb_services_feature_maps_and_annualized_outputs() -> None: + donor = clean_etb_services_table(_raw_etb()) + + assert donor["is_adult"].tolist() == [2, 1] + assert donor["hbai_household_net_income"].tolist() == [5200.0, 10400.0] + assert donor["count_primary_education"].tolist() == [1, 0] + assert donor["dla"].tolist() == [7.0, 8.0] + assert donor["pip"].tolist() == [3.0, 4.0] + assert donor["weight"].tolist() == [5.0, 6.0] + assert donor["dfe_education_spending"].tolist() == [520.0, 1040.0] + assert donor["rail_subsidy_spending"].tolist() == [104.0, 208.0] + assert donor["bus_subsidy_spending"].tolist() == [52.0, 156.0] + + +def test_household_grain_predictors_match_per_capita_round_trip_identity() -> None: + person_level = pd.DataFrame( + { + "household_id": [1, 1, 2], + "is_adult": [1, 1, 1], + "is_child": [0, 1, 0], + "is_SP_age": [0, 0, 1], + "count_primary_education": [0, 1, 0], + "count_secondary_education": [1, 0, 0], + "count_further_education": [0, 0, 1], + "dla": [2.0, 3.0, 4.0], + "pip": [0.5, 0.5, 1.0], + "hbai_household_net_income": [50.0, 50.0, 200.0], + } + ) + + direct = household_grain_services_predictors(person_level) + + per_capita = direct.copy() + counts = person_level.groupby("household_id").size() + for column in ["dfe_education_spending", "rail_subsidy_spending"]: + per_capita[column] = [100.0, 300.0] + person_level[column] = person_level["household_id"].map( + per_capita[column] / counts + ) + + assert direct.loc[1, "is_adult"] == 2 + assert person_level.groupby("household_id")[ + ["dfe_education_spending", "rail_subsidy_spending"] + ].sum().loc[1].tolist() == [100.0, 100.0] + + +def test_etb_services_chain_order_and_records(monkeypatch: pytest.MonkeyPatch) -> None: + class _FakeModel: + def __init__(self, *, n_estimators, seed): + assert n_estimators == 100 + assert seed == 0 + self.calls = [] + + def start_chain(self, donor, predictors, targets, *, weights): + assert weights == "weight" + assert targets == [ + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending", + ] + return {"targets": targets} + + def fit_draw_next(self, donor, recipient_base, raw, *, state, weights): + return type( + "Result", + (), + { + "raw_draw": pd.Series( + [float(len(raw.columns) + 1)], index=raw.index + ), + "weight_kind": "explicit", + "state": state, + }, + )() + + import microcosm.fit as fit_module + + monkeypatch.setattr(fit_module, "RegimeGatedQRF", _FakeModel) + + donor = clean_etb_services_table(_raw_etb()) + recipient = donor.iloc[:1].drop( + columns=[ + "weight", + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending", + ] + ) + + draws, records = impute_etb_services(donor, recipient, seed=0) + + assert draws.columns.tolist() == [ + "dfe_education_spending", + "rail_subsidy_spending", + "bus_subsidy_spending", + ] + assert draws.iloc[0].tolist() == [1.0, 2.0, 3.0] + assert [record.fit_name for record in records] == [ + f"{UK_ETB_SERVICES_FIT_NAME}:dfe_education_spending", + f"{UK_ETB_SERVICES_FIT_NAME}:rail_subsidy_spending", + f"{UK_ETB_SERVICES_FIT_NAME}:bus_subsidy_spending", + ] + + +def test_services_support_clip_ranges_and_rail_ratio() -> None: + donor = clean_etb_services_table(_raw_etb()) + draws = pd.DataFrame( + { + "dfe_education_spending": [-1.0, 9999.0], + "rail_subsidy_spending": [-1.0, 9999.0], + "bus_subsidy_spending": [-1.0, 9999.0], + } + ) + + clipped = support_clip_to_donor(draws, donor) + + assert clipped["dfe_education_spending"].tolist() == [520.0, 1040.0] + assert donor_realized_ranges(donor)["rail_subsidy_spending"] == (104.0, 208.0) + assert 111.0 / RAIL_FARE_INDEX_2023 == pytest.approx(100.0) + + +def _nhs_raw() -> pd.DataFrame: + rows = [] + for age_group, activity, cost in [ + ("0 years", 10.0, 100.0), + ("85-89", 20.0, 300.0), + ("90-94", 30.0, 600.0), + ("95 years or older", 40.0, 1000.0), + ]: + for metric, total in [ + ("Activity Count", activity), + ("Total Cost", cost), + ]: + rows.append( + { + "Age group": age_group, + "Gender": "Female", + "Service": "A&E", + "Metric": metric, + "Total": total, + } + ) + return pd.DataFrame(rows) + + +def test_nhs_age_parsing_and_85_plus_fold_in_uses_full_table_denominator() -> None: + assert parse_nhs_age_bounds("0 years") == (0, 1) + assert parse_nhs_age_bounds("95 years or older") == (95, 120) + assert parse_nhs_age_bounds("85-89") == (85, 90) + + person = pd.DataFrame( + { + "person_id": [1, 2, 3], + "person_household_id": [1, 1, 2], + "age": [0, 85, 95], + "gender": ["FEMALE", "FEMALE", "FEMALE"], + } + ) + household = pd.DataFrame( + { + "household_id": [1, 2], + "household_weight": [2.0, 3.0], + } + ) + + cells = build_nhs_cell_table(_nhs_raw(), person, household) + top = cells[cells["Lower age"] == 85].iloc[0] + + assert top["Upper age"] == 120 + assert top["Activity Count"] == 90.0 + assert top["Total Cost"] == 1900.0 + assert top["Total people"] == 5.0 + assert np.isclose( + cells["Per-person average spending"].mul(cells["Total people"]).sum(), + NHS_BUDGET_2025_26, + ) diff --git a/packages/microcosm-build/tests/test_uk_etb_vat.py b/packages/microcosm-build/tests/test_uk_etb_vat.py new file mode 100644 index 00000000..8f60b2bf --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_etb_vat.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.uk_runtime.etb_vat import ( + UK_ETB_VAT_FIT_NAME, + clean_etb_vat_table, + donor_realized_ranges, + impute_etb_vat, + support_clip_to_donor, +) + + +def _raw_etb() -> pd.DataFrame: + return pd.DataFrame( + { + "year": [2022, 2023, 2023, 2023], + "adults": [9, 2, 1, " "], + "childs": [9, 1, 0, 0], + "noretd": [9, 0, 1, 0], + "disinc": [9, 100.0, 200.0, 300.0], + "totvat": [9, 20.0, 10.0, 30.0], + "expdis": [9, 120.0, 110.0, 130.0], + "hhold_adj_weight": [9, 3.0, 4.0, 5.0], + } + ) + + +def test_etb_vat_cleaning_filters_2023_and_computes_target() -> None: + donor = clean_etb_vat_table(_raw_etb()) + + assert donor["is_adult"].tolist() == [2.0, 1.0] + assert donor["is_child"].tolist() == [1.0, 0.0] + assert donor["is_SP_age"].tolist() == [0.0, 1.0] + assert donor["household_net_income"].tolist() == [5200.0, 10400.0] + assert donor["weight"].tolist() == [3.0, 4.0] + expected = [ + (20.0 * 0.975 / 0.20) / (120.0 - 20.0), + (10.0 * 0.975 / 0.20) / (110.0 - 10.0), + ] + np.testing.assert_allclose(donor["full_rate_vat_expenditure_rate"], expected) + + +def test_etb_vat_cleaning_fails_loud_on_missing_rate() -> None: + with pytest.raises(ValueError, match="standard_rate"): + clean_etb_vat_table(_raw_etb(), standard_rate=np.nan) + + +def test_etb_vat_support_clip_and_ranges() -> None: + donor = clean_etb_vat_table(_raw_etb()) + draws = pd.DataFrame({"full_rate_vat_expenditure_rate": [-99.0, 99.0]}) + + clipped = support_clip_to_donor(draws, donor) + + assert clipped["full_rate_vat_expenditure_rate"].tolist() == [ + donor["full_rate_vat_expenditure_rate"].min(), + donor["full_rate_vat_expenditure_rate"].max(), + ] + assert donor_realized_ranges(donor) == { + "full_rate_vat_expenditure_rate": ( + float(donor["full_rate_vat_expenditure_rate"].min()), + float(donor["full_rate_vat_expenditure_rate"].max()), + ) + } + + +def test_etb_vat_weighted_fit_record(monkeypatch: pytest.MonkeyPatch) -> None: + class _FakeFitted: + def predict(self, recipient): + return pd.DataFrame( + {"full_rate_vat_expenditure_rate": [0.1]}, index=recipient.index + ) + + class _FakeModel: + def __init__(self, *, n_estimators, seed): + assert n_estimators == 100 + assert seed == 0 + + def fit(self, donor, predictors, targets, *, weights): + assert weights == "weight" + assert targets == ["full_rate_vat_expenditure_rate"] + return _FakeFitted() + + import microcosm.fit as fit_module + + monkeypatch.setattr(fit_module, "RegimeGatedQRF", _FakeModel) + + donor = clean_etb_vat_table(_raw_etb()) + draws, record = impute_etb_vat( + donor, + pd.DataFrame( + { + "is_adult": [1.0], + "is_child": [0.0], + "is_SP_age": [0.0], + "household_net_income": [1.0], + } + ), + seed=0, + ) + + assert draws["full_rate_vat_expenditure_rate"].tolist() == [0.1] + assert record.fit_name == UK_ETB_VAT_FIT_NAME + assert record.weight_kind == "explicit" diff --git a/packages/microcosm-build/tests/test_uk_frs_spine.py b/packages/microcosm-build/tests/test_uk_frs_spine.py index b4a6e0b3..e01ed24a 100644 --- a/packages/microcosm-build/tests/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/test_uk_frs_spine.py @@ -653,9 +653,7 @@ def source_stage( "declarations": [ { "name": "e7_spi_synthetic_preclone", - "selector": { - "household_is_spi_synthetic": True - }, + "selector": {"household_is_spi_synthetic": True}, "maximum_zero_weight_rows": 10000, "reason": "synthetic driver fixture", } @@ -1100,14 +1098,14 @@ def test_driver_writes_spine_h5_sidecars_and_logbook( [ "--frs-raw-dir", str(raw_dir), - "--spine-h5", - str(output), - "--spi-tab", - str(spi_tab), - "--hmrc-ods", - str(hmrc_ods), - "--emit-nonzero-shares", - str(shares), + "--spine-h5", + str(output), + "--spi-tab", + str(spi_tab), + "--hmrc-ods", + str(hmrc_ods), + "--emit-nonzero-shares", + str(shares), ] ) == 0 @@ -1425,14 +1423,28 @@ def test_input_artifact_pins_bind_spi_donor_and_ods() -> None: pins = tool._input_artifact_pins(stages) - assert set(pins) == {"qrf_donor", "was_qrf_donor", "published_fact_surface"} + assert set(pins) == { + "etb_household_tab", + "lcfs_household_tab", + "lcfs_person_tab", + "published_fact_surface", + "qrf_donor", + "was_bridge_donor", + "was_qrf_donor", + } for pin in pins.values(): assert len(str(pin["sha256"])) == 64 assert int(pin["size_bytes"]) > 0 assert str(pin["filename"]) declared = { str(artifact["role"]): str(artifact["sha256"]) - for stage_name in ("hmrc_spi_income_spine", "was_wealth") + for stage_name in ( + "was_wealth", + "lcfs_consumption", + "etb_vat", + "etb_services", + "hmrc_spi_income_spine", + ) for artifact in stage_map[stage_name].artifacts if "table" not in artifact and "resource" not in artifact } diff --git a/packages/microcosm-build/tests/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py new file mode 100644 index 00000000..732b8af3 --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime.lcfs_consumption import ( + BUS_FARE_LCFS_CODES, + LCFS_ACCOMM_MAP, + LCFS_TENURE_MAP, + UK_LCFS_CONSUMPTION_TARGET_COLUMNS, + assign_recipient_has_fuel, + clean_lcfs_consumption_table, + derive_energy_from_lcfs, + support_clip_to_donor, +) +from microcosm.build.uk_runtime.national_frame import uk_national_frame + + +def _household() -> pd.DataFrame: + base = { + "case": [1, 2, 3, 4], + "g018": [2, 1, 3, 1], + "g019": [0, 1, 2, 0], + "gorx": [7, 12, 10, 1], + "p389p": [100.0, 200.0, 300.0, 400.0], + "p344p": [150.0, 250.0, 350.0, 450.0], + "weighta": [1.5, 2.0, 2.5, 3.0], + "a122": [4, 8, 5, 7], + "a121": [4, 5, 6, 7], + "b226": [6.0, 0.0, 0.0, 0.0], + "b489": [0.0, 9.0, 8.0, 0.0], + "b490": [0.0, 4.0, 0.0, 0.0], + "p537": [10.0, 20.0, 30.0, -1.0], + } + for source in ( + "p601", + "p602", + "p603", + "p604", + "p605", + "p606", + "p607", + "p608", + "p609", + "p610", + "p611", + "p612", + "c72211", + "c72212", + *BUS_FARE_LCFS_CODES, + ): + base[source] = [1.0, 2.0, 3.0, 4.0] + return pd.DataFrame(base) + + +def _person() -> pd.DataFrame: + return pd.DataFrame( + { + "case": [1, 1, 3], + "b303p": [10.0, 5.0, 7.0], + "b3262p": [1.0, 2.0, 3.0], + "p049p": [4.0, 5.0, 6.0], + } + ) + + +def test_lcfs_donor_cleaning_arithmetic_and_lossy_maps() -> None: + donor = clean_lcfs_consumption_table(_person(), _household()) + + assert donor["region"].tolist() == [ + "LONDON", + "NORTHERN_IRELAND", + "WALES", + "NORTH_EAST", + ] + assert LCFS_TENURE_MAP[4] == "RENT_PRIVATELY" + assert LCFS_TENURE_MAP[8] == "RENT_PRIVATELY" + assert LCFS_ACCOMM_MAP[4] == "FLAT" + assert LCFS_ACCOMM_MAP[5] == "FLAT" + assert donor["household_weight"].tolist() == [1500.0, 2000.0, 2500.0, 3000.0] + assert np.isclose( + donor.loc[0, "employment_income"], + (10.0 + 5.0) * (365.25 / 7), + ) + assert donor.loc[1, "employment_income"] == 0.0 + assert np.isclose(donor.loc[0, "bus_fare_spending"], 3.0 * (365.25 / 7)) + + +def test_energy_split_exercises_four_cases_fallback_and_clamp() -> None: + household = _household() + split = derive_energy_from_lcfs(household) + + assert split["electricity_consumption"].tolist() == [6.0, 5.0, 4.8, 0.0] + assert split["gas_consumption"].tolist() == [4.0, 4.0, 3.2, 0.0] + + fallback = household.copy() + fallback["b226"] = 0.0 + fallback["b489"] = 0.0 + fallback["p537"] = [10.0, 20.0, 30.0, 40.0] + + split = derive_energy_from_lcfs(fallback) + + np.testing.assert_allclose( + split["electricity_consumption"], [5.2, 10.4, 15.6, 20.8] + ) + np.testing.assert_allclose(split["gas_consumption"], [4.8, 9.6, 14.4, 19.2]) + + +def test_recipient_has_fuel_is_conditioned_on_vehicle_count_and_deterministic() -> None: + frame = uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [10, 20], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + { + "household_id": [10, 20], + "household_weight": [1.0, 1.0], + "num_vehicles": [0, 2], + } + ), + time_period="2023", + ) + + first = assign_recipient_has_fuel(frame, rate=1.0, seed=0) + second = assign_recipient_has_fuel(frame, rate=1.0, seed=0) + + assert first.tolist() == [False, True] + assert second.tolist() == first.tolist() + + +def test_support_clip_exempts_raked_energy_columns() -> None: + donor = pd.DataFrame( + {column: [1.0, 5.0] for column in UK_LCFS_CONSUMPTION_TARGET_COLUMNS} + ) + draws = pd.DataFrame( + {column: [0.0, 10.0] for column in UK_LCFS_CONSUMPTION_TARGET_COLUMNS} + ) + + clipped = support_clip_to_donor( + draws, + donor, + exempt={ + "electricity_consumption", + "gas_consumption", + "domestic_energy_consumption", + }, + ) + + assert clipped["food_and_non_alcoholic_beverages_consumption"].tolist() == [ + 1.0, + 5.0, + ] + assert clipped["electricity_consumption"].tolist() == [0.0, 10.0] diff --git a/packages/microcosm-build/tests/test_uk_national_build.py b/packages/microcosm-build/tests/test_uk_national_build.py index 9a932bc6..45d77959 100644 --- a/packages/microcosm-build/tests/test_uk_national_build.py +++ b/packages/microcosm-build/tests/test_uk_national_build.py @@ -1003,6 +1003,7 @@ def test_national_build_real_terminal_batch_blocks_incomplete_qrf_before_staging "uk_weights_audit": "passed", "uk_nonnegative_columns": "passed", "uk_support": "passed", + "uk_aggregate_admin": "evidence_absent", "uk_take_up_signal": "passed", "uk_brma_enum_domain": "passed", # The legacy report omitted unevidenced gates; the battery names diff --git a/packages/microcosm-build/tests/test_uk_nhs_allocation.py b/packages/microcosm-build/tests/test_uk_nhs_allocation.py new file mode 100644 index 00000000..51933a1b --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_nhs_allocation.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd + +from microcosm.build.uk_runtime.etb_services import ( + NHS_BUDGET_2025_26, + allocate_nhs_by_age_gender, + build_nhs_cell_table, + parse_nhs_age_bounds, +) + + +def _raw_nhs_rows() -> pd.DataFrame: + rows = [] + for age_group, activity, cost in [ + ("80-84", 10.0, 100.0), + ("85-89", 20.0, 300.0), + ("90-94", 30.0, 600.0), + ("95 years or older", 40.0, 1000.0), + ]: + for metric, total in ( + ("Activity Count", activity), + ("Total Cost", cost), + ): + rows.append( + { + "Age group": age_group, + "Gender": "Female", + "Service": "A&E", + "Metric": metric, + "Total": total, + } + ) + return pd.DataFrame(rows) + + +def test_nhs_age_bound_parsing_uses_half_open_top_code() -> None: + assert parse_nhs_age_bounds("0 years") == (0, 1) + assert parse_nhs_age_bounds("85-89") == (85, 90) + assert parse_nhs_age_bounds("95 years or older") == (95, 120) + + +def test_nhs_85_plus_fold_in_and_budget_normalization_use_full_table() -> None: + person = pd.DataFrame( + { + "person_id": [1, 2], + "person_household_id": [1, 2], + "age": [84, 85], + "gender": ["female", "FEMALE"], + } + ) + household = pd.DataFrame( + { + "household_id": [1, 2], + "household_weight": [2.0, 3.0], + } + ) + + cells = build_nhs_cell_table(_raw_nhs_rows(), person, household) + top = cells[cells["Lower age"] == 85].iloc[0] + + assert top["Upper age"] == 120 + assert top["Activity Count"] == 90.0 + assert top["Total Cost"] == 1900.0 + assert top["Total people"] == 3.0 + assert np.isclose( + cells["Per-person average spending"].mul(cells["Total people"]).sum(), + NHS_BUDGET_2025_26, + ) + + allocated = allocate_nhs_by_age_gender( + person, + household_weights=household["household_weight"].to_numpy(dtype=float), + household=household, + nhs_table=_raw_nhs_rows(), + ) + + assert allocated.loc[0, "a_and_e_visits"] == 5.0 + assert allocated.loc[1, "a_and_e_visits"] == 30.0 + assert allocated.loc[0, "nhs_a_and_e_spending"] < allocated.loc[ + 1, "nhs_a_and_e_spending" + ] diff --git a/packages/microcosm-build/tests/test_uk_raking.py b/packages/microcosm-build/tests/test_uk_raking.py new file mode 100644 index 00000000..42f5af6a --- /dev/null +++ b/packages/microcosm-build/tests/test_uk_raking.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd + +from microcosm.build.raking import MarginSpec, iterative_proportional_fit + + +def test_two_margin_sweep_matches_hand_computed_ratios() -> None: + frame = pd.DataFrame( + { + "group": ["a", "a", "b", "b"], + "kind": ["x", "y", "x", "y"], + "value": [1.0, 3.0, 2.0, 4.0], + } + ) + + raked = iterative_proportional_fit( + frame, + columns=("value",), + margins=( + MarginSpec("group", {"a": {"value": 8.0}, "b": {"value": 12.0}}), + MarginSpec("kind", {"x": {"value": 9.0}, "y": {"value": 11.0}}), + ), + iterations=1, + ) + + np.testing.assert_allclose( + raked["value"], + [6.0, 66.0 / 7.0, 12.0, 88.0 / 7.0], + ) + + +def test_single_pass_income_margin_degenerates_to_incumbent_training_calibration(): + frame = pd.DataFrame( + { + "income_band": ["low", "low", "high", "high"], + "gas": [10.0, 30.0, 2.0, 0.0], + "electricity": [4.0, 6.0, 0.0, 0.0], + } + ) + + raked = iterative_proportional_fit( + frame, + columns=("gas", "electricity"), + margins=( + MarginSpec( + "income_band", + { + "low": {"gas": 80.0, "electricity": 20.0}, + "high": {"gas": 10.0, "electricity": 5.0}, + }, + ), + ), + iterations=1, + ) + + assert raked["gas"].tolist() == [40.0, 120.0, 20.0, 0.0] + assert raked["electricity"].tolist() == [16.0, 24.0, 0.0, 0.0] + + +def test_weighted_and_unweighted_means_use_distinct_denominators() -> None: + frame = pd.DataFrame( + { + "band": ["a", "a"], + "value": [10.0, 30.0], + "weight": [1.0, 3.0], + } + ) + + unweighted = iterative_proportional_fit( + frame, + columns=("value",), + margins=(MarginSpec("band", {"a": {"value": 40.0}}),), + iterations=1, + ) + weighted = iterative_proportional_fit( + frame, + columns=("value",), + margins=(MarginSpec("band", {"a": {"value": 40.0}}),), + iterations=1, + weight_column="weight", + ) + + assert unweighted["value"].tolist() == [20.0, 60.0] + np.testing.assert_allclose(weighted["value"], [16.0, 48.0]) + + +def test_zero_empty_and_unmapped_cells_are_left_untouched() -> None: + frame = pd.DataFrame( + { + "band": ["zero", "empty", "unmapped"], + "value": [0.0, 5.0, 7.0], + } + ) + + raked = iterative_proportional_fit( + frame, + columns=("value",), + margins=( + MarginSpec( + "band", + { + "zero": {"value": 10.0}, + "absent": {"value": 20.0}, + }, + ), + ), + iterations=1, + ) + + assert raked["value"].tolist() == [0.0, 5.0, 7.0] + + +def test_margin_sweep_order_is_observable_and_pinned() -> None: + frame = pd.DataFrame( + { + "first": ["a", "a"], + "second": ["x", "y"], + "value": [1.0, 3.0], + } + ) + + raked = iterative_proportional_fit( + frame, + columns=("value",), + margins=( + MarginSpec("first", {"a": {"value": 8.0}}), + MarginSpec("second", {"x": {"value": 2.0}, "y": {"value": 6.0}}), + ), + iterations=1, + ) + + assert raked["value"].tolist() == [2.0, 6.0] diff --git a/packages/microcosm-build/tests/test_uk_release_input_coverage.py b/packages/microcosm-build/tests/test_uk_release_input_coverage.py index a92e3b43..7467bf20 100644 --- a/packages/microcosm-build/tests/test_uk_release_input_coverage.py +++ b/packages/microcosm-build/tests/test_uk_release_input_coverage.py @@ -607,6 +607,9 @@ def test_shipped_manifest_is_current(self) -> None: "hmrc_cgt_gains", "was_wealth", "regional_property_uprating", + "lcfs_consumption", + "etb_vat", + "etb_services", } ) assert RESTORED_REFERENCE_EFRS_REQUIRED_INPUTS == frozenset( diff --git a/packages/microcosm-build/tests/test_uk_source_runtime.py b/packages/microcosm-build/tests/test_uk_source_runtime.py index b73dabe5..0a288ab5 100644 --- a/packages/microcosm-build/tests/test_uk_source_runtime.py +++ b/packages/microcosm-build/tests/test_uk_source_runtime.py @@ -116,11 +116,17 @@ def hmrc(frame: Frame) -> Frame: hmrc_income_transform=hmrc, was_wealth_transform=retained, regional_property_uprating_transform=hmrc, + lcfs_consumption_transform=retained, + etb_vat_transform=hmrc, + etb_services_transform=retained, ) == { "frs_hmrc_retained_leaves": retained, "hmrc_spi_income": hmrc, "was_wealth": retained, "regional_property_uprating": hmrc, + "lcfs_consumption": retained, + "etb_vat": hmrc, + "etb_services": retained, } diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index 5f9c5a33..d413aa9e 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -36,6 +36,11 @@ "was_wealth", "regional_property_uprating", ] +E6_STAGE_NAMES = [ + "lcfs_consumption", + "etb_vat", + "etb_services", +] E7_STAGE_NAMES = [ "frs_hmrc_spine_leaves", "spi_support_channel", @@ -46,6 +51,7 @@ *E3_STAGE_NAMES, *E4_STAGE_NAMES, *E5_STAGE_NAMES, + *E6_STAGE_NAMES, *E7_STAGE_NAMES, "frs_hmrc_retained_leaves", "hmrc_spi_income", @@ -54,6 +60,7 @@ "frs_spine", *E3_STAGE_NAMES, *E4_STAGE_NAMES, + *E6_STAGE_NAMES, *E7_STAGE_NAMES, ] FROZEN_SOURCE_STAGES_SHA256 = ( @@ -103,6 +110,19 @@ def test_country_spec_declares_uk_source_stages(self) -> None: assert spec.sources is not None assert [stage.stage for stage in spec.sources.stages] == UK_SOURCE_STAGE_NAMES + def test_e6_block_sits_between_e5_and_e7(self) -> None: + canonical = _load_json(CANONICAL_SOURCE_STAGES) + names = [stage["stage"] for stage in canonical["stages"]] + + assert ( + names[ + names.index("regional_property_uprating") + 1 : names.index( + "frs_hmrc_spine_leaves" + ) + ] + == E6_STAGE_NAMES + ) + def test_e7_block_is_contiguous_before_certified_pair(self) -> None: canonical = _load_json(CANONICAL_SOURCE_STAGES) names = [stage["stage"] for stage in canonical["stages"]] @@ -208,6 +228,9 @@ def test_country_stage_plan_assembles_fourteen_stage_spine_plan(self) -> None: "frs_brma": _identity, "was_wealth": _identity, "regional_property_uprating": _identity, + "lcfs_consumption": _identity, + "etb_vat": _identity, + "etb_services": _identity, "frs_hmrc_spine_leaves": _identity, "spi_support_channel": _identity, "hmrc_spi_income_spine": _identity, @@ -253,6 +276,14 @@ def test_stage1_outputs_are_exactly_the_retained_leaf_columns(self) -> None: assert stage1.outputs == tuple(FRS_HMRC_RETAINED_LEAF_COLUMNS) def test_e3_outputs_are_backed_by_runtime_written_columns(self) -> None: + from microcosm.build.uk_runtime.etb_services import ( + UK_ETB_SERVICES_NONNEGATIVE_OUTPUT_COLUMNS, + UK_ETB_SERVICES_OUTPUT_COLUMNS, + ) + from microcosm.build.uk_runtime.etb_vat import ( + UK_ETB_VAT_NONNEGATIVE_OUTPUT_COLUMNS, + UK_ETB_VAT_OUTPUT_COLUMNS, + ) from microcosm.build.uk_runtime.frs_brma import FRS_BRMA_OUTPUT_COLUMNS from microcosm.build.uk_runtime.frs_council_tax import ( FRS_COUNCIL_TAX_OUTPUT_COLUMNS, @@ -284,6 +315,10 @@ def test_e3_outputs_are_backed_by_runtime_written_columns(self) -> None: FRS_TAKE_UP_NONNEGATIVE_OUTPUT_COLUMNS, FRS_TAKE_UP_OUTPUT_COLUMNS, ) + from microcosm.build.uk_runtime.lcfs_consumption import ( + UK_LCFS_CONSUMPTION_NONNEGATIVE_OUTPUT_COLUMNS, + UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS, + ) from microcosm.build.uk_runtime.regional_uprating import ( UK_REGIONAL_PROPERTY_REWRITES, ) @@ -331,6 +366,21 @@ def test_e3_outputs_are_backed_by_runtime_written_columns(self) -> None: stages["regional_property_uprating"].rewrites == UK_REGIONAL_PROPERTY_REWRITES ) + assert stages["lcfs_consumption"].outputs == UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS + assert ( + stages["lcfs_consumption"].nonnegative_outputs + == UK_LCFS_CONSUMPTION_NONNEGATIVE_OUTPUT_COLUMNS + ) + assert stages["etb_vat"].outputs == UK_ETB_VAT_OUTPUT_COLUMNS + assert ( + stages["etb_vat"].nonnegative_outputs + == UK_ETB_VAT_NONNEGATIVE_OUTPUT_COLUMNS + ) + assert stages["etb_services"].outputs == UK_ETB_SERVICES_OUTPUT_COLUMNS + assert ( + stages["etb_services"].nonnegative_outputs + == UK_ETB_SERVICES_NONNEGATIVE_OUTPUT_COLUMNS + ) def test_e7_outputs_and_rewrites_are_backed_by_runtime_constants(self) -> None: from microcosm.build.uk_runtime.spi_spine import ( @@ -450,6 +500,32 @@ def test_e3_operation_kinds_are_declared_in_order(self) -> None: assert [op.kind for op in stages["regional_property_uprating"].operations] == [ "uprate_to_regional_reference", ] + assert [op.kind for op in stages["lcfs_consumption"].operations] == [ + "derive", + "iterative_proportional_fit", + "bridge_donor_column_via_qrf", + "assign_binary_from_rate", + "materialize_rules_engine_predictors", + "fit_weighted_qrf_chain", + "support_clip", + "iterative_proportional_fit", + "fold_into", + "zero_when_false", + ] + assert [op.kind for op in stages["etb_vat"].operations] == [ + "derive", + "materialize_rules_engine_predictors", + "fit_weighted_qrf", + "support_clip", + ] + assert [op.kind for op in stages["etb_services"].operations] == [ + "derive", + "materialize_rules_engine_predictors", + "fit_weighted_qrf_chain", + "support_clip", + "compute_ratio", + "allocate_per_capita_from_cell_table", + ] assert [op.kind for op in stages["frs_hmrc_spine_leaves"].operations] == [ "retain_adjudicated_frs_hmrc_leaves", "derive", @@ -471,6 +547,11 @@ def test_e3_operation_kinds_are_declared_in_order(self) -> None: ] def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: + from microcosm.build.uk_runtime.etb_services import ( + UK_ETB_SERVICES_OUTPUT_COLUMNS, + UK_ETB_SERVICES_PREDICTORS, + ) + from microcosm.build.uk_runtime.etb_vat import UK_ETB_VAT_PREDICTORS from microcosm.build.uk_runtime.frs_brma import UK_BRMA_PREDICTORS from microcosm.build.uk_runtime.frs_education_grants import ( FRS_EDUCATION_GRANT_REWRITES, @@ -482,6 +563,12 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: from microcosm.build.uk_runtime.frs_take_up import ( UK_TAKE_UP_ANCHOR_AGGREGATES, ) + from microcosm.build.uk_runtime.lcfs_consumption import ( + UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS, + UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS, + UK_LCFS_CONSUMPTION_PREDICTORS, + UK_LCFS_HAS_FUEL_PREDICTORS, + ) from microcosm.build.uk_runtime.was_wealth import ( UK_WAS_ENGINE_PREDICTORS, UK_WAS_WEALTH_PREDICTORS, @@ -517,6 +604,40 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: tuple(stages["was_wealth"].operations[2].parameters["predictors"]) == UK_WAS_WEALTH_PREDICTORS ) + lcfs = stages["lcfs_consumption"] + lcfs_ops = {op.kind: op for op in lcfs.operations} + assert ( + tuple(lcfs_ops["bridge_donor_column_via_qrf"].parameters["predictors"]) + == UK_LCFS_HAS_FUEL_PREDICTORS + ) + assert ( + tuple( + lcfs_ops["materialize_rules_engine_predictors"].parameters[ + "predictors" + ] + ) + == UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS + ) + assert ( + tuple(lcfs_ops["fit_weighted_qrf_chain"].parameters["predictors"]) + == UK_LCFS_CONSUMPTION_PREDICTORS + ) + assert ( + tuple(lcfs_ops["fit_weighted_qrf_chain"].parameters["targets"]) + == UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS[:-1] + ) + assert ( + tuple(stages["etb_vat"].operations[1].parameters["predictors"]) + == UK_ETB_VAT_PREDICTORS + ) + assert ( + tuple(stages["etb_services"].operations[1].parameters["predictors"]) + == UK_ETB_SERVICES_PREDICTORS + ) + assert ( + tuple(stages["etb_services"].operations[2].parameters["targets"]) + == UK_ETB_SERVICES_OUTPUT_COLUMNS[:3] + ) rate_keys = [ op.parameters["rate_key"] for stage_name in ( @@ -572,19 +693,30 @@ def test_e5_qrf_operation_declares_integer_seed(self) -> None: assert qrf.kind == "fit_weighted_qrf_chain" assert qrf.parameters["seed"] == 0 + def test_e6_declared_seed_lockstep(self) -> None: + spec = load_country_spec("uk") + stages = {stage.stage: stage for stage in spec.sources.stages} + + lcfs_seeded = { + op.kind: op.parameters["seed"] + for op in stages["lcfs_consumption"].operations + if "seed" in op.parameters + } + assert lcfs_seeded == { + "bridge_donor_column_via_qrf": 0, + "assign_binary_from_rate": 0, + "fit_weighted_qrf_chain": 0, + } + assert stages["etb_vat"].operations[2].parameters["seed"] == 0 + assert stages["etb_services"].operations[2].parameters["seed"] == 0 + def test_e7_declared_seed_lockstep(self) -> None: spec = load_country_spec("uk") stages = {stage.stage: stage for stage in spec.sources.stages} - assert ( - stages["spi_support_channel"].operations[0].parameters["seed"] == 42 - ) - assert ( - stages["hmrc_spi_income_spine"].operations[2].parameters["seed"] == 42 - ) - assert ( - stages["hmrc_spi_income_spine"].operations[3].parameters["seed"] == 43 - ) + assert stages["spi_support_channel"].operations[0].parameters["seed"] == 42 + assert stages["hmrc_spi_income_spine"].operations[2].parameters["seed"] == 42 + assert stages["hmrc_spi_income_spine"].operations[3].parameters["seed"] == 43 def test_full_uk_source_stage_plan_compiles_with_e4_stages(self) -> None: spec = load_country_spec("uk") diff --git a/packages/microcosm-build/tests/test_uk_spi_spine.py b/packages/microcosm-build/tests/test_uk_spi_spine.py index 8b1f33e4..7543f26c 100644 --- a/packages/microcosm-build/tests/test_uk_spi_spine.py +++ b/packages/microcosm-build/tests/test_uk_spi_spine.py @@ -26,6 +26,7 @@ from microcosm.build.uk_runtime.spi_support import ( HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, SPI_SYNTHETIC_SUPPORT_CHANNEL, + UKSPISupportResult, build_uk_spi_support_channel, support_channel_column, ) @@ -392,9 +393,9 @@ def test_reviewed_absent_incapacity_signal_raises(tmp_path: Path) -> None: lambda person, spi_people, build_period: person, ) support = build_uk_spi_support_channel( - person=_base_frame().table("person").assign( - incapacity_benefit_reported=[1.0, 0.0] - ), + person=_base_frame() + .table("person") + .assign(incapacity_benefit_reported=[1.0, 0.0]), benunit=_base_frame().table("benunit"), household=pd.DataFrame( { @@ -438,7 +439,10 @@ def _with_mutated_operation( ) -> SourceStageSpec: operations = [] for operation in stage.operations: - payload: dict[str, object] = {"kind": operation.kind, **dict(operation.parameters)} + payload: dict[str, object] = { + "kind": operation.kind, + **dict(operation.parameters), + } if operation.kind == kind: payload.update(overrides) operations.append(payload) @@ -509,6 +513,31 @@ def test_support_stage_parameters_accept_the_committed_manifest() -> None: assert len(declarations) == 1 +def test_support_transform_refuses_missing_builder_weight_kind(monkeypatch) -> None: + frame = _base_frame() + + def _stub_builder(*_args, **_kwargs) -> UKSPISupportResult: + return UKSPISupportResult( + person=frame.table("person").copy(), + benunit=frame.table("benunit").copy(), + household=frame.table("household").copy(), + id_multiplier=1, + spi_household_ids=(), + household_weight_kind=None, + ) + + monkeypatch.setattr( + "microcosm.build.uk_runtime.spi_spine.build_uk_spi_support_channel", + _stub_builder, + ) + transform = UKSPISupportChannelStageTransform( + stage=_committed_stage("spi_support_channel") + ) + + with pytest.raises(ValueError, match="importance household weights"): + transform(frame) + + def test_support_stage_parameters_refuse_gate_declaration_drift() -> None: committed = _committed_stage("spi_support_channel") gate = next( diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 92273382..2bdd2f18 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -275,6 +275,7 @@ "failing_targets", } ), + "aggregate_vs_admin": frozenset({"anchors_checked"}), "input_mass_parity": frozenset( { "candidate_name", @@ -343,13 +344,13 @@ # fingerprint derives from the manifest digest. Editing the spec moves all # three here in the same reviewed change. _UK_GATE_BATTERY_POLICY_SHA256 = ( - "5ddd3da9a52b0dc19ba1c97315f0e4f8acdedf2b74ea29bff512cbdb57de1cab" + "91ba70060b87eeca1e35d2aebe2ad79da61e33105b8f1352a2a05846e0780d4b" ) _UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "8d58fffe5e6542a7f10578076bbcc943cf587f9f22017ccc630450007b5b6166" + "4092f5012cddc4c878ea3a727c09f23212475ca246a4555ecea6f39219656a98" ) _UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "b7b645deee1b15750403f98a4c7dac09d6e08440a878b8d1ff83a15e9195b809" + "59f050a3a1ef1364107140083d548a873a9494b11472d4e4fd2a86f64ea8bb6b" ) #: Spec entry id -> the legacy gate name whose observable detail checks #: apply unchanged (the battery re-keys the report by entry id; the gate @@ -363,6 +364,7 @@ "uk_weights_audit": "weights_audit", "uk_nonnegative_columns": "nonnegative_columns", "uk_support": "support", + "uk_aggregate_admin": "aggregate_vs_admin", "uk_export_surface": "export_surface", "uk_take_up_signal": "take_up_signal", "uk_brma_enum_domain": "enum_domain", @@ -389,6 +391,7 @@ "uk_weights_audit": ("weights_audit", "terminal"), "uk_nonnegative_columns": ("nonnegative_columns", "terminal"), "uk_support": ("support", "terminal"), + "uk_aggregate_admin": ("aggregate_admin", "terminal"), "uk_export_surface": ("export_surface", "terminal"), "uk_take_up_signal": ("take_up_signal", "terminal"), "uk_brma_enum_domain": ("enum_domain", "terminal"), diff --git a/packages/microcosm-data/tests/test_contract.py b/packages/microcosm-data/tests/test_contract.py index 1c92fd75..77ddad51 100644 --- a/packages/microcosm-data/tests/test_contract.py +++ b/packages/microcosm-data/tests/test_contract.py @@ -103,7 +103,8 @@ "between adjacent seeds, the same realization-variance class " "the archived incumbent data repo records at uk-data#448 (4.6x " "Wales swing across releases). Register parity at this grain " - "is not meaningful until the whole-spine comparison; the " + "is not meaningful " + "until the whole-spine comparison; the " "one-month expiry enforces the end-of-workstream revisit " "registered on microcosm#145 (winsorised donor or separate " "land imputation are the candidate remedies)." @@ -143,13 +144,13 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: UK_GATE_BATTERY_PRODUCER = "microcosm.build.gate_battery" UK_GATE_BATTERY_SIGNING_KEY_ENV = "MICROCOSM_UK_TERMINAL_GATE_SIGNING_KEY" UK_GATE_BATTERY_POLICY_SHA256 = ( - "5ddd3da9a52b0dc19ba1c97315f0e4f8acdedf2b74ea29bff512cbdb57de1cab" + "91ba70060b87eeca1e35d2aebe2ad79da61e33105b8f1352a2a05846e0780d4b" ) UK_GATE_BATTERY_GATES_MANIFEST_SHA256 = ( - "8d58fffe5e6542a7f10578076bbcc943cf587f9f22017ccc630450007b5b6166" + "4092f5012cddc4c878ea3a727c09f23212475ca246a4555ecea6f39219656a98" ) UK_GATE_BATTERY_SPEC_FINGERPRINT = ( - "b7b645deee1b15750403f98a4c7dac09d6e08440a878b8d1ff83a15e9195b809" + "59f050a3a1ef1364107140083d548a873a9494b11472d4e4fd2a86f64ea8bb6b" ) UK_GATE_BATTERY_DEGENERATE_EVIDENCE_SHA256 = ( "d0d024043132fa07c378c393dbe2b24fe99bf19e876bcc39997d2c80cc9bd4f6" @@ -185,6 +186,7 @@ def _trusted_terminal_gate_signing_key(monkeypatch) -> None: "nonnegative_columns", ), "uk_support": ("support", "terminal", "support"), + "uk_aggregate_admin": ("aggregate_admin", "terminal", "aggregate_vs_admin"), "uk_export_surface": ("export_surface", "terminal", "export_surface"), "uk_take_up_signal": ("take_up_signal", "terminal", "take_up_signal"), "uk_brma_enum_domain": ("enum_domain", "terminal", "enum_domain"), @@ -771,6 +773,8 @@ def _terminal_gate_details(name: str) -> dict: } if name == "support": return {"columns_checked": 13} + if name == "aggregate_vs_admin": + return {"anchors_checked": 3} if name == "export_surface": return { "candidate_columns": 1, diff --git a/tools/build_uk_e6_support_bounds.py b/tools/build_uk_e6_support_bounds.py new file mode 100644 index 00000000..b1b51a32 --- /dev/null +++ b/tools/build_uk_e6_support_bounds.py @@ -0,0 +1,174 @@ +"""Build disclosure-safe E6 UK support bounds from pinned donor tabs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path + +import pandas as pd + +from microcosm.build.uk_runtime.etb_services import clean_etb_services_table +from microcosm.build.uk_runtime.etb_services import ( + donor_realized_ranges as etb_services_ranges, +) +from microcosm.build.uk_runtime.etb_vat import clean_etb_vat_table +from microcosm.build.uk_runtime.etb_vat import ( + donor_realized_ranges as etb_vat_ranges, +) +from microcosm.build.uk_runtime.lcfs_consumption import clean_lcfs_consumption_table +from microcosm.build.uk_runtime.lcfs_consumption import ( + donor_realized_ranges as lcfs_ranges, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +UK_PACKAGE = REPO_ROOT / "packages/microcosm-build/src/microcosm/build/uk" + + +def build_lcfs_support_bounds( + household_tab: Path, person_tab: Path +) -> dict[str, object]: + hh_sha = _sha256(household_tab) + person_sha = _sha256(person_tab) + donor = clean_lcfs_consumption_table( + pd.read_csv(person_tab, sep="\t", low_memory=False), + pd.read_csv(household_tab, sep="\t", low_memory=False), + ) + return _payload( + source={ + "ukds_study_number": 9468, + "doi": "10.5255/UKDA-SN-9468-3", + "household_tab_sha256": hh_sha, + "person_tab_sha256": person_sha, + }, + bounds=lcfs_ranges(donor), + label="LCFS consumption", + ) + + +def build_etb_vat_support_bounds(etb_tab: Path) -> dict[str, object]: + sha = _sha256(etb_tab) + donor = clean_etb_vat_table(pd.read_csv(etb_tab, sep="\t", low_memory=False)) + return _payload( + source={ + "ukds_study_number": 8856, + "doi": "10.5255/UKDA-SN-8856-4", + "tab_sha256": sha, + }, + bounds=etb_vat_ranges(donor), + label="ETB VAT", + ) + + +def build_etb_services_support_bounds(etb_tab: Path) -> dict[str, object]: + sha = _sha256(etb_tab) + donor = clean_etb_services_table(pd.read_csv(etb_tab, sep="\t", low_memory=False)) + return _payload( + source={ + "ukds_study_number": 8856, + "doi": "10.5255/UKDA-SN-8856-4", + "tab_sha256": sha, + }, + bounds=etb_services_ranges(donor), + label="ETB services", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--lcfs-hh-tab", type=Path) + parser.add_argument("--lcfs-person-tab", type=Path) + parser.add_argument("--etb-tab", type=Path) + parser.add_argument("--check", action="store_true") + args = parser.parse_args(argv) + jobs: list[tuple[Path, dict[str, object]]] = [] + if args.lcfs_hh_tab or args.lcfs_person_tab: + if not args.lcfs_hh_tab or not args.lcfs_person_tab: + raise SystemExit("LCFS support bounds require both LCFS tabs.") + jobs.append( + ( + UK_PACKAGE / "lcfs_consumption_support_bounds.json", + build_lcfs_support_bounds(args.lcfs_hh_tab, args.lcfs_person_tab), + ) + ) + if args.etb_tab: + jobs.extend( + [ + ( + UK_PACKAGE / "etb_vat_support_bounds.json", + build_etb_vat_support_bounds(args.etb_tab), + ), + ( + UK_PACKAGE / "etb_services_support_bounds.json", + build_etb_services_support_bounds(args.etb_tab), + ), + ] + ) + if not jobs: + raise SystemExit("No support-bound inputs supplied.") + for path, payload in jobs: + rendered = json.dumps(payload, indent=2, sort_keys=False) + "\n" + if args.check: + if path.read_text(encoding="utf-8") != rendered: + raise SystemExit(f"{path} is stale.") + else: + path.write_text(rendered, encoding="utf-8") + return 0 + + +def _payload( + *, + source: dict[str, object], + bounds: dict[str, tuple[float, float]], + label: str, +) -> dict[str, object]: + return { + "version": 1, + "country": "uk", + "policy": ( + f"Disclosure-safe outward-rounded {label} support bounds generated " + "from pinned licensed donor tabs. Values are rounded outward to " + "one significant figure; exact donor min/max values are not committed." + ), + "source": { + **source, + "sdc_treatment": ( + "Exact donor min/max values are rounded outward to one " + "significant figure before commit." + ), + }, + "bounds": { + column: list(_outward_round(pair)) + for column, pair in sorted(bounds.items()) + }, + "chronicle": [f"Support bounds generated for {label} from source SHA pins."], + } + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _outward_round(bounds: tuple[float, float]) -> tuple[float, float]: + lo, hi = bounds + return (_round_down(lo), _round_up(hi)) + + +def _round_down(value: float) -> float: + if value == 0: + return 0.0 + magnitude = 10 ** math.floor(math.log10(abs(value))) + return math.floor(value / magnitude) * magnitude + + +def _round_up(value: float) -> float: + if value == 0: + return 0.0 + magnitude = 10 ** math.floor(math.log10(abs(value))) + return math.ceil(value / magnitude) * magnitude + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index d0ce3267..e3a0f57a 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -28,6 +28,8 @@ sha256_argument, write_error_receipt, ) +from microcosm.build.uk_runtime.etb_services import UKETBServicesStageTransform +from microcosm.build.uk_runtime.etb_vat import UKETBVATStageTransform from microcosm.build.uk_runtime.frs_brma import UKFRSBRMAStageTransform from microcosm.build.uk_runtime.frs_council_tax import UKFRSCouncilTaxStageTransform from microcosm.build.uk_runtime.frs_disability import UKFRSDisabilityStageTransform @@ -50,6 +52,9 @@ ) from microcosm.build.uk_runtime.frs_take_up import UKFRSTakeUpStageTransform from microcosm.build.uk_runtime.hmrc_replay import write_hmrc_replay_report +from microcosm.build.uk_runtime.lcfs_consumption import ( + UKLCFSConsumptionStageTransform, +) from microcosm.build.uk_runtime.national_build import write_uk_national_frame from microcosm.build.uk_runtime.national_frame import uk_household_weight_kind from microcosm.build.uk_runtime.regional_uprating import ( @@ -81,6 +86,9 @@ "frs_brma", "was_wealth", "regional_property_uprating", + "lcfs_consumption", + "etb_vat", + "etb_services", "frs_hmrc_spine_leaves", "spi_support_channel", "hmrc_spi_income_spine", @@ -129,6 +137,21 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=Path, help="Caller-supplied private WAS round-8 household tab for was_wealth.", ) + parser.add_argument( + "--lcfs-hh-tab", + type=Path, + help="Caller-supplied private LCFS 2023-24 household tab for lcfs_consumption.", + ) + parser.add_argument( + "--lcfs-person-tab", + type=Path, + help="Caller-supplied private LCFS 2023-24 person tab for lcfs_consumption.", + ) + parser.add_argument( + "--etb-tab", + type=Path, + help="Caller-supplied private ETB 1977-2024 household tab for ETB stages.", + ) parser.add_argument( "--emit-nonzero-shares", type=Path, @@ -263,8 +286,7 @@ def _input_artifact_pins(stages) -> dict[str, dict[str, object]]: } if role in pins and pins[role] != pin: raise ValueError( - f"input artifact role {role!r} has inconsistent pins " - "across stages." + f"input artifact role {role!r} has inconsistent pins across stages." ) pins[role] = pin return dict(sorted(pins.items())) @@ -321,8 +343,18 @@ def _declared_seeds(stages) -> dict[str, dict[str, int]]: stage_seeds["stage1"] = seed elif operation.kind == "fit_weighted_qrf_stage2": stage_seeds["stage2"] = seed + elif operation.kind == "bridge_donor_column_via_qrf": + stage_seeds["bridge_donor_column_via_qrf"] = seed + elif operation.kind == "assign_binary_from_rate": + target = operation.parameters.get("target") + if isinstance(target, str): + stage_seeds[target] = seed + else: + stage_seeds["assign_binary_from_rate"] = seed elif operation.kind == "fit_weighted_qrf_chain": stage_seeds[stage.stage] = seed + elif operation.kind == "fit_weighted_qrf": + stage_seeds[stage.stage] = seed if stage_seeds: declared[stage.stage] = stage_seeds return declared @@ -465,6 +497,27 @@ def main(argv: list[str] | None = None) -> int: raise ValueError( "--was-tab is required when the was_wealth stage is scheduled." ) + if "lcfs_consumption" in stage_names: + missing_lcfs = [ + flag + for flag, value in ( + ("--lcfs-hh-tab", args.lcfs_hh_tab), + ("--lcfs-person-tab", args.lcfs_person_tab), + ("--was-tab", args.was_tab), + ) + if value is None + ] + if missing_lcfs: + raise ValueError( + "lcfs_consumption requires caller-supplied private inputs: " + f"{', '.join(missing_lcfs)}." + ) + if ( + "etb_vat" in stage_names or "etb_services" in stage_names + ) and args.etb_tab is None: + raise ValueError( + "--etb-tab is required when etb_vat or etb_services is scheduled." + ) stages = [stages_by_name[name] for name in stage_names] artifact_pins = _artifact_pins(stages) resource_pins = _resource_pins(stages, spec) @@ -555,6 +608,26 @@ def main(argv: list[str] | None = None) -> int: stage=stages_by_name["regional_property_uprating"], ) ) + if "lcfs_consumption" in stage_names: + implementations["lcfs_consumption"] = UKLCFSConsumptionStageTransform( + stage=stages_by_name["lcfs_consumption"], + engine=engine, + lcfs_hh_tab_path=args.lcfs_hh_tab, + lcfs_person_tab_path=args.lcfs_person_tab, + was_tab_path=args.was_tab, + ) + if "etb_vat" in stage_names: + implementations["etb_vat"] = UKETBVATStageTransform( + stage=stages_by_name["etb_vat"], + engine=engine, + etb_tab_path=args.etb_tab, + ) + if "etb_services" in stage_names: + implementations["etb_services"] = UKETBServicesStageTransform( + stage=stages_by_name["etb_services"], + engine=engine, + etb_tab_path=args.etb_tab, + ) implementations["frs_hmrc_spine_leaves"] = UKFRSHMRCSpineLeavesStageTransform( args.frs_raw_dir, stage=stages_by_name["frs_hmrc_spine_leaves"], diff --git a/tools/build_uk_release_input_coverage_manifest.py b/tools/build_uk_release_input_coverage_manifest.py index b037486e..557eb7bd 100644 --- a/tools/build_uk_release_input_coverage_manifest.py +++ b/tools/build_uk_release_input_coverage_manifest.py @@ -804,6 +804,18 @@ def build_manifest( stage_name="regional_property_uprating", candidate_source=candidate_source, ), + "lcfs_consumption": _source_stage_family_coverage_contract( + stage_name="lcfs_consumption", + candidate_source=candidate_source, + ), + "etb_vat": _source_stage_family_coverage_contract( + stage_name="etb_vat", + candidate_source=candidate_source, + ), + "etb_services": _source_stage_family_coverage_contract( + stage_name="etb_services", + candidate_source=candidate_source, + ), }, "derivation": ( "Surface = efrs_parity_reference.json populated effective loader " From d45911ba05e87363aa3460ad6542f6e7e8a93f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:35:47 +0200 Subject: [PATCH 02/10] Add the e6 identity receipt to verify_uk_identity_stability Covers the domestic-energy fold, rail_usage ratio, petrol/diesel zeroing idempotence, and the NHS age-gender allocation recomputed from the committed resource, under row permutation keyed by entity id and scoped to the FRS spine rows (stacked SPI clones are E7's receipt surface). Co-Authored-By: Claude Fable 5 --- tools/verify_uk_identity_stability.py | 154 +++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 2 deletions(-) diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 477f8707..0b667a25 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -328,11 +328,153 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: } +def e6_identity_receipt( + frame, + *, + permutation_seed: int, +) -> dict[str, object]: + """Receipt E6 deterministic layers under row permutation by entity id. + + Covered: the domestic-energy fold (elec + gas), the rail_usage ratio, + petrol/diesel zeroing idempotence for non-fuel households, and the NHS + age-gender person allocation recomputed from the committed resource. + The QRF chain draws and the NEED raking outcome are covered by + twin-build determinism and the aggregate_admin NEED-margin receipt + respectively (raking inputs are consumed by the stage and are not + reconstructible from the artifact). + """ + + from microcosm.build.uk_runtime.etb_services import ( + RAIL_FARE_INDEX_2023, + allocate_nhs_by_age_gender, + ) + + def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: + del benunit_t + household = household_t.copy() + household_out = pd.DataFrame(index=household_t["household_id"].to_numpy()) + person_out = pd.DataFrame(index=person_t["person_id"].to_numpy()) + if {"electricity_consumption", "gas_consumption"} <= set(household.columns): + household_out["domestic_energy_consumption"] = household[ + "electricity_consumption" + ].to_numpy(dtype=float) + household["gas_consumption"].to_numpy( + dtype=float + ) + if "rail_subsidy_spending" in household.columns: + household_out["rail_usage"] = ( + household["rail_subsidy_spending"].to_numpy(dtype=float) + / RAIL_FARE_INDEX_2023 + ) + if {"has_fuel_consumption", "petrol_spending", "diesel_spending"} <= set( + household.columns + ): + no_fuel = household["has_fuel_consumption"].to_numpy(dtype=float) == 0.0 + for column in ("petrol_spending", "diesel_spending"): + household_out[column] = np.where( + no_fuel, 0.0, household[column].to_numpy(dtype=float) + ) + if {"age", "gender"} <= set(person_t.columns): + nhs = allocate_nhs_by_age_gender( + person_t, + household_weights=household["household_weight"].to_numpy(dtype=float), + household=household, + nhs_table=None, + ) + for column in nhs.columns: + person_out[column] = nhs[column].to_numpy(dtype=float) + return {"household": household_out, "person": person_out} + + person = frame.table("person") + benunit = frame.table("benunit") + household = frame.table("household").copy() + household["household_weight"] = frame.weights_for("household").values + # Scope to the FRS spine rows: the SPI channel stages stack synthetic + # households AFTER the E6 stages ran (clones inherit their donors' + # consumption/services values), so the deterministic-layer identity + # claims apply to the population the consumption stages actually saw. + # The stacked rows are E7's receipt surface, not E6's. + if "household_is_spi_synthetic" in household.columns: + spine_mask = ~household["household_is_spi_synthetic"].astype(bool) + household = household.loc[spine_mask].reset_index(drop=True) + spine_household_ids = set(household["household_id"].tolist()) + person = person.loc[ + person["person_household_id"].isin(spine_household_ids) + ].reset_index(drop=True) + original = recompute(person, benunit, household) + rng = np.random.default_rng(permutation_seed) + permuted = recompute( + person.iloc[rng.permutation(len(person))].reset_index(drop=True), + benunit.iloc[rng.permutation(len(benunit))].reset_index(drop=True), + household.iloc[rng.permutation(len(household))].reset_index(drop=True), + ) + # The fold, ratio, and zeroing layers are order-independent elementwise + # arithmetic on stored columns: bitwise under permutation and against + # the store. The NHS layer's cell normalization sums weights per + # (age-band, gender) cell, so permutation changes float summation + # order: one rounding generation of tolerance applies, and the same + # tolerance covers the stored cross-check. + nhs_columns = tuple(original["person"].columns) + tolerances = {("person", column): (1e-12, 1e-9) for column in nhs_columns} + stored_tolerances = dict(tolerances) + mismatches: dict[str, list[str]] = {} + stored_mismatches: dict[str, list[str]] = {} + stored_tables = {"household": household, "person": person} + for entity, values in original.items(): + for column in values.columns: + rtol, atol = tolerances.get((entity, column), (0.0, 0.0)) + left = values[column] + right = permuted[entity][column].reindex(left.index) + if not np.allclose( + left.to_numpy(dtype=float), + right.to_numpy(dtype=float), + rtol=rtol, + atol=atol, + ): + mismatches.setdefault(entity, []).append(column) + stored_table = stored_tables[entity] + if column in stored_table.columns: + stored_rtol, stored_atol = stored_tolerances.get( + (entity, column), (rtol, atol) + ) + if not np.allclose( + left.to_numpy(dtype=float), + stored_table[column].to_numpy(dtype=float), + rtol=stored_rtol, + atol=stored_atol, + ): + stored_mismatches.setdefault(entity, []).append(column) + return { + "check": "uk_e6_identity_stability", + "permutation_seed": permutation_seed, + "identical_under_permutation": not mismatches, + "permutation_mismatches": mismatches, + "matches_stored_columns": not stored_mismatches, + "stored_column_mismatches": stored_mismatches, + "tolerance_policy": ( + "permutation and stored-column: bitwise for the domestic-energy " + "fold, the rail_usage ratio, and petrol/diesel zeroing " + "(order-independent elementwise arithmetic); rtol 1e-12 / " + "atol 1e-9 for the NHS allocation (cell weight sums cost one " + "float rounding generation under reordering)" + ), + "columns_by_entity": { + entity: list(values.columns) for entity, values in original.items() + }, + "qrf_draw_columns_scope": ( + "excluded: seeded-stream QRF draws are covered by twin-build " + "determinism; the NEED raking outcome is covered by the " + "aggregate_admin NEED-margin receipt (raking inputs are " + "consumed by the stage and not reconstructible from the " + "artifact)" + ), + } + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-h5", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--check", choices=("e4", "e5"), default="e4") + parser.add_argument("--check", choices=("e4", "e5", "e6"), default="e4") parser.add_argument("--permutation-seed", type=int, default=123) args = parser.parse_args() @@ -355,7 +497,7 @@ def main() -> int: ok = bool( receipt["identical_under_permutation"] and receipt["matches_stored_columns"] ) - else: + elif args.check == "e5": receipt = e5_identity_receipt( frame, permutation_seed=args.permutation_seed, @@ -363,6 +505,14 @@ def main() -> int: ok = bool( receipt["identical_under_permutation"] and receipt["matches_stored_columns"] ) + else: + receipt = e6_identity_receipt( + frame, + permutation_seed=args.permutation_seed, + ) + ok = bool( + receipt["identical_under_permutation"] and receipt["matches_stored_columns"] + ) receipt["input_h5"] = str(args.input_h5) args.output.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") print( From 38aa4667952d4ba692ab6baad211a1c291ffb863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:44:13 +0200 Subject: [PATCH 03/10] Fix the has-fuel bridge's licensed-build crash: LCFS predictor renames and the fitted-QRF predict API The licensed twin build surfaced two defects PR CI could not see (the bridge had no direct unit coverage): the LCFS donor frame carries hbai_household_net_income / is_adult / is_child, which the bridge must rename to the WAS predictor names before predicting (incumbent consumption.py:556-574), and encode_qrf_predictor_pair hardcoded the WAS wealth predictor list (now parameterized, default unchanged). Also FittedRegimeGatedQRF exposes predict, not draw. Adds a native-name regression test that executes the bridge end to end. Co-Authored-By: Claude Fable 5 --- .../build/uk_runtime/lcfs_consumption.py | 15 +++++- .../microcosm/build/uk_runtime/was_wealth.py | 11 +++-- .../tests/test_uk_lcfs_consumption.py | 48 +++++++++++++++++++ 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 69305208..6b284c04 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -118,6 +118,13 @@ "self_employment_income", "region", ) +# LCFS-native names for the three bridge predictors the WAS donor names +# differently (incumbent consumption.py:556-574). +LCFS_TO_WAS_HAS_FUEL_RENAMES = { + "hbai_household_net_income": "household_net_income", + "is_adult": "num_adults", + "is_child": "num_children", +} UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS = ( "is_adult", "is_child", @@ -392,9 +399,13 @@ def bridge_has_fuel_to_lcfs( < NTS_ICE_SHARE ) ).astype(float) + # The LCFS frame carries its own names for three of the WAS bridge + # predictors (incumbent consumption.py:556-574 renames before predicting). + recipient = lcfs.rename(columns=LCFS_TO_WAS_HAS_FUEL_RENAMES) donor_encoded, recipient_encoded, predictors = encode_qrf_predictor_pair( donor[[*UK_LCFS_HAS_FUEL_PREDICTORS, "has_fuel_consumption", "weight"]], - lcfs[list(UK_LCFS_HAS_FUEL_PREDICTORS)], + recipient[list(UK_LCFS_HAS_FUEL_PREDICTORS)], + predictors=UK_LCFS_HAS_FUEL_PREDICTORS, ) model = RegimeGatedQRF(n_estimators=n_estimators, seed=seed) result = model.fit( @@ -402,7 +413,7 @@ def bridge_has_fuel_to_lcfs( list(predictors), ["has_fuel_consumption"], weights="weight", - ).draw(recipient_encoded) + ).predict(recipient_encoded) out = lcfs.copy() out["has_fuel_consumption"] = np.clip( np.asarray(result["has_fuel_consumption"], dtype=float), 0.0, 1.0 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py index 5a5b9c44..9c2dc72b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/was_wealth.py @@ -428,17 +428,22 @@ def run_segment(base_predictors: Sequence[str], targets: Sequence[str]) -> None: def encode_qrf_predictor_pair( - donor: pd.DataFrame, recipient: pd.DataFrame + donor: pd.DataFrame, + recipient: pd.DataFrame, + *, + predictors: Sequence[str] = UK_WAS_WEALTH_PREDICTORS, ) -> tuple[pd.DataFrame, pd.DataFrame, tuple[str, ...]]: """One-hot the region predictor jointly across donor and recipient. Mirrors the SPI stage's paired dummy encoding and the incumbent's dummy-encoded region. Donor rows with an unmapped region code (the - incumbent's absent GOR code 3) become all-zero dummy rows. + incumbent's absent GOR code 3) become all-zero dummy rows. The + predictor list defaults to the WAS wealth set; the E6 has-fuel bridge + passes its own. """ numeric_predictors = tuple( - predictor for predictor in UK_WAS_WEALTH_PREDICTORS if predictor != "region" + predictor for predictor in predictors if predictor != "region" ) combined_region = pd.concat( [ diff --git a/packages/microcosm-build/tests/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py index 732b8af3..27a16336 100644 --- a/packages/microcosm-build/tests/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py @@ -156,3 +156,51 @@ def test_support_clip_exempts_raked_energy_columns() -> None: 5.0, ] assert clipped["electricity_consumption"].tolist() == [0.0, 10.0] + + +def test_has_fuel_bridge_accepts_lcfs_native_predictor_names() -> None: + # Regression for the licensed-build failure: the LCFS donor frame carries + # hbai_household_net_income / is_adult / is_child, not the WAS names the + # bridge model is fit on. The bridge must rename before predicting. + from microcosm.build.uk_runtime.lcfs_consumption import ( + UK_LCFS_HAS_FUEL_PREDICTORS, + bridge_has_fuel_to_lcfs, + ) + + rng = np.random.default_rng(7) + n = 120 + was = pd.DataFrame( + { + "household_net_income": rng.uniform(1e4, 6e4, n), + "num_adults": rng.integers(1, 4, n).astype(float), + "num_children": rng.integers(0, 3, n).astype(float), + "private_pension_income": rng.uniform(0, 1e4, n), + "employment_income": rng.uniform(0, 5e4, n), + "self_employment_income": rng.uniform(0, 1e4, n), + "region": rng.choice(["LONDON", "WALES"], n), + "num_vehicles": rng.integers(0, 3, n).astype(float), + "weight": rng.uniform(0.5, 2.0, n), + } + ) + lcfs = pd.DataFrame( + { + "hbai_household_net_income": [2e4, 3e4, 4e4], + "is_adult": [1.0, 2.0, 3.0], + "is_child": [0.0, 1.0, 2.0], + "private_pension_income": [0.0, 1e3, 2e3], + "employment_income": [1e4, 2e4, 3e4], + "self_employment_income": [0.0, 0.0, 5e3], + "region": ["LONDON", "WALES", "LONDON"], + } + ) + assert not set(UK_LCFS_HAS_FUEL_PREDICTORS) <= set(lcfs.columns) + + first, record = bridge_has_fuel_to_lcfs(lcfs, was, seed=0, n_estimators=10) + second, _ = bridge_has_fuel_to_lcfs(lcfs, was, seed=0, n_estimators=10) + + assert record.fit_name.endswith("has_fuel") + values = first["has_fuel_consumption"].to_numpy(dtype=float) + assert ((values >= 0.0) & (values <= 1.0)).all() + assert first["has_fuel_consumption"].tolist() == ( + second["has_fuel_consumption"].tolist() + ) From a169071de1f380d51e37fca14e485a97852cb294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:46:19 +0200 Subject: [PATCH 04/10] e6 identity receipt: restore stage-time grossing scale for the NHS recompute The spi_support_channel stage scales survey-channel weights by (1 - share) after the E6 stages ran; the NHS allocation's budget normalization is absolute, so the receipt restores the declared share before recomputing. Co-Authored-By: Claude Fable 5 --- tools/verify_uk_identity_stability.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 0b667a25..01779003 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -392,7 +392,11 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: # households AFTER the E6 stages ran (clones inherit their donors' # consumption/services values), so the deterministic-layer identity # claims apply to the population the consumption stages actually saw. - # The stacked rows are E7's receipt surface, not E6's. + # The stacked rows are E7's receipt surface, not E6's. The + # spi_support_channel stage also scales the survey channel's weights by + # (1 - share) after the E6 stages ran; the NHS allocation's budget + # normalization is absolute, so restore the stage-time grossing scale + # from the declared share before recomputing. if "household_is_spi_synthetic" in household.columns: spine_mask = ~household["household_is_spi_synthetic"].astype(bool) household = household.loc[spine_mask].reset_index(drop=True) @@ -400,6 +404,26 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: person = person.loc[ person["person_household_id"].isin(spine_household_ids) ].reset_index(drop=True) + from importlib.resources import files as _files + + spec = json.loads( + _files("microcosm.build.uk") + .joinpath("source_stages.json") + .read_text(encoding="utf-8") + ) + channel = next( + (s for s in spec["stages"] if s["stage"] == "spi_support_channel"), + None, + ) + if channel is not None: + share = next( + op["share"] + for op in channel["operations"] + if op["kind"] == "allocate_zero_weight_prior_mass" + ) + household["household_weight"] = household[ + "household_weight" + ].to_numpy(dtype=float) / (1.0 - float(share)) original = recompute(person, benunit, household) rng = np.random.default_rng(permutation_seed) permuted = recompute( From 0f94cfabdc4b9a504886d30851ee7090942681c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:52:48 +0200 Subject: [PATCH 05/10] Fix etb_vat recipient predictors: aggregate person entities to household Second licensed-build crash in the PR-CI-blind class: is_adult / is_child / is_SP_age materialize at person grain (36,248 values) and were assigned directly to the household index (16,754). Mirrors the lcfs/services native-entity aggregation (the E5 review class); adds a native-grain fake-engine regression test that executes recipient_predictors. Co-Authored-By: Claude Fable 5 --- .../src/microcosm/build/uk_runtime/etb_vat.py | 24 +++++++- .../microcosm-build/tests/test_uk_etb_vat.py | 57 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py index 56b138a7..ca712a4b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py @@ -136,14 +136,36 @@ def clean_etb_vat_table( def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: + """Materialize ETB VAT recipient predictors at household grain. + + Predictors materialize at their native entity (is_adult / is_child / + is_SP_age are person booleans) and aggregate to household by + person_household_id — direct engine arrays would crash the licensed + build on the person/household length mismatch (the E5 review class). + """ + materialized = engine.materialize( frame, UK_ETB_VAT_PREDICTORS, uk_time_period(frame) ) household = frame.table("household") + person = frame.table("person") result = pd.DataFrame(index=household.index) for predictor in UK_ETB_VAT_PREDICTORS: + declared = str(engine.variable_metadata(predictor).entity) values = np.asarray(materialized[predictor]) - result[predictor] = values + if declared == "household": + result[predictor] = values + elif declared == "person": + summed = ( + pd.Series(values.astype(float)) + .groupby(person["person_household_id"].to_numpy()) + .sum() + ) + result[predictor] = ( + summed.reindex(household["household_id"]).fillna(0.0).to_numpy() + ) + else: + raise ValueError(f"unsupported ETB VAT predictor entity {declared!r}.") return result diff --git a/packages/microcosm-build/tests/test_uk_etb_vat.py b/packages/microcosm-build/tests/test_uk_etb_vat.py index 8f60b2bf..384f6924 100644 --- a/packages/microcosm-build/tests/test_uk_etb_vat.py +++ b/packages/microcosm-build/tests/test_uk_etb_vat.py @@ -104,3 +104,60 @@ def fit(self, donor, predictors, targets, *, weights): assert draws["full_rate_vat_expenditure_rate"].tolist() == [0.1] assert record.fit_name == UK_ETB_VAT_FIT_NAME assert record.weight_kind == "explicit" + + +def test_recipient_predictors_aggregate_person_entities_to_household() -> None: + # Regression for the licensed-build crash: is_adult / is_child / is_SP_age + # materialize at person grain and must aggregate to household — direct + # engine arrays fail on the person/household length mismatch. + from types import SimpleNamespace + + from microcosm.build.uk_runtime.etb_vat import recipient_predictors + from microcosm.build.uk_runtime.national_frame import uk_national_frame + + entities = { + "is_adult": "person", + "is_child": "person", + "is_SP_age": "person", + "household_net_income": "household", + } + values = { + "is_adult": np.array([1.0, 1.0, 0.0, 1.0]), + "is_child": np.array([0.0, 0.0, 1.0, 0.0]), + "is_SP_age": np.array([0.0, 1.0, 0.0, 0.0]), + "household_net_income": np.array([1e4, 2e4]), + } + + class _FakeEngine: + country = "uk" + + def variable_metadata(self, name): + return SimpleNamespace(entity=entities[name]) + + def materialize(self, frame, variables, period): + return {variable: values[variable] for variable in variables} + + person = pd.DataFrame( + { + "person_id": [1, 2, 3, 4], + "person_household_id": [10, 10, 10, 20], + "person_benunit_id": [100, 100, 100, 200], + } + ) + benunit = pd.DataFrame({"benunit_id": [100, 200], "benunit_household_id": [10, 20]}) + household = pd.DataFrame( + {"household_id": [10, 20], "household_weight": [1.0, 1.0]} + ) + frame = uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2023", + ) + + result = recipient_predictors(frame, _FakeEngine()) + + assert result["is_adult"].tolist() == [2.0, 1.0] + assert result["is_child"].tolist() == [1.0, 0.0] + assert result["is_SP_age"].tolist() == [1.0, 0.0] + assert result["household_net_income"].tolist() == [1e4, 2e4] From 102d0d7818cc48ed99123a97c8f0b006107fa1f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:58:22 +0200 Subject: [PATCH 06/10] Fix etb_services education counts: derive from current_education, not engine variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third licensed-build crash in the PR-CI-blind class: count_*_education are not policyengine-uk variables — the incumbent derives them from person current_education (etb.py:180-186). The transform now materializes the seven real engine variables and derives the three counts at person grain before household aggregation; the manifest declares the derivation under derived_predictors; native-grain regression test added. Co-Authored-By: Claude Fable 5 --- .../uk/release_input_coverage_manifest.json | 10 +-- .../src/microcosm/build/uk/source_stages.json | 13 ++-- .../build/uk_runtime/etb_services.py | 47 ++++++++++--- .../tests/test_uk_etb_services.py | 69 +++++++++++++++++++ .../tests/test_uk_source_stages.py | 16 ++++- 5 files changed, 134 insertions(+), 21 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index ffd194d0..c339b649 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -478,7 +478,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", + "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -497,7 +497,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", + "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -639,7 +639,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", + "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -659,7 +659,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", + "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -690,7 +690,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "6b2a523ba9ca9102560b29c723662908a99e0f7d4733f00c2636b814c92af432", + "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 92932406..6434924b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -1372,13 +1372,16 @@ "is_adult", "is_child", "is_SP_age", - "count_primary_education", - "count_secondary_education", - "count_further_education", "dla", "pip", - "hbai_household_net_income" - ] + "hbai_household_net_income", + "current_education" + ], + "derived_predictors": { + "count_primary_education": "current_education == PRIMARY", + "count_secondary_education": "current_education == LOWER_SECONDARY", + "count_further_education": "current_education in (UPPER_SECONDARY, TERTIARY)" + } }, { "kind": "fit_weighted_qrf_chain", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py index 8b3256c3..2b849a8d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -36,6 +36,22 @@ "pip", "hbai_household_net_income", ) +# The education counts are not engine variables: the incumbent derives them +# from person current_education (etb.py:180-186). Only these materialize. +UK_ETB_SERVICES_ENGINE_VARIABLES = ( + "is_adult", + "is_child", + "is_SP_age", + "dla", + "pip", + "hbai_household_net_income", + "current_education", +) +UK_ETB_SERVICES_EDUCATION_COUNTS = { + "count_primary_education": ("PRIMARY",), + "count_secondary_education": ("LOWER_SECONDARY",), + "count_further_education": ("UPPER_SECONDARY", "TERTIARY"), +} UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS = ( "dfe_education_spending", "rail_subsidy_spending", @@ -175,26 +191,39 @@ def household_grain_services_predictors(person_level: pd.DataFrame) -> pd.DataFr def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: + """Materialize ETB services recipient predictors at household grain. + + The three education counts derive from person current_education (the + incumbent's construction, etb.py:180-186) — they are not engine + variables. Everything else materializes at its native entity and + aggregates to household by person_household_id. + """ + materialized = engine.materialize( - frame, UK_ETB_SERVICES_PREDICTORS, uk_time_period(frame) + frame, UK_ETB_SERVICES_ENGINE_VARIABLES, uk_time_period(frame) ) household = frame.table("household") person = frame.table("person") + group_keys = person["person_household_id"].to_numpy() + household_ids = household["household_id"] + + def person_sum(values: np.ndarray) -> np.ndarray: + summed = pd.Series(values.astype(float)).groupby(group_keys).sum() + return summed.reindex(household_ids).fillna(0.0).to_numpy() + result = pd.DataFrame(index=household.index) + education = np.asarray(materialized["current_education"]).astype(str) for predictor in UK_ETB_SERVICES_PREDICTORS: + if predictor in UK_ETB_SERVICES_EDUCATION_COUNTS: + labels = UK_ETB_SERVICES_EDUCATION_COUNTS[predictor] + result[predictor] = person_sum(np.isin(education, labels)) + continue values = np.asarray(materialized[predictor]) entity = str(engine.variable_metadata(predictor).entity) if entity == "household": result[predictor] = values elif entity == "person": - summed = ( - pd.Series(values.astype(float)) - .groupby(person["person_household_id"].to_numpy()) - .sum() - ) - result[predictor] = ( - summed.reindex(household["household_id"]).fillna(0.0).to_numpy() - ) + result[predictor] = person_sum(values) else: raise ValueError(f"unsupported ETB services predictor entity {entity!r}.") return result diff --git a/packages/microcosm-build/tests/test_uk_etb_services.py b/packages/microcosm-build/tests/test_uk_etb_services.py index 3b88e2d5..80fae448 100644 --- a/packages/microcosm-build/tests/test_uk_etb_services.py +++ b/packages/microcosm-build/tests/test_uk_etb_services.py @@ -215,3 +215,72 @@ def test_nhs_age_parsing_and_85_plus_fold_in_uses_full_table_denominator() -> No cells["Per-person average spending"].mul(cells["Total people"]).sum(), NHS_BUDGET_2025_26, ) + + +def test_recipient_predictors_derive_education_counts_and_aggregate() -> None: + # Regression for the licensed-build crash: count_*_education are not + # engine variables — they derive from person current_education and + # aggregate to household, like the person-entity benefit predictors. + from types import SimpleNamespace + + import numpy as np + + from microcosm.build.uk_runtime.etb_services import recipient_predictors + from microcosm.build.uk_runtime.national_frame import uk_national_frame + + entities = { + "is_adult": "person", + "is_child": "person", + "is_SP_age": "person", + "dla": "person", + "pip": "person", + "hbai_household_net_income": "household", + "current_education": "person", + } + values = { + "is_adult": np.array([1.0, 1.0, 0.0, 1.0]), + "is_child": np.array([0.0, 0.0, 1.0, 0.0]), + "is_SP_age": np.array([0.0, 1.0, 0.0, 0.0]), + "dla": np.array([0.0, 100.0, 0.0, 0.0]), + "pip": np.array([50.0, 0.0, 0.0, 0.0]), + "hbai_household_net_income": np.array([1e4, 2e4]), + "current_education": np.array( + ["NOT_IN_EDUCATION", "TERTIARY", "PRIMARY", "LOWER_SECONDARY"] + ), + } + + class _FakeEngine: + country = "uk" + + def variable_metadata(self, name): + return SimpleNamespace(entity=entities[name]) + + def materialize(self, frame, variables, period): + return {variable: values[variable] for variable in variables} + + person = pd.DataFrame( + { + "person_id": [1, 2, 3, 4], + "person_household_id": [10, 10, 10, 20], + "person_benunit_id": [100, 100, 100, 200], + } + ) + benunit = pd.DataFrame({"benunit_id": [100, 200], "benunit_household_id": [10, 20]}) + household = pd.DataFrame( + {"household_id": [10, 20], "household_weight": [1.0, 1.0]} + ) + frame = uk_national_frame( + person=person, + benunit=benunit, + household=household, + time_period="2023", + ) + + result = recipient_predictors(frame, _FakeEngine()) + + assert result["count_primary_education"].tolist() == [1.0, 0.0] + assert result["count_secondary_education"].tolist() == [0.0, 1.0] + assert result["count_further_education"].tolist() == [1.0, 0.0] + assert result["is_SP_age"].tolist() == [1.0, 0.0] + assert result["dla"].tolist() == [100.0, 0.0] + assert result["hbai_household_net_income"].tolist() == [1e4, 2e4] diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index d413aa9e..25f8acc2 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -549,7 +549,6 @@ def test_e3_operation_kinds_are_declared_in_order(self) -> None: def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: from microcosm.build.uk_runtime.etb_services import ( UK_ETB_SERVICES_OUTPUT_COLUMNS, - UK_ETB_SERVICES_PREDICTORS, ) from microcosm.build.uk_runtime.etb_vat import UK_ETB_VAT_PREDICTORS from microcosm.build.uk_runtime.frs_brma import UK_BRMA_PREDICTORS @@ -630,9 +629,22 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: tuple(stages["etb_vat"].operations[1].parameters["predictors"]) == UK_ETB_VAT_PREDICTORS ) + from microcosm.build.uk_runtime.etb_services import ( + UK_ETB_SERVICES_EDUCATION_COUNTS, + UK_ETB_SERVICES_ENGINE_VARIABLES, + ) + assert ( tuple(stages["etb_services"].operations[1].parameters["predictors"]) - == UK_ETB_SERVICES_PREDICTORS + == UK_ETB_SERVICES_ENGINE_VARIABLES + ) + assert ( + tuple( + stages["etb_services"] + .operations[1] + .parameters["derived_predictors"] + ) + == tuple(UK_ETB_SERVICES_EDUCATION_COUNTS) ) assert ( tuple(stages["etb_services"].operations[2].parameters["targets"]) From 7b687edbeb3ee783ad5fa2bbabf7b114dcac07b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:02:38 +0200 Subject: [PATCH 07/10] Fix the NHS age parser for the committed resource's real labels Fourth licensed-build crash: banded labels read "01-04 years" and the parser never stripped the unit suffix (the incumbent slices the first five characters). The unit test had used bare synthetic labels; it now parses every label in the committed resource. Co-Authored-By: Claude Fable 5 --- .../build/uk_runtime/etb_services.py | 7 +++++-- .../tests/test_uk_nhs_allocation.py | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py index 2b849a8d..2dfc8d0b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -286,8 +286,11 @@ def parse_nhs_age_bounds(age_group: str) -> tuple[int, int]: return 0, 1 if age_group == "95 years or older": return 95, 120 - if "-" in age_group: - lo, hi = age_group.split("-", maxsplit=1) + # Banded labels read "01-04 years": strip the unit suffix before + # splitting (the incumbent slices the first five characters). + stripped = age_group.removesuffix(" years").strip() + if "-" in stripped: + lo, hi = stripped.split("-", maxsplit=1) return int(lo.strip()), int(hi.strip()) + 1 raise ValueError(f"unsupported NHS age group {age_group!r}") diff --git a/packages/microcosm-build/tests/test_uk_nhs_allocation.py b/packages/microcosm-build/tests/test_uk_nhs_allocation.py index 51933a1b..bc362355 100644 --- a/packages/microcosm-build/tests/test_uk_nhs_allocation.py +++ b/packages/microcosm-build/tests/test_uk_nhs_allocation.py @@ -81,3 +81,23 @@ def test_nhs_85_plus_fold_in_and_budget_normalization_use_full_table() -> None: assert allocated.loc[0, "nhs_a_and_e_spending"] < allocated.loc[ 1, "nhs_a_and_e_spending" ] + + +def test_nhs_age_bounds_parse_every_committed_resource_label() -> None: + # Regression for the licensed-build crash on "01-04 years": the parser + # must handle the committed resource's REAL labels, not just synthetic + # fixtures. + import json + from pathlib import Path + + resource = ( + Path(__file__).resolve().parents[1] + / "src/microcosm/build/uk/nhs_consumption_by_age_gender.json" + ) + payload = json.loads(resource.read_text(encoding="utf-8")) + labels = sorted({row["Age group"] for row in payload["rows"]}) + assert labels, "committed NHS resource has no rows" + bounds = [parse_nhs_age_bounds(label) for label in labels] + assert parse_nhs_age_bounds("01-04 years") == (1, 5) + for lower, upper in bounds: + assert 0 <= lower < upper <= 120 From 89d1c8cf08119b1259428b80ef43bda01ec003e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:06:56 +0200 Subject: [PATCH 08/10] Thread caller-supplied weights into the NHS cell table Fifth licensed-build crash: the frame keeps household weights in the typed vector, not as a table column, but the weighted person counts read household['household_weight']. allocate_nhs_by_age_gender now assigns the passed array onto the table; the test fixture drops the column to match the real frame. Co-Authored-By: Claude Fable 5 --- .../src/microcosm/build/uk_runtime/etb_services.py | 6 ++++++ packages/microcosm-build/tests/test_uk_nhs_allocation.py | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py index 2dfc8d0b..2e8f9abc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -345,6 +345,12 @@ def allocate_nhs_by_age_gender( / "uk/nhs_consumption_by_age_gender.json" ) nhs_table = pd.DataFrame(json.loads(path.read_text(encoding="utf-8"))["rows"]) + # The frame keeps weights in the typed vector, not as a table column — + # thread the caller-supplied weights onto the household table the + # weighted person counts read. + household = household.assign( + household_weight=np.asarray(household_weights, dtype=float) + ) cells = build_nhs_cell_table(nhs_table, person, household) output = pd.DataFrame(0.0, index=person.index, columns=UK_NHS_OUTPUT_COLUMNS) service_to_columns = { diff --git a/packages/microcosm-build/tests/test_uk_nhs_allocation.py b/packages/microcosm-build/tests/test_uk_nhs_allocation.py index bc362355..eeebe267 100644 --- a/packages/microcosm-build/tests/test_uk_nhs_allocation.py +++ b/packages/microcosm-build/tests/test_uk_nhs_allocation.py @@ -69,10 +69,12 @@ def test_nhs_85_plus_fold_in_and_budget_normalization_use_full_table() -> None: NHS_BUDGET_2025_26, ) + # The real frame's household table has no household_weight column (weights + # live in the typed vector): the allocation must run from the passed array. allocated = allocate_nhs_by_age_gender( person, household_weights=household["household_weight"].to_numpy(dtype=float), - household=household, + household=household.drop(columns=["household_weight"]), nhs_table=_raw_nhs_rows(), ) From 2bc98b3ee995108d69ea330c27c7ff2d13d2dce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:32:40 +0200 Subject: [PATCH 09/10] Implement the declared four-margin NEED raking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Licensed acceptance caught the gap: the manifest declares the incumbent's four-margin post-imputation rake (income -> tenure -> accommodation -> region, 50 iterations, weighted), but only the income margin was implemented — the incumbent artifact hits the tenure/accommodation NEED cells to ~1 percent while ours was off by up to ±32 percent. rake_energy_to_need now takes the three categorical groupers, builds their MarginSpecs from the committed NEED resource (one source of values for the raking and the aggregate_admin anchors), and sweeps the incumbent's order; unmapped categories stay exempt. Four-margin regression test added. Co-Authored-By: Claude Fable 5 --- .../build/uk_runtime/lcfs_consumption.py | 69 ++++++++++++++++-- .../tests/test_uk_lcfs_consumption.py | 72 +++++++++++++++++++ 2 files changed, 137 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 6b284c04..604f1756 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path @@ -253,6 +254,9 @@ def __call__(self, frame: Frame) -> Frame: household_draws = rake_energy_to_need( household_draws.join(recipient[["household_gross_income"]]), weights=frame.weights_for("household").values, + tenure=recipient["tenure_type"].astype(str).to_numpy(), + accommodation=recipient["accommodation_type"].astype(str).to_numpy(), + region=recipient["region"].astype(str).to_numpy(), ) household_draws["domestic_energy_consumption"] = ( household_draws["electricity_consumption"] @@ -553,23 +557,49 @@ def rake_energy_to_need( *, weights: Sequence[float] | None, iterations: int = 50, + tenure: Sequence[str] | None = None, + accommodation: Sequence[str] | None = None, + region: Sequence[str] | None = None, ) -> pd.DataFrame: + """Rake electricity/gas to the NEED margins. + + The donor-side single-pass call rakes the income margin only (the + incumbent's training-side calibration). The post-imputation call passes + all four groupers and sweeps income -> tenure -> accommodation -> region + per iteration, the incumbent's order. Categories absent from the NEED + maps (CONVERTED_HOUSE/OTHER/UNKNOWN accommodation; Scotland and Northern + Ireland regions) are deliberately untouched by that margin. + """ + frame = household.copy() frame["_need_income_band"] = _income_band(frame["household_gross_income"]) + margins = [MarginSpec("_need_income_band", _NEED_INCOME_TARGETS)] + scratch = ["_need_income_band"] + for name, values in ( + ("tenure", tenure), + ("accommodation", accommodation), + ("region", region), + ): + if values is None: + continue + column = f"_need_{name}" + frame[column] = np.asarray(values).astype(str) + targets, _ = _need_categorical_targets(name) + margins.append(MarginSpec(column, targets)) + scratch.append(column) weight_column = None if weights is not None: frame["_weight"] = np.asarray(weights, dtype=float) weight_column = "_weight" + scratch.append("_weight") raked = iterative_proportional_fit( frame, columns=("electricity_consumption", "gas_consumption"), - margins=(MarginSpec("_need_income_band", _NEED_INCOME_TARGETS),), + margins=tuple(margins), iterations=iterations, weight_column=weight_column, ) - return raked.drop( - columns=[c for c in ("_need_income_band", "_weight") if c in raked] - ) + return raked.drop(columns=[c for c in scratch if c in raked]) _NEED_INCOME_BANDS = ( @@ -595,6 +625,37 @@ def rake_energy_to_need( } +def _need_categorical_targets(margin: str) -> tuple[dict, dict]: + """(category -> column -> spend target, frs-value -> need-key map). + + Built from the committed NEED resource so the raking and the + aggregate_admin anchors share one source of values. + """ + + from importlib.resources import files + + need = json.loads( + files("microcosm.build.uk") + .joinpath("need_energy_targets.json") + .read_text(encoding="utf-8") + ) + block = need[margin] + if margin == "region": + mapping = {name: name for name in block["gas_kwh"]} + else: + mapping = dict(block["map"]) + targets = { + frs_value: { + "gas_consumption": block["gas_kwh"][need_key] * _GAS_RATE, + "electricity_consumption": ( + block["electricity_kwh"][need_key] * _ELEC_RATE + ), + } + for frs_value, need_key in mapping.items() + } + return targets, mapping + + def _income_band(values: pd.Series) -> pd.Series: income = _numeric(values) result = pd.Series(index=income.index, dtype=object) diff --git a/packages/microcosm-build/tests/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py index 27a16336..6b349b98 100644 --- a/packages/microcosm-build/tests/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/test_uk_lcfs_consumption.py @@ -204,3 +204,75 @@ def test_has_fuel_bridge_accepts_lcfs_native_predictor_names() -> None: assert first["has_fuel_consumption"].tolist() == ( second["has_fuel_consumption"].tolist() ) + + +def test_post_imputation_rake_fits_all_four_need_margins() -> None: + # Regression for the licensed-build finding: the manifest declares a + # four-margin post-imputation rake (income -> tenure -> accommodation -> + # region), but only the income margin was implemented — the incumbent + # hits the tenure/accommodation cells to ~1% and ours was off ±30%. + from microcosm.build.uk_runtime.lcfs_consumption import rake_energy_to_need + + rng = np.random.default_rng(3) + n = 400 + household = pd.DataFrame( + { + "household_gross_income": rng.uniform(5e3, 2e5, n), + "electricity_consumption": rng.uniform(200.0, 2000.0, n), + "gas_consumption": rng.uniform(100.0, 1500.0, n), + } + ) + tenure = rng.choice(["OWNED_OUTRIGHT", "RENT_PRIVATELY", "RENT_FROM_COUNCIL"], n) + accommodation = rng.choice(["HOUSE_DETACHED", "FLAT", "OTHER"], n) + region = rng.choice(["LONDON", "WALES", "SCOTLAND"], n) + weights = rng.uniform(0.5, 2.0, n) + + raked = rake_energy_to_need( + household, + weights=weights, + tenure=tenure, + accommodation=accommodation, + region=region, + ) + + import json as json_module + from importlib.resources import files + + need = json_module.loads( + files("microcosm.build.uk") + .joinpath("need_energy_targets.json") + .read_text(encoding="utf-8") + ) + rates = need["source"]["ofgem_q2_2026"] + + def wmean(values, mask): + return float((values[mask] * weights[mask]).sum() / weights[mask].sum()) + + # Region is the last margin swept, so it fits essentially exactly; the + # earlier margins settle within a tight band over 50 iterations. + elec = raked["electricity_consumption"].to_numpy(dtype=float) + target = need["region"]["electricity_kwh"]["LONDON"] * ( + rates["electricity_gbp_per_kwh"] + ) + assert abs(wmean(elec, region == "LONDON") - target) / target < 1e-6 + gas = raked["gas_consumption"].to_numpy(dtype=float) + tenure_target = need["tenure"]["gas_kwh"]["owner"] * rates["gas_gbp_per_kwh"] + assert ( + abs(wmean(gas, tenure == "OWNED_OUTRIGHT") - tenure_target) / tenure_target + < 0.02 + ) + accomm_target = ( + need["accommodation"]["electricity_kwh"]["detached"] + * rates["electricity_gbp_per_kwh"] + ) + assert ( + abs(wmean(elec, accommodation == "HOUSE_DETACHED") - accomm_target) + / accomm_target + < 0.02 + ) + # Unmapped categories stay outside their margin: SCOTLAND has no NEED + # region row and OTHER has no accommodation row, but both still move via + # the other margins — assert they were not pinned to any region target. + scotland_mean = wmean(elec, region == "SCOTLAND") + for kwh in need["region"]["electricity_kwh"].values(): + assert abs(scotland_mean - kwh * rates["electricity_gbp_per_kwh"]) > 1.0 From fea19ef2297f717b9227ecc2fa836f9f61477f5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:20:39 +0200 Subject: [PATCH 10/10] Fix E6 anchor provenance and validation --- .../src/microcosm/build/raking.py | 28 +- .../spec_engine/schema/sources.schema.json | 146 ++++ .../build/uk/etb_policy_anchors.json | 4 + .../build/uk/etb_services_anchors.json | 4 + .../build/uk/lcfs_consumption_anchors.json | 5 + .../build/uk/need_energy_targets.json | 4 + .../uk/release_input_coverage_manifest.json | 10 +- .../src/microcosm/build/uk/source_stages.json | 15 +- .../src/microcosm/build/uk/spec/sources.yaml | 649 +++++++++++++++--- .../build/uk_runtime/etb_services.py | 84 ++- .../src/microcosm/build/uk_runtime/etb_vat.py | 63 +- .../build/uk_runtime/lcfs_consumption.py | 101 ++- .../tests/test_uk_consumption_resources.py | 4 + .../tests/test_uk_etb_services.py | 8 +- .../microcosm-build/tests/test_uk_etb_vat.py | 8 + .../tests/test_uk_nhs_allocation.py | 4 +- .../microcosm-build/tests/test_uk_raking.py | 18 +- .../tests/test_uk_source_stages.py | 4 +- tools/verify_uk_identity_stability.py | 7 +- 19 files changed, 997 insertions(+), 169 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/raking.py b/packages/microcosm-build/src/microcosm/build/raking.py index 88fa9953..dcabe29c 100644 --- a/packages/microcosm-build/src/microcosm/build/raking.py +++ b/packages/microcosm-build/src/microcosm/build/raking.py @@ -31,12 +31,14 @@ def iterative_proportional_fit( margins: Sequence[MarginSpec], iterations: int, weight_column: str | None = None, + fail_on_unattainable: bool = False, ) -> pd.DataFrame: """Scale columns in-place-by-copy to match declared cell means. - Empty cells, zero-current-mean cells, and categories absent from the - declared targets are skipped. That preserves support zeros and lets a - country-specific caller intentionally leave unmapped enum values alone. + Empty cells and categories absent from the declared targets are skipped. + Populated zero-current-mean cells with positive targets are recorded in the + returned frame's ``raking_zero_current_cells`` evidence attribute. Callers + that require fail-closed behavior can set ``fail_on_unattainable``. """ if iterations < 1: @@ -58,6 +60,7 @@ def iterative_proportional_fit( if (weights < 0).any(): raise ValueError("raking weights must be nonnegative") + zero_current_cells: list[dict[str, object]] = [] for _ in range(iterations): for margin in margins: if margin.column not in result: @@ -73,15 +76,30 @@ def iterative_proportional_fit( result.loc[mask, column], None if weights is None else weights.loc[mask], ) - if current <= 0 or not np.isfinite(current): - continue target = float(target_by_column[column]) if not np.isfinite(target) or target < 0: raise ValueError( f"target for {margin.column!r}={category!r}, " f"{column!r} must be finite and nonnegative" ) + if current <= 0 or not np.isfinite(current): + if target > 0: + evidence = { + "margin": margin.column, + "category": category, + "column": column, + "target": target, + } + zero_current_cells.append(evidence) + if fail_on_unattainable: + raise ValueError( + f"cannot rake {margin.column!r}={category!r} " + f"for {column!r}: current mean is " + f"zero/non-finite but target is {target}." + ) + continue result.loc[mask, column] *= target / current + result.attrs["raking_zero_current_cells"] = tuple(zero_current_cells) return result diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 4273c881..31175588 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -5013,6 +5013,12 @@ }, "runtime_sha256_required": { "type": "boolean" + }, + "format": { + "type": "string" + }, + "vintage": { + "type": "string" } }, "required": [ @@ -5771,6 +5777,140 @@ } } }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "role", + "resource", + "format" + ], + "properties": { + "kind": { + "const": "public_parameter_reference" + }, + "role": { + "type": "string" + }, + "resource": { + "type": "string" + }, + "format": { + "type": "string" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { + "enum": [ + "allocate_per_capita_from_cell_table", + "assign_binary_from_rate", + "bridge_donor_column_via_qrf", + "compute_ratio", + "derive", + "iterative_proportional_fit", + "zero_when_false" + ] + }, + "age_bands": {"type": "string"}, + "annualization_weeks": {"type": "number"}, + "budget_resource": {"type": "string"}, + "categorical_predictors": {"type": "array", "items": {"type": "string"}}, + "chain_order": {"type": "array", "items": {"type": "string"}}, + "columns": {"type": "array", "items": {"type": "string"}}, + "condition": {"type": "string"}, + "denominator_key": {"type": "string"}, + "denominator_resource": {"type": "string"}, + "donor_weight": {"type": "string"}, + "exempt": {"type": "array", "items": {"type": "string"}}, + "fail_loud_on_missing_rate": {"type": "boolean"}, + "iterations": {"type": "integer", "minimum": 1}, + "lossy_mappings": {"type": "array", "items": {"type": "string"}}, + "logged_dropna_row_count": {"type": "boolean"}, + "margins": {"type": "array", "items": {"type": "string"}}, + "n_estimators": {"type": "integer", "minimum": 1}, + "numerator": {"type": "string"}, + "output": {"type": "string"}, + "predictors": {"type": "array", "items": {"type": "string"}}, + "rate_key": {"type": "string"}, + "range": {"type": "string"}, + "reduced_rate_share": {"type": "number"}, + "resource": {"type": "string"}, + "seed": {"type": "integer"}, + "source": {"type": "string"}, + "standard_rate": {"type": "number"}, + "target": {"type": "string"}, + "targets": {"type": "array", "items": {"type": "string"}}, + "top_band_fold_in": {"type": "string"}, + "weights": {"type": "string"}, + "weighted": {"type": "boolean"}, + "year": {"type": ["string", "integer"]}, + "lowercase_columns": {"type": "boolean"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "predictors", "targets", "seed"], + "properties": { + "kind": {"const": "fit_weighted_qrf"}, + "predictors": {"type": "array", "items": {"type": "string"}}, + "targets": {"type": "array", "items": {"type": "string"}}, + "weights": {"type": "string"}, + "n_estimators": {"type": "integer", "minimum": 1}, + "seed": {"type": "integer"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "predictors", "targets", "seed"], + "properties": { + "kind": {"const": "fit_weighted_qrf_chain"}, + "predictors": {"type": "array", "items": {"type": "string"}}, + "targets": {"type": "array", "items": {"type": "string"}}, + "categorical_predictors": {"type": "array", "items": {"type": "string"}}, + "weights": {"type": "string"}, + "n_estimators": {"type": "integer", "minimum": 1}, + "seed": {"type": "integer"} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "range", "exempt"], + "properties": { + "kind": {"const": "support_clip"}, + "range": {"type": "string"}, + "exempt": {"type": "array", "items": {"type": "string"}} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "predictors", "derived_predictors"], + "properties": { + "kind": {"const": "materialize_rules_engine_predictors"}, + "predictors": {"type": "array", "items": {"type": "string"}}, + "derived_predictors": {"type": "object", "additionalProperties": {"type": "string"}} + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "output", "inputs"], + "properties": { + "kind": {"const": "fold_into"}, + "output": {"type": "string"}, + "inputs": {"type": "array", "items": {"type": "string"}}, + "drop_inputs": {"type": "boolean"} + } + }, { "type": "object", "additionalProperties": false, @@ -6446,6 +6586,12 @@ }, "runtime_sha256_required": { "type": "boolean" + }, + "format": { + "type": "string" + }, + "vintage": { + "type": "string" } } }, diff --git a/packages/microcosm-build/src/microcosm/build/uk/etb_policy_anchors.json b/packages/microcosm-build/src/microcosm/build/uk/etb_policy_anchors.json index 8979de14..5e26e3ca 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/etb_policy_anchors.json +++ b/packages/microcosm-build/src/microcosm/build/uk/etb_policy_anchors.json @@ -3,6 +3,10 @@ "country": "uk", "source": { "citation": "PolicyEngine UK VAT parameters for 2023; incumbent fallback removed.", + "urls": [ + "https://www.gov.uk/guidance/rates-and-allowances-for-vat", + "https://github.com/PolicyEngine/policyengine-uk/tree/main/policyengine_uk/parameters/gov/hmrc/vat" + ], "chronicle_candidate": true }, "vat": { diff --git a/packages/microcosm-build/src/microcosm/build/uk/etb_services_anchors.json b/packages/microcosm-build/src/microcosm/build/uk/etb_services_anchors.json index 2d9bd2b1..c24c7ab2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/etb_services_anchors.json +++ b/packages/microcosm-build/src/microcosm/build/uk/etb_services_anchors.json @@ -3,6 +3,10 @@ "country": "uk", "source": { "citation": "DfT rail fare index and NHS 2025/26 budget anchor.", + "urls": [ + "https://dataportal.orr.gov.uk/statistics/finance/rail-fares/rail-fares-index-2023/", + "https://assets.publishing.service.gov.uk/media/6849172b860362efc8e78836/E03349913_HMT_Spending_Review_June_2025_TEXT_PRINT.pdf" + ], "chronicle_candidate": true }, "rail_fare_index_2023": { diff --git a/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_anchors.json b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_anchors.json index 5bddd1b7..3da17c11 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_anchors.json +++ b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_anchors.json @@ -3,6 +3,11 @@ "country": "uk", "source": { "citation": "NTS 2024 ICE vehicle share, DESNZ pump prices, road-fuel volume/population indices, and PolicyEngine UK CPI parameter paths.", + "urls": [ + "https://www.gov.uk/government/statistics/national-travel-survey-2024/nts-2024-household-car-availability-and-trends-in-car-trips", + "https://www.gov.uk/government/collections/road-fuel-and-other-petroleum-product-prices", + "https://github.com/PolicyEngine/policyengine-uk/tree/main/policyengine_uk/parameters/gov/economic_assumptions/indices/obr" + ], "chronicle_candidate": true }, "nts_ice_share": { diff --git a/packages/microcosm-build/src/microcosm/build/uk/need_energy_targets.json b/packages/microcosm-build/src/microcosm/build/uk/need_energy_targets.json index fb50e995..30920e31 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/need_energy_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/need_energy_targets.json @@ -3,6 +3,10 @@ "country": "uk", "source": { "citation": "NEED 2023 headline tables 5b/6b, 9b/10b, 11b/12b, 15b/16b; Ofgem Q2 2026 unit rates.", + "urls": [ + "https://www.gov.uk/government/statistics/national-energy-efficiency-data-framework-need-consumption-data-tables-2025", + "https://www.ofgem.gov.uk/energy-price-cap" + ], "ofgem_q2_2026": { "electricity_gbp_per_kwh": 0.2467, "gas_gbp_per_kwh": 0.0574 diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index c339b649..72f881b3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -478,7 +478,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", + "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -497,7 +497,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", + "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -639,7 +639,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", + "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -659,7 +659,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", + "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -690,7 +690,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "7ac534e8ebc3ff0a100efe0cca81a7b767fdc5d3c73043285911212820512256", + "source_manifest_sha256": "e0cef76862d76135038500b7cae217c5a1304b5afaca8929e5b4ff92479fa9fc", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 6434924b..20a473d7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -1029,7 +1029,8 @@ "locator": "dvhh_ukanon_v2_2023.tab", "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", "size_bytes": 22812887, - "runtime_sha256_required": true + "runtime_sha256_required": true, + "filename": "dvhh_ukanon_v2_2023.tab" }, { "role": "lcfs_person_tab", @@ -1039,7 +1040,8 @@ "locator": "dvper_ukanon_202324_2023.tab", "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", "size_bytes": 6545146, - "runtime_sha256_required": true + "runtime_sha256_required": true, + "filename": "dvper_ukanon_202324_2023.tab" }, { "role": "was_bridge_donor", @@ -1049,7 +1051,8 @@ "locator": "was_round_8_hhold_eul_may_2025_230525.tab", "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", "size_bytes": 39073613, - "runtime_sha256_required": true + "runtime_sha256_required": true, + "filename": "was_round_8_hhold_eul_may_2025_230525.tab" }, { "role": "need_energy_targets", @@ -1272,7 +1275,8 @@ "locator": "householdv2_1977-2024.tab", "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", "size_bytes": 216967663, - "runtime_sha256_required": true + "runtime_sha256_required": true, + "filename": "householdv2_1977-2024.tab" }, { "role": "etb_policy_anchors", @@ -1345,7 +1349,8 @@ "locator": "householdv2_1977-2024.tab", "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", "size_bytes": 216967663, - "runtime_sha256_required": true + "runtime_sha256_required": true, + "filename": "householdv2_1977-2024.tab" }, { "role": "nhs_consumption_by_age_gender", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 91c2ddcf..99f44cc1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -13,11 +13,14 @@ sources: stage_manifest: version: 1 country: uk - policy: The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure. + policy: The UK HMRC/SPI income family is source-manifest-defined. Private donor data must be supplied locally, every artifact + must be SHA-256 verified at runtime, retained FRS constituents and published bands fail closed, and the current replay + keeps importance-kind weights because all 208 banded facts require an unavailable full FRS total-income measure. stages: - stage: frs_spine survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9367, DOI 10.5255/UKDA-SN-9367-2; local licensed 2023_24 tabs. + source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9367, DOI 10.5255/UKDA-SN-9367-2; + local licensed 2023_24 tabs. grain: household artifacts: - role: frs_table @@ -154,7 +157,9 @@ stages: numeric_errors: coerce runtime_sha256_required: true - kind: replace_sentinels - sentinel_policy: 'raw numeric blanks and nonnumeric sentinels are coerced to NaN, then produced E2 columns are filled to the Frame no-NaN contract. Clamping is deliberately per-column, mirroring the incumbent base build: redamt stays unclamped, tuborr clamps at zero, and the property/royalties components clamp only in their aggregate.' + sentinel_policy: 'raw numeric blanks and nonnumeric sentinels are coerced to NaN, then produced E2 columns are filled + to the Frame no-NaN contract. Clamping is deliberately per-column, mirroring the incumbent base build: redamt stays + unclamped, tuborr clamps at zero, and the property/royalties components clamp only in their aggregate.' - kind: assemble_group_entities household_id: sernum benunit_id: sernum * 100 + benunit @@ -273,7 +278,9 @@ stages: - structural_insurance_payments - housing_service_charges - external_child_payments - notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage. + notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields + remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally + absent from this stage. - stage: frs_employment survey: Family Resources Survey 2023-24 source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs. @@ -303,7 +310,8 @@ stages: - sic_industry_division nonnegative_outputs: - sic_industry_division - notes: Ports FRS employment derivations. empstati code 11 preserves the incumbent truncated-map artifact as LONG_TERM_DISABLED, so OTHER_INACTIVE is not emitted; mjobsect and sic are direct-indexed and fail loudly if absent. + notes: Ports FRS employment derivations. empstati code 11 preserves the incumbent truncated-map artifact as LONG_TERM_DISABLED, + so OTHER_INACTIVE is not emitted; mjobsect and sic are direct-indexed and fail loudly if absent. - stage: frs_council_tax survey: Family Resources Survey 2023-24 source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs. @@ -337,10 +345,12 @@ stages: - council_tax nonnegative_outputs: - council_tax - notes: Re-reads raw househol.tab because spine council_tax_reported clips missing values. Scottish Water charges are netted before cell means; no-donor cells impute zero. The dead ct_mean.replace(-1, ...) branch is intentionally dropped. + notes: Re-reads raw househol.tab because spine council_tax_reported clips missing values. Scottish Water charges are netted + before cell means; no-donor cells impute zero. The dead ct_mean.replace(-1, ...) branch is intentionally dropped. - stage: frs_disability survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus policyengine-uk DWP parameters. + source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus + policyengine-uk DWP parameters. grain: person artifacts: [] operations: @@ -357,7 +367,8 @@ stages: - is_disabled_for_benefits - is_enhanced_disabled_for_benefits - is_severely_disabled_for_benefits - notes: Consumes E2 reported disability amount carriers. The five internal amount carriers are retained through E7 and stripped at E10; export allowlists stay fail-closed. + notes: Consumes E2 reported disability amount carriers. The five internal amount carriers are retained through E7 and stripped + at E10; export allowlists stay fail-closed. - stage: frs_education survey: Family Resources Survey 2023-24 source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs. @@ -392,7 +403,8 @@ stages: scope: current/highest education, QYP inputs, EMA cell-mean degenerate fills, and benefits-in-own-right flag - kind: impute_cell_means cells: - - 'single cell: EMA participants (code == 1; adema/ademaamt pair, eduma/edumaamt when adema is absent; chema/chemaamt for children)' + - 'single cell: EMA participants (code == 1; adema/ademaamt pair, eduma/edumaamt when adema is absent; chema/chemaamt + for children)' donor_filter: participants with non-negative reported amounts missing: participants with sentinel negative reported amounts value: donor-mean fill, floored at zero, annualized with 365.25 / 7 @@ -410,10 +422,15 @@ stages: - adult_ema - child_ema - age_started_or_accepted_current_education_or_training - notes: 'Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2023-24 data dictionary (UK Data Service SN 9367, DOI 10.5255/UKDA-SN-9367-2, adult table), corroborated against the raw aggregates and adopted as a signed difference (PR #703); an upstream defect report records the incumbent inversion. Code 87 is undocumented in the dictionary and falls to the default. EMA uses the shared weeks-in-year constant.' + notes: 'Ports the incumbent education cascade including its unreachable POST_SECONDARY branch order. EDUCQUAL_MAP carries + the corrected highest-qualification codeframe (1 = Doctorate, descending) per the FRS 2023-24 data dictionary (UK Data + Service SN 9367, DOI 10.5255/UKDA-SN-9367-2, adult table), corroborated against the raw aggregates and adopted as a signed + difference (PR #703); an upstream defect report records the incumbent inversion. Code 87 is undocumented in the dictionary + and falls to the default. EMA uses the shared weeks-in-year constant.' - stage: frs_legacy_proxies survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus policyengine-uk DWP parameters. + source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus + policyengine-uk DWP parameters. grain: person artifacts: - role: frs_table @@ -442,10 +459,12 @@ stages: - legacy_jobseeker_proxy - esa_health_condition_proxy - esa_support_group_proxy - notes: The proxies are labels, not entitlement determinations. JSA hours compare against 16 * (365.25 / 7) on the E2 spine hours scale; state_pension_age is consumed but not persisted. + notes: The proxies are labels, not entitlement determinations. JSA hours compare against 16 * (365.25 / 7) on the E2 spine + hours scale; state_pension_age is consumed but not persisted. - stage: frs_education_grant_split survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus policyengine-uk DfE grant parameters. + source: Department for Work and Pensions Family Resources Survey 2023-24, UK Data Service SN 9252 local licensed tabs plus + policyengine-uk DfE grant parameters. grain: person artifacts: [] operations: @@ -463,7 +482,8 @@ stages: - education_grants nonnegative_outputs: - disabled_students_allowance_eligible_expenses - notes: Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned zero vector rather than an engine shape-sizing read. + notes: Runs before BRMA and always runs; both are signed inert differences for 2023-24. Pre-2025 DSA capacity is an aligned + zero vector rather than an engine shape-sizing read. - stage: frs_take_up survey: Family Resources Survey 2023-24 source: Family Resources Survey 2023-24 reported receipt anchors plus sourced UK take-up contract rates. @@ -537,7 +557,8 @@ stages: - maximum_extended_childcare_hours_usage nonnegative_outputs: - maximum_extended_childcare_hours_usage - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. + Reported positive receipt anchors are transient and consumed only. - stage: frs_person_draws survey: Family Resources Survey 2023-24 source: Sourced UK take-up contract rates and identity-keyed deterministic draws. @@ -567,7 +588,8 @@ stages: - attends_private_school_random_draw nonnegative_outputs: - attends_private_school_random_draw - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally + not produced. - stage: frs_household_draws survey: Family Resources Survey 2023-24 source: Sourced UK stochastic contract rates and identity-keyed deterministic draws. @@ -599,7 +621,8 @@ stages: - would_evade_tv_licence_fee - main_residential_property_purchased_is_first_home - property_purchased - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent + draw, matching the incumbent reference share. - stage: frs_brma survey: Family Resources Survey 2023-24 source: Valuation Office Agency LHA list-of-rents count table and policyengine-uk LHA_category predictor. @@ -626,10 +649,12 @@ stages: seed: 0 outputs: - brma - notes: Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick. + notes: Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt + brma:household_pick. - stage: was_wealth survey: Wealth and Assets Survey round 8 - source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab. + source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local + licensed 2006-22 household tab. grain: household base_candidate: filename: populace_uk_2023.h5 @@ -765,7 +790,18 @@ stages: - cash_isa - stocks_and_shares_isa - student_loan_balance - notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded.' + notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces + the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; corporate_wealth folds + stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales + for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. + Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the + incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy + encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully + like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes + (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: + 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality + revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s + -9 is structural not-applicable so the is_renting mapping is unchanged; genuinely negative domains are never recoded.' - stage: regional_property_uprating survey: Public regional property reference source: MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices. @@ -790,10 +826,372 @@ stages: rewrites: - main_residence_value - property_wealth - notes: Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior. + notes: Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. + Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean + follows the incumbent behavior. +- stage: lcfs_consumption + survey: Living Costs and Food Survey 2023-24 + source: UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, + Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor. + grain: household + base_candidate: + filename: populace_uk_2023.h5 + revision: populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z + sha256: f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833 + tier: frs + artifacts: + - role: lcfs_household_tab + kind: private_microdata + format: tab + vintage: '2023_24' + locator: dvhh_ukanon_v2_2023.tab + sha256: 6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72 + size_bytes: 22812887 + runtime_sha256_required: true + filename: dvhh_ukanon_v2_2023.tab + - role: lcfs_person_tab + kind: private_microdata + format: tab + vintage: '2023_24' + locator: dvper_ukanon_202324_2023.tab + sha256: f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50 + size_bytes: 6545146 + runtime_sha256_required: true + filename: dvper_ukanon_202324_2023.tab + - role: was_bridge_donor + kind: private_microdata + format: tab + vintage: '2018_20' + locator: was_round_8_hhold_eul_may_2025_230525.tab + sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 + size_bytes: 39073613 + runtime_sha256_required: true + filename: was_round_8_hhold_eul_may_2025_230525.tab + - role: need_energy_targets + resource: need_energy_targets.json + kind: public_aggregate_reference + format: json + - role: lcfs_consumption_anchors + resource: lcfs_consumption_anchors.json + kind: public_aggregate_reference + format: json + operations: + - kind: derive + lowercase_columns: true + annualization_weeks: 52.17857142857143 + donor_weight: weighta * 1000 + lossy_mappings: + - LCFS tenure 4 and 8 -> RENT_PRIVATELY + - LCFS accommodation 4 and 5 -> FLAT + logged_dropna_row_count: true + - kind: iterative_proportional_fit + columns: + - electricity_consumption + - gas_consumption + margins: + - gross_income_band + iterations: 1 + weighted: false + - kind: bridge_donor_column_via_qrf + source: was_wealth + target: has_fuel_consumption + predictors: + - household_net_income + - num_adults + - num_children + - private_pension_income + - employment_income + - self_employment_income + - region + weights: explicit + seed: 0 + n_estimators: 100 + - kind: assign_binary_from_rate + target: has_fuel_consumption + rate_key: nts_ice_share + condition: num_vehicles > 0 + seed: 0 + - kind: materialize_rules_engine_predictors + predictors: + - is_adult + - is_child + - employment_income + - self_employment_income + - private_pension_income + - hbai_household_net_income + - kind: fit_weighted_qrf_chain + predictors: + - is_adult + - is_child + - region + - employment_income + - self_employment_income + - private_pension_income + - hbai_household_net_income + - tenure_type + - accommodation_type + - has_fuel_consumption + targets: + - food_and_non_alcoholic_beverages_consumption + - alcohol_and_tobacco_consumption + - clothing_and_footwear_consumption + - housing_water_and_electricity_consumption + - household_furnishings_consumption + - health_consumption + - transport_consumption + - communication_consumption + - recreation_consumption + - education_consumption + - restaurants_and_hotels_consumption + - miscellaneous_consumption + - petrol_spending + - diesel_spending + - bus_fare_spending + - domestic_energy_consumption + - electricity_consumption + - gas_consumption + categorical_predictors: + - region + - tenure_type + - accommodation_type + weights: explicit + seed: 0 + n_estimators: 100 + - kind: support_clip + range: donor_realized + exempt: + - electricity_consumption + - gas_consumption + - domestic_energy_consumption + - kind: iterative_proportional_fit + columns: + - electricity_consumption + - gas_consumption + margins: + - income + - tenure + - accommodation + - region + iterations: 50 + weighted: true + - kind: fold_into + output: domestic_energy_consumption + inputs: + - electricity_consumption + - gas_consumption + drop_inputs: false + - kind: zero_when_false + columns: + - petrol_spending + - diesel_spending + condition: has_fuel_consumption == false + outputs: + - food_and_non_alcoholic_beverages_consumption + - alcohol_and_tobacco_consumption + - clothing_and_footwear_consumption + - housing_water_and_electricity_consumption + - household_furnishings_consumption + - health_consumption + - transport_consumption + - communication_consumption + - recreation_consumption + - education_consumption + - restaurants_and_hotels_consumption + - miscellaneous_consumption + - petrol_spending + - diesel_spending + - bus_fare_spending + - domestic_energy_consumption + - electricity_consumption + - gas_consumption + - has_fuel_consumption + nonnegative_outputs: + - food_and_non_alcoholic_beverages_consumption + - alcohol_and_tobacco_consumption + - clothing_and_footwear_consumption + - housing_water_and_electricity_consumption + - household_furnishings_consumption + - health_consumption + - transport_consumption + - communication_consumption + - recreation_consumption + - education_consumption + - restaurants_and_hotels_consumption + - miscellaneous_consumption + - petrol_spending + - diesel_spending + - bus_fare_spending + - domestic_energy_consumption + - electricity_consumption + - gas_consumption + - has_fuel_consumption + notes: Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and identity-keyed + seed-0 fuel flags. Energy support clipping is exempt because NEED raking governs those columns. Donor uprating is identity + at this vintage (LCFS survey year equals the 2023 build year), so the incumbent's CPI and fuel litre-proxy donor uprating + is deliberately not declared here; the machinery lands with the FRS 2024-25 refresh (microcosm#687), where a donor/build + year gap first exists. The DESNZ pump-price anchors stay committed as the cited litre-proxy denominators for that refresh. +- stage: etb_vat + survey: Effects of Taxes and Benefits 1977-2024 + source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. + grain: household + base_candidate: + filename: populace_uk_2023.h5 + revision: populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z + sha256: f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833 + tier: frs + artifacts: + - role: etb_household_tab + kind: private_microdata + format: tab + vintage: '1977_24' + locator: householdv2_1977-2024.tab + sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 + size_bytes: 216967663 + runtime_sha256_required: true + filename: householdv2_1977-2024.tab + - role: etb_policy_anchors + resource: etb_policy_anchors.json + kind: public_parameter_reference + format: json + operations: + - kind: derive + year: 2023 + annualization_weeks: 52 + standard_rate: 0.2 + reduced_rate_share: 0.025 + fail_loud_on_missing_rate: true + - kind: materialize_rules_engine_predictors + predictors: + - is_adult + - is_child + - is_SP_age + - household_net_income + - kind: fit_weighted_qrf + predictors: + - is_adult + - is_child + - is_SP_age + - household_net_income + targets: + - full_rate_vat_expenditure_rate + weights: explicit + seed: 0 + n_estimators: 100 + - kind: support_clip + range: donor_realized + outputs: + - full_rate_vat_expenditure_rate + nonnegative_outputs: [] + notes: 'Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather + than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: + totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the + support gate is the guard - the net_financial_wealth precedent from was_wealth.' +- stage: etb_services + survey: Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table + source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost + table. + grain: household+person + base_candidate: + filename: populace_uk_2023.h5 + revision: populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z + sha256: f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833 + tier: frs + artifacts: + - role: etb_household_tab + kind: private_microdata + format: tab + vintage: '1977_24' + locator: householdv2_1977-2024.tab + sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 + size_bytes: 216967663 + runtime_sha256_required: true + filename: householdv2_1977-2024.tab + - role: nhs_consumption_by_age_gender + resource: nhs_consumption_by_age_gender.json + kind: public_aggregate_reference + format: json + - role: etb_services_anchors + resource: etb_services_anchors.json + kind: public_parameter_reference + format: json + operations: + - kind: derive + year: max + annualization_weeks: 52 + - kind: materialize_rules_engine_predictors + predictors: + - is_adult + - is_child + - is_SP_age + - dla + - pip + - hbai_household_net_income + - current_education + derived_predictors: + count_primary_education: current_education == PRIMARY + count_secondary_education: current_education == LOWER_SECONDARY + count_further_education: current_education in (UPPER_SECONDARY, TERTIARY) + - kind: fit_weighted_qrf_chain + predictors: + - is_adult + - is_child + - is_SP_age + - count_primary_education + - count_secondary_education + - count_further_education + - dla + - pip + - hbai_household_net_income + targets: + - dfe_education_spending + - rail_subsidy_spending + - bus_subsidy_spending + weights: explicit + seed: 0 + n_estimators: 100 + - kind: support_clip + range: donor_realized + - kind: compute_ratio + output: rail_usage + numerator: rail_subsidy_spending + denominator_resource: etb_services_anchors.json + denominator_key: rail_fare_index_2023 + - kind: allocate_per_capita_from_cell_table + resource: nhs_consumption_by_age_gender.json + budget_resource: etb_services_anchors.json + age_bands: half_open + top_band_fold_in: 85+ + outputs: + - dfe_education_spending + - rail_subsidy_spending + - bus_subsidy_spending + - rail_usage + - a_and_e_visits + - admitted_patient_visits + - outpatient_visits + - nhs_a_and_e_spending + - nhs_admitted_patient_spending + - nhs_outpatient_spending + nonnegative_outputs: + - dfe_education_spending + - rail_subsidy_spending + - bus_subsidy_spending + - rail_usage + - a_and_e_visits + - admitted_patient_visits + - outpatient_visits + - nhs_a_and_e_spending + - nhs_admitted_patient_spending + - nhs_outpatient_spending + notes: Ports ETB public-services QRF at household grain, computes rail_usage from the 2023 fare index, and allocates NHS + visits/spending to persons using the signed half-open age-band and 85+ fold-in fixes. The year-max donor filter resolves + to 2023 on the pinned tab (the file labels financial year ending 2024 as year 2023), so the services training year coincides + with the VAT training year and the fare-index year - the incumbent's apparent three-way year mismatch is vacuous on this + vintage. - stage: frs_hmrc_spine_leaves survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied local input + source: Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied + local input grain: person artifacts: - role: frs_table @@ -893,7 +1291,8 @@ stages: - ossben_identifiable_subset - srp_regular_code5 - employer_pension_contributions - notes: Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, then ports the incumbent employer-pension-contributions estimate. + notes: Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, + then ports the incumbent employer-pension-contributions estimate. - stage: spi_support_channel survey: Family Resources Survey 2023-24 source: Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning. @@ -945,7 +1344,8 @@ stages: - household_source_id - source_household_id - source_year - notes: Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates 50 percent of each region stratum prior mass to the SPI channel with exact national conservation. + notes: Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, + and allocates 50 percent of each region stratum prior mass to the SPI channel with exact national conservation. - stage: hmrc_spi_income_spine survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods @@ -1111,11 +1511,15 @@ stages: - hmrc_spi_other_income - hmrc_spi_state_pension_income joint_draw: true - savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input - employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure - hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels + savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine + gross input + employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; + it is not the Table 3.6 measure + hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels self_employment_income_source_semantics: max(0, PROFITS - CAPALL - LOSSBF) - assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly + assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates + and TI equals TEI + TII exactly source_ti_identity_fields: - TI - TEI @@ -1124,7 +1528,8 @@ stages: documentation_url: https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf composite_indicator: AGERANGE == -1 formulas: - TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF) + TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, + PROFITS - CAPALL - LOSSBF) TII: OTHERINV + DIVIDENDS + INCPROP + INCBBS TI: TEI + TII maximum_absolute_difference_gbp: @@ -1136,7 +1541,9 @@ stages: TEI: 180 TII: 10 TI: 180 - rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact. + rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income + fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain + exact. ti_identity_absolute_tolerance_gbp: 5 stochastic_aggregates_forbidden: - hmrc_spi_employed_income @@ -1164,7 +1571,9 @@ stages: - private_pension_income - property_income reviewed_absent_predictors: - other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column.' + other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only + stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified + Microcosm UK base candidate has no other_investment_income column.' categorical_predictors: - gender - region @@ -1201,8 +1610,10 @@ stages: - esa_contrib_reported - esa_income_reported reviewed_absent_outputs: - incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer. - maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage. + incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not + a populated loader layer. + maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training + source can be materialized for this stage. postprocess: gross_savings_interest_income: stage1 INCBBS draw + stage2 tax_free_savings_income refresh_disability_categories: @@ -1312,7 +1723,9 @@ stages: directional_pass: 0 directional_fail: 0 excluded_with_fence: 208 - classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid. + classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize + full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound + is valid. reviewed_fences: - fence_id: frs_epb_source_absent constituents: @@ -1324,9 +1737,12 @@ stages: - JOB.FUELAMT - JOB.VCHAMT - JOB.CHVAMT - finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits. - mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support. - rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy. + finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce + complete taxable expenses payments and benefits. + mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this + is not monetary support. + rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an + imputation or proxy. dependent_fence_ids: [] - fence_id: frs_exps_source_absent constituents: @@ -1337,9 +1753,12 @@ stages: - JOB.UMILEAMT/JOB.UMOTAMT - JOB.DEDUC1-DEDUC9 - JOB.UDEDUC1-UDEDUC9 - finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI. - mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable. - rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign. + finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense + amount required by SPI. + mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the + true EXPS mass is not estimable. + rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity + with a negative sign. dependent_fence_ids: [] - fence_id: frs_taxterm_source_absent constituents: @@ -1348,7 +1767,8 @@ stages: - ADULT.REDAMT - ADULT and JOB taxable-termination split search finding: Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay. - mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown. + mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable + mass is unknown. rationale: Gross redundancy pay cannot be relabeled as taxable termination pay. dependent_fence_ids: [] - fence_id: frs_mothinc_source_absent @@ -1359,7 +1779,8 @@ stages: - ADULT.ALLPAY2 - ADULT.ROYYR2-ROYYR4 - JOB.OWNOTHER - finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy. + finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI + miscellaneous employment income would be a proxy. mass_implication: Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%. rationale: The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully. dependent_fence_ids: [] @@ -1372,8 +1793,10 @@ stages: - ACCOUNTS - ASSETS - BENEFITS - finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence. - mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass. + finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be + split between MOTHINC and OTHERINC from source evidence. + mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate + FRS effective person mass. rationale: A union of heterogeneous residual fields would be a new proxy, not a retained source constituent. dependent_fence_ids: [] - fence_id: frs_ossben_identifiable_subset @@ -1385,8 +1808,10 @@ stages: - BENEFITS.BENEFIT - BENEFITS.VAR2 - BENEFITS codes 13, 16, 6, and 30 - finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. - mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support. + finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax + treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. + mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; + it is not full OSSBEN support. rationale: The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept. dependent_fence_ids: [] - fence_id: frs_srp_regular_code5_subset @@ -1396,9 +1821,12 @@ stages: raw_sources_searched: - BENEFITS.BENAMT where BENEFIT == 5 - BENEFITS codes 6 and 9 - finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. - mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support. - rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure. + finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination + of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. + mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; + it is not complete SRP support. + rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published + state-pension measure. dependent_fence_ids: [] - fence_id: full_frs_tei_band_unavailable constituents: @@ -1410,9 +1838,12 @@ stages: - OSSBEN - SRP raw_sources_searched: [] - finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel. - mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure. - rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' + finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income + band assignment is unavailable on the FRS channel. + mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable + like-for-like measure. + rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into + or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' dependent_fence_ids: - frs_epb_source_absent - frs_exps_source_absent @@ -1505,10 +1936,13 @@ stages: - hmrc_spi_pay - hmrc_spi_unemployment_benefit_income - hmrc_spi_incapacity_benefit_income - notes: Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine path. + notes: Runs the SPI-trained income QRFs on the raw-spine support channel, initializes FRS charity columns to zero, trains + FRS-only stage 2 before redrawing base-channel dividends, and emits a sidecar-only 208-fact replay report for the spine + path. - stage: frs_hmrc_retained_leaves survey: Family Resources Survey 2023-24 - source: Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied local input + source: Department for Work and Pensions Family Resources Survey 2023-24 raw adult.tab and benefits.tab, caller-supplied + local input grain: person artifacts: [] operations: @@ -1582,7 +2016,9 @@ stages: - hmrc_spi_incapacity_benefit_income - ossben_identifiable_subset - srp_regular_code5 - notes: 'Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these leaves.' + notes: 'Retains the adjudicated source-faithful FRS HMRC leaf columns before the SPI income rebuild: full PAY, UBISJA, and + INCPBEN, plus explicitly named OSSBEN and SRP subsets. The runtime verifies the certified candidate before retaining these + leaves.' - stage: hmrc_spi_income survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods @@ -1639,7 +2075,8 @@ stages: output_weight_kind: importance preserve_total_household_mass: true require_mass_change_record: true - mass_change_reason: Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; total national mass is conserved. + mass_change_reason: Allocate 50% of certified UK national household prior mass to the rebuilt 2022-23 SPI support channel; + total national mass is conserved. fail_on_live_existing_spi_mass: true - kind: strict_read_private_table artifact_role: qrf_donor @@ -1760,11 +2197,15 @@ stages: - hmrc_spi_other_income - hmrc_spi_state_pension_income joint_draw: true - savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine gross input - employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; it is not the Table 3.6 measure - hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels + savings_interest_source_semantics: INCBBS is taxable bank/building-society interest before reconstruction to the PolicyEngine + gross input + employment_income_source_semantics: PolicyEngine input = PAY + EPB + TAXTERM, matching the pinned enhanced-FRS pipeline; + it is not the Table 3.6 measure + hmrc_employed_income_source_semantics: Derived after each draw as max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + + UBISJA + MOTHINC, using normalized leaves identically on FRS and SPI channels self_employment_income_source_semantics: max(0, PROFITS - CAPALL - LOSSBF) - assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates and TI equals TEI + TII exactly + assessable_income_source_semantics: QRF draws leaves only; TEI, TII, and TI are deterministic post-draw accounting aggregates + and TI equals TEI + TII exactly source_ti_identity_fields: - TI - TEI @@ -1773,7 +2214,8 @@ stages: documentation_url: https://doc.ukdataservice.ac.uk/doc/9422/mrdoc/pdf/9422_put_2223_full_documentation.pdf composite_indicator: AGERANGE == -1 formulas: - TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, PROFITS - CAPALL - LOSSBF) + TEI: max(0, PAY + EPB - EXPS) + INCPBEN + OSSBEN + TAXTERM + UBISJA + MOTHINC + OTHERINC + SRP + PENSION + max(0, + PROFITS - CAPALL - LOSSBF) TII: OTHERINV + DIVIDENDS + INCPROP + INCBBS TI: TEI + TII maximum_absolute_difference_gbp: @@ -1785,7 +2227,9 @@ stages: TEI: 180 TII: 10 TI: 180 - rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain exact. + rationale: The official PUT rounds source fields, averages documented composite records, then rounds remaining income + fields to GBP 5. These are the observed envelopes in the exact sha-pinned donor; post-draw synthetic identities remain + exact. ti_identity_absolute_tolerance_gbp: 5 stochastic_aggregates_forbidden: - hmrc_spi_employed_income @@ -1808,7 +2252,9 @@ stages: - private_pension_income - property_income reviewed_absent_predictors: - other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified Microcosm UK base candidate has no other_investment_income column.' + other_investment_income: 'This remains a stage-1 SPI draw and an official HMRC fact component, but it is not an FRS-only + stage-2 predictor: the incumbent UK data build''s frs_only.py defines exactly six income predictors and the certified + Microcosm UK base candidate has no other_investment_income column.' categorical_predictors: - gender - region @@ -1845,8 +2291,10 @@ stages: - esa_contrib_reported - esa_income_reported reviewed_absent_outputs: - incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not a populated loader layer. - maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training source can be materialized for this stage. + incapacity_benefit_reported: Absent/all-default on the pinned enhanced-FRS export and certified Microcosm UK base; not + a populated loader layer. + maternity_allowance_reported: Absent from the pinned enhanced-FRS export and certified Microcosm UK base; no training + source can be materialized for this stage. postprocess: gross_savings_interest_income: stage1 INCBBS draw + stage2 tax_free_savings_income refresh_disability_categories: @@ -1949,7 +2397,9 @@ stages: directional_pass: 0 directional_fail: 0 excluded_with_fence: 208 - classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound is valid. + classification_rationale: Every published fact uses non-overlapping total-income bands. The FRS channel cannot materialize + full TEI, and omitted income can move a person between bands, so neither an exact fact nor a per-band directional bound + is valid. reviewed_fences: - fence_id: frs_epb_source_absent constituents: @@ -1961,9 +2411,12 @@ stages: - JOB.FUELAMT - JOB.VCHAMT - JOB.CHVAMT - finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce complete taxable expenses payments and benefits. - mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this is not monetary support. - rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an imputation or proxy. + finding: Missing. EXPBEN* are receipt flags, and the amount fields cover only selected benefits; they cannot produce + complete taxable expenses payments and benefits. + mass_implication: 12.9485464% of certified-candidate FRS effective person mass has at least one receipt flag, but this + is not monetary support. + rationale: Receipt flags and selected benefit amounts cannot be promoted to the SPI EPB monetary concept without an + imputation or proxy. dependent_fence_ids: [] - fence_id: frs_exps_source_absent constituents: @@ -1974,9 +2427,12 @@ stages: - JOB.UMILEAMT/JOB.UMOTAMT - JOB.DEDUC1-DEDUC9 - JOB.UDEDUC1-UDEDUC9 - finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense amount required by SPI. - mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the true EXPS mass is not estimable. - rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity with a negative sign. + finding: Missing. These fields describe reimbursements or payroll deductions, not the complete tax-deductible employment-expense + amount required by SPI. + mass_implication: 5.1302528% of certified-candidate FRS effective person mass has an adjacent reimbursement flag; the + true EXPS mass is not estimable. + rationale: The nearby fields do not measure the required deductible amount, and EXPS enters the employed-income identity + with a negative sign. dependent_fence_ids: [] - fence_id: frs_taxterm_source_absent constituents: @@ -1985,7 +2441,8 @@ stages: - ADULT.REDAMT - ADULT and JOB taxable-termination split search finding: Missing. REDAMT is gross redundancy pay and has neither the taxable amount nor non-redundancy termination pay. - mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable mass is unknown. + mass_implication: 0.3746084% of certified-candidate FRS effective person mass has positive gross redundancy pay; taxable + mass is unknown. rationale: Gross redundancy pay cannot be relabeled as taxable termination pay. dependent_fence_ids: [] - fence_id: frs_mothinc_source_absent @@ -1996,7 +2453,8 @@ stages: - ADULT.ALLPAY2 - ADULT.ROYYR2-ROYYR4 - JOB.OWNOTHER - finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI miscellaneous employment income would be a proxy. + finding: Missing. The fields are heterogeneous and belong to distinct income concepts; assigning their union to SPI + miscellaneous employment income would be a proxy. mass_implication: Odd-job-only effective person mass is 0.1724207%; the broader unresolved miscellaneous pool is 1.4650566%. rationale: The FRS instrument cannot separate the SPI miscellaneous-employment concept source-faithfully. dependent_fence_ids: [] @@ -2009,8 +2467,10 @@ stages: - ACCOUNTS - ASSETS - BENEFITS - finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be split between MOTHINC and OTHERINC from source evidence. - mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate FRS effective person mass. + finding: Missing. No person-level raw FRS variable has SPI OTHERINC semantics, and the miscellaneous pool cannot be + split between MOTHINC and OTHERINC from source evidence. + mass_implication: No separable mass estimate exists; the unresolved miscellaneous pool is 1.4650566% of certified-candidate + FRS effective person mass. rationale: A union of heterogeneous residual fields would be a new proxy, not a retained source constituent. dependent_fence_ids: [] - fence_id: frs_ossben_identifiable_subset @@ -2022,8 +2482,10 @@ stages: - BENEFITS.BENEFIT - BENEFITS.VAR2 - BENEFITS codes 13, 16, 6, and 30 - finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. - mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; it is not full OSSBEN support. + finding: Incomplete. Carer's Allowance and contribution-based ESA form an identifiable subset, but code 6 mixes tax + treatments and code 30 is an undifferentiated catch-all, so the complete taxable family cannot be emitted. + mass_implication: 1.8045088% of certified-candidate FRS effective person mass carries the identifiable lower-bound subset; + it is not full OSSBEN support. rationale: The retained column must remain explicitly named as a subset and cannot satisfy the full SPI concept. dependent_fence_ids: [] - fence_id: frs_srp_regular_code5_subset @@ -2033,9 +2495,12 @@ stages: raw_sources_searched: - BENEFITS.BENAMT where BENEFIT == 5 - BENEFITS codes 6 and 9 - finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. - mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; it is not complete SRP support. - rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published state-pension measure. + finding: Incomplete. Code 5 supplies regular State Pension, but the FRS source does not identify the full SPI combination + of State Pension lump sums and widow's pension; code 6 mixes benefits and code 9 is tax-free War Widow's Pension. + mass_implication: 18.1567916% of certified-candidate FRS effective person mass carries regular code-5 State Pension; + it is not complete SRP support. + rationale: The retained column must remain explicitly named as a subset and cannot be reported as the full published + state-pension measure. dependent_fence_ids: [] - fence_id: full_frs_tei_band_unavailable constituents: @@ -2047,9 +2512,12 @@ stages: - OSSBEN - SRP raw_sources_searched: [] - finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income band assignment is unavailable on the FRS channel. - mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable like-for-like measure. - rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' + finding: The complete FRS TEI measure cannot be materialized from retained source constituents, so exact HMRC total-income + band assignment is unavailable on the FRS channel. + mass_implication: Every one of the 208 published facts is banded by total income and therefore depends on this unavailable + like-for-like measure. + rationale: 'A component-level subset does not imply a per-band lower bound: omitted income can move a taxpayer into + or out of any non-overlapping published band. Biased partial bands are not emitted as estimates.' dependent_fence_ids: - frs_epb_source_absent - frs_exps_source_absent @@ -2101,4 +2569,15 @@ stages: - hmrc_spi_total_earned_income - hmrc_spi_total_investment_income - hmrc_spi_assessable_income - notes: 'Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent UK data build''s frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor.' + notes: 'Current-source adjudicated replay contract: the private 2022-23 SPI donor and public 2023-24 HMRC ODS are pinned + by reviewed SHA-256 and size and verified together before either is opened. The QRF draws source leaves; HMRC employed + income, TEI, TII, and TI are deterministic post-draw aggregates on the SPI channel, with TI exactly equal to TEI + TII. + PolicyEngine employment_income remains the narrow PAY + EPB + TAXTERM input on SPI rows. Stage 2 mirrors the incumbent + UK data build''s frs_only.py exactly: its income predictors are employment, self-employment, savings interest, dividends, + private pension, and property income. Other investment income remains a stage-1 SPI draw and official HMRC fact component, + but is excluded from stage 2 because the certified FRS candidate does not carry it. The FRS channel retains source-faithful + full PAY, UBISJA, and INCPBEN plus explicitly named ossben_identifiable_subset and srp_regular_code5; EPB, EXPS, TAXTERM, + MOTHINC, OTHERINC, full OSSBEN, and full SRP remain forbidden. Because the missing legs prevent a complete FRS TEI measure, + none of the 208 non-overlapping total-income-band facts is exact or directional. Every fact is an excluded-with-fence + record, no calibration is performed, and weights remain importance-kind. Gift Aid restoration still requires the rebuilt + positive-mass SPI channel to clear the reviewed 1ppm effective-mass floor.' diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py index 2e8f9abc..0c3f32de 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -4,6 +4,7 @@ import json from dataclasses import dataclass, field +from importlib.resources import files from pathlib import Path import numpy as np @@ -21,10 +22,7 @@ from microcosm.frame import Frame from microcosm.frame.rules import assert_rules_engine_country -ETB_SERVICES_YEAR = 2024 ETB_SERVICES_WEEKS_IN_YEAR = 52 -RAIL_FARE_INDEX_2023 = 1.110 -NHS_BUDGET_2025_26 = 202_000_000_000.0 UK_ETB_SERVICES_PREDICTORS = ( "is_adult", "is_child", @@ -95,6 +93,7 @@ def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: def __call__(self, frame: Frame) -> Frame: assert_rules_engine_country(self.engine, "uk") + config = etb_services_configuration(self.stage) raw = ( self.donor if self.donor is not None @@ -102,13 +101,19 @@ def __call__(self, frame: Frame) -> Frame: _require_path(self.etb_tab_path), self.stage.artifacts[0] ) ) - donor = clean_etb_services_table(raw) + donor = clean_etb_services_table( + raw, + year=config["year"], + weeks_in_year=config["weeks_in_year"], + ) predictors = recipient_predictors(frame, self.engine) draws, records = impute_etb_services( donor, predictors, seed=_qrf_seed(self.stage) ) draws = support_clip_to_donor(draws, donor) - draws["rail_usage"] = draws["rail_subsidy_spending"] / RAIL_FARE_INDEX_2023 + draws["rail_usage"] = ( + draws["rail_subsidy_spending"] / config["rail_fare_index"] + ) household = frame.table("household").copy() for column in UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS: household[column] = draws[column].to_numpy() @@ -118,6 +123,7 @@ def __call__(self, frame: Frame) -> Frame: household_weights=frame.weights_for("household").values, household=household, nhs_table=self.nhs_table, + nhs_budget=config["nhs_budget"], ) for column in UK_NHS_OUTPUT_COLUMNS: person[column] = nhs[column].to_numpy() @@ -139,12 +145,20 @@ def output_columns() -> tuple[str, ...]: return UK_ETB_SERVICES_OUTPUT_COLUMNS -def clean_etb_services_table(raw: pd.DataFrame) -> pd.DataFrame: +def clean_etb_services_table( + raw: pd.DataFrame, + *, + year: int | str | None = None, + weeks_in_year: int = ETB_SERVICES_WEEKS_IN_YEAR, +) -> pd.DataFrame: data = raw.replace(r"^\s*$", np.nan, regex=True).copy() if "year" not in data: raise ValueError("ETB services donor is missing 'year'.") data["year"] = pd.to_numeric(data["year"], errors="coerce") - data = data[data["year"] == data["year"].max()].copy() + selected_year = data["year"].max() if year in (None, "max") else year + if not np.isfinite(selected_year): + raise ValueError("ETB services donor has no finite year to select.") + data = data[data["year"] == int(selected_year)].copy() required = [ "adults", "childs", @@ -171,7 +185,7 @@ def clean_etb_services_table(raw: pd.DataFrame) -> pd.DataFrame: train = pd.DataFrame() train["is_adult"] = data["adults"] train["is_child"] = data["childs"] - train["hbai_household_net_income"] = data["disinc"] * ETB_SERVICES_WEEKS_IN_YEAR + train["hbai_household_net_income"] = data["disinc"] * weeks_in_year train["is_SP_age"] = data["noretd"] train["count_primary_education"] = data["primed"] train["count_secondary_education"] = data["secoed"] @@ -179,12 +193,45 @@ def clean_etb_services_table(raw: pd.DataFrame) -> pd.DataFrame: train["dla"] = data["disliv"] train["pip"] = data["pips"] train["weight"] = data["hhold_adj_weight"] - train["dfe_education_spending"] = data["educ"] * ETB_SERVICES_WEEKS_IN_YEAR - train["rail_subsidy_spending"] = data["rail"] * ETB_SERVICES_WEEKS_IN_YEAR - train["bus_subsidy_spending"] = data["bussub"] * ETB_SERVICES_WEEKS_IN_YEAR + train["dfe_education_spending"] = data["educ"] * weeks_in_year + train["rail_subsidy_spending"] = data["rail"] * weeks_in_year + train["bus_subsidy_spending"] = data["bussub"] * weeks_in_year return train +def load_etb_services_anchors() -> dict: + return json.loads( + files("microcosm.build.uk") + .joinpath("etb_services_anchors.json") + .read_text(encoding="utf-8") + ) + + +def etb_services_configuration(stage: SourceStageSpec | None = None) -> dict: + """Load service anchors and derive settings from the committed manifest.""" + + anchors = load_etb_services_anchors() + config = { + "year": "max", + "weeks_in_year": ETB_SERVICES_WEEKS_IN_YEAR, + "rail_fare_index": float(anchors["rail_fare_index_2023"]["value"]), + "nhs_budget": float(anchors["nhs_budget_2025_26"]["value"]), + } + if stage is not None: + derive = next( + operation + for operation in stage.operations + if operation.kind == "derive" + ) + if "year" in derive.parameters: + config["year"] = derive.parameters["year"] + if "annualization_weeks" in derive.parameters: + config["weeks_in_year"] = int(derive.parameters["annualization_weeks"]) + if config["year"] != "max": + raise ValueError("ETB services must select the manifest's maximum year.") + return config + + def household_grain_services_predictors(person_level: pd.DataFrame) -> pd.DataFrame: grouped = person_level.groupby("household_id", sort=False).sum(numeric_only=True) return grouped.loc[:, list(UK_ETB_SERVICES_PREDICTORS)] @@ -296,8 +343,14 @@ def parse_nhs_age_bounds(age_group: str) -> tuple[int, int]: def build_nhs_cell_table( - raw: pd.DataFrame, person: pd.DataFrame, household: pd.DataFrame + raw: pd.DataFrame, + person: pd.DataFrame, + household: pd.DataFrame, + *, + nhs_budget: float | None = None, ) -> pd.DataFrame: + if nhs_budget is None: + nhs_budget = float(load_etb_services_anchors()["nhs_budget_2025_26"]["value"]) nhs = raw.copy() bounds = nhs["Age group"].map(parse_nhs_age_bounds) nhs["Lower age"] = [lo for lo, _ in bounds] @@ -325,7 +378,7 @@ def build_nhs_cell_table( for _, row in pivot.iterrows() ] pivot["Per-person average units"] = pivot["Activity Count"] / pivot["Total people"] - factor = NHS_BUDGET_2025_26 / pivot["Total Cost"].sum() + factor = nhs_budget / pivot["Total Cost"].sum() pivot["Per-person average spending"] = ( pivot["Total Cost"] / pivot["Total people"] * factor ) @@ -338,6 +391,7 @@ def allocate_nhs_by_age_gender( household_weights: np.ndarray, household: pd.DataFrame, nhs_table: pd.DataFrame | None, + nhs_budget: float | None = None, ) -> pd.DataFrame: if nhs_table is None: path = ( @@ -351,7 +405,9 @@ def allocate_nhs_by_age_gender( household = household.assign( household_weight=np.asarray(household_weights, dtype=float) ) - cells = build_nhs_cell_table(nhs_table, person, household) + cells = build_nhs_cell_table( + nhs_table, person, household, nhs_budget=nhs_budget + ) output = pd.DataFrame(0.0, index=person.index, columns=UK_NHS_OUTPUT_COLUMNS) service_to_columns = { "A&E": ("a_and_e_visits", "nhs_a_and_e_spending"), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py index ca712a4b..1d21493e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py @@ -2,7 +2,9 @@ from __future__ import annotations +import json from dataclasses import dataclass, field +from importlib.resources import files from pathlib import Path import numpy as np @@ -23,9 +25,6 @@ ETB_FILENAME = "householdv2_1977-2024.tab" ETB_SHA256 = "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8" ETB_SIZE_BYTES = 216_967_663 -DEFAULT_ETB_VAT_YEAR = 2023 -VAT_STANDARD_RATE_2023 = 0.20 -VAT_REDUCED_RATE_SHARE_2023 = 0.025 UK_ETB_VAT_PREDICTORS = ( "is_adult", "is_child", @@ -60,6 +59,7 @@ def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: def __call__(self, frame: Frame) -> Frame: assert_rules_engine_country(self.engine, "uk") + config = etb_vat_configuration(self.stage) raw = ( self.donor if self.donor is not None @@ -67,7 +67,7 @@ def __call__(self, frame: Frame) -> Frame: _require_path(self.etb_tab_path), self.stage.artifacts[0] ) ) - donor = clean_etb_vat_table(raw) + donor = clean_etb_vat_table(raw, **config) predictors = recipient_predictors(frame, self.engine) imputed, record = impute_etb_vat(donor, predictors, seed=_qrf_seed(self.stage)) imputed = support_clip_to_donor(imputed, donor) @@ -96,10 +96,17 @@ def output_columns() -> tuple[str, ...]: def clean_etb_vat_table( raw: pd.DataFrame, *, - year: int = DEFAULT_ETB_VAT_YEAR, - standard_rate: float = VAT_STANDARD_RATE_2023, - reduced_rate_share: float = VAT_REDUCED_RATE_SHARE_2023, + year: int | None = None, + standard_rate: float | None = None, + reduced_rate_share: float | None = None, ) -> pd.DataFrame: + resource = _load_etb_vat_resource() + if year is None: + year = int(resource["vat"]["standard_rate"]["period"]) + if standard_rate is None: + standard_rate = float(resource["vat"]["standard_rate"]["value"]) + if reduced_rate_share is None: + reduced_rate_share = float(resource["vat"]["reduced_rate_share"]["value"]) if not np.isfinite(standard_rate) or standard_rate <= 0: raise ValueError("VAT standard_rate must be positive and finite.") if not np.isfinite(reduced_rate_share): @@ -129,12 +136,52 @@ def clean_etb_vat_table( train["is_SP_age"] = data["noretd"] train["household_net_income"] = data["disinc"] * 52 train["weight"] = data["hhold_adj_weight"] + denominator = data["expdis"] - data["totvat"] + if (denominator == 0).any(): + raise ValueError("ETB VAT donor contains zero disposable-expenditure denominators.") train["full_rate_vat_expenditure_rate"] = ( data["totvat"] * (1 - reduced_rate_share) / standard_rate - ) / (data["expdis"] - data["totvat"]) + ) / denominator + if not np.isfinite(train["full_rate_vat_expenditure_rate"]).all(): + raise ValueError("ETB VAT donor produced a non-finite VAT expenditure rate.") return train.dropna() +def _load_etb_vat_resource() -> dict: + return json.loads( + files("microcosm.build.uk") + .joinpath("etb_policy_anchors.json") + .read_text(encoding="utf-8") + ) + + +def etb_vat_configuration( + stage: SourceStageSpec | None = None, +) -> dict[str, float | int]: + """Load VAT settings from the committed resource and validate the manifest.""" + + resource = _load_etb_vat_resource()["vat"] + config: dict[str, float | int] = { + "year": int(resource["standard_rate"]["period"]), + "standard_rate": float(resource["standard_rate"]["value"]), + "reduced_rate_share": float(resource["reduced_rate_share"]["value"]), + } + if stage is not None: + derive = next( + operation + for operation in stage.operations + if operation.kind == "derive" + ) + for key, value in config.items(): + declared = derive.parameters.get(key) + if declared is not None and float(declared) != float(value): + raise ValueError( + f"ETB VAT manifest {key}={declared!r} disagrees with " + f"the committed anchor value {value!r}." + ) + return config + + def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: """Materialize ETB VAT recipient predictors at household grain. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 604f1756..3f3968b1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -42,7 +42,6 @@ LCFS_PERSON_SHA256 = "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50" LCFS_PERSON_SIZE_BYTES = 6_545_146 -NTS_ICE_SHARE = 0.90 UK_LCFS_CONSUMPTION_DECLARED_SEEDS = {"lcfs_consumption": 0} LCFS_REGIONS: Mapping[int, str] = { @@ -201,6 +200,7 @@ def fit_weight_records(self) -> tuple[FitWeightRecord, ...]: def __call__(self, frame: Frame) -> Frame: assert_rules_engine_country(self.engine, "uk") + anchors = load_lcfs_consumption_anchors() lcfs_household = ( self.lcfs_household if self.lcfs_household is not None @@ -228,12 +228,15 @@ def __call__(self, frame: Frame) -> Frame: was = clean_was_household_table(was_raw) donor = clean_lcfs_consumption_table(lcfs_person, lcfs_household) donor, bridge_record = bridge_has_fuel_to_lcfs( - donor, was, seed=_operation_seed(self.stage, "bridge_donor_column_via_qrf") + donor, + was, + seed=_operation_seed(self.stage, "bridge_donor_column_via_qrf"), + nts_ice_share=float(anchors["nts_ice_share"]["value"]), ) recipient = recipient_predictors(frame, self.engine) recipient["has_fuel_consumption"] = assign_recipient_has_fuel( frame, - rate=NTS_ICE_SHARE, + rate=float(anchors["nts_ice_share"]["value"]), seed=_operation_seed(self.stage, "assign_binary_from_rate"), ) imputation = impute_lcfs_consumption( @@ -388,11 +391,16 @@ def bridge_has_fuel_to_lcfs( *, seed: int, n_estimators: int = 100, + nts_ice_share: float | None = None, ) -> tuple[pd.DataFrame, FitWeightRecord]: """Fit a WAS has-fuel bridge and predict a clipped rate onto LCFS.""" from microcosm.fit import RegimeGatedQRF + if nts_ice_share is None: + nts_ice_share = float( + load_lcfs_consumption_anchors()["nts_ice_share"]["value"] + ) donor = was.copy() donor["has_fuel_consumption"] = ( (_numeric(donor["num_vehicles"]) > 0) @@ -400,7 +408,7 @@ def bridge_has_fuel_to_lcfs( stable_identity_uniforms( donor.index.to_numpy(), seed=seed, salt="was_has_fuel" ) - < NTS_ICE_SHARE + < nts_ice_share ) ).astype(float) # The LCFS frame carries its own names for three of the WAS bridge @@ -573,7 +581,7 @@ def rake_energy_to_need( frame = household.copy() frame["_need_income_band"] = _income_band(frame["household_gross_income"]) - margins = [MarginSpec("_need_income_band", _NEED_INCOME_TARGETS)] + margins = [MarginSpec("_need_income_band", _need_income_targets())] scratch = ["_need_income_band"] for name, values in ( ("tenure", tenure), @@ -602,27 +610,53 @@ def rake_energy_to_need( return raked.drop(columns=[c for c in scratch if c in raked]) -_NEED_INCOME_BANDS = ( - (0, 15_000, "under_15k", 7_755, 2_412), - (15_000, 20_000, "15k_20k", 9_196, 2_700), - (20_000, 30_000, "20k_30k", 9_886, 2_915), - (30_000, 40_000, "30k_40k", 10_697, 3_114), - (40_000, 50_000, "40k_50k", 11_230, 3_276), - (50_000, 60_000, "50k_60k", 11_721, 3_410), - (60_000, 70_000, "60k_70k", 12_200, 3_548), - (70_000, 100_000, "70k_100k", 13_244, 3_872), - (100_000, 150_000, "100k_150k", 15_727, 4_598), - (150_000, np.inf, "over_150k", 20_359, 5_944), -) -_ELEC_RATE = 24.67 / 100 -_GAS_RATE = 5.74 / 100 -_NEED_INCOME_TARGETS = { - name: { - "gas_consumption": gas * _GAS_RATE, - "electricity_consumption": elec * _ELEC_RATE, +def load_lcfs_consumption_anchors() -> dict: + from importlib.resources import files + + return json.loads( + files("microcosm.build.uk") + .joinpath("lcfs_consumption_anchors.json") + .read_text(encoding="utf-8") + ) + + +def _load_need_energy_targets() -> dict: + from importlib.resources import files + + return json.loads( + files("microcosm.build.uk") + .joinpath("need_energy_targets.json") + .read_text(encoding="utf-8") + ) + + +def _need_income_bands() -> tuple[tuple[float, float, str, float, float], ...]: + bands = [] + for band in _load_need_energy_targets()["income_bands"]: + upper = np.inf if band["upper"] is None else float(band["upper"]) + bands.append( + ( + float(band["lower"]), + upper, + str(band["label"]), + float(band["gas_kwh"]), + float(band["electricity_kwh"]), + ) + ) + return tuple(bands) + + +def _need_income_targets() -> dict: + rates = _load_need_energy_targets()["source"]["ofgem_q2_2026"] + gas_rate = float(rates["gas_gbp_per_kwh"]) + electricity_rate = float(rates["electricity_gbp_per_kwh"]) + return { + name: { + "gas_consumption": gas * gas_rate, + "electricity_consumption": electricity * electricity_rate, + } + for _, _, name, gas, electricity in _need_income_bands() } - for _, _, name, gas, elec in _NEED_INCOME_BANDS -} def _need_categorical_targets(margin: str) -> tuple[dict, dict]: @@ -632,13 +666,10 @@ def _need_categorical_targets(margin: str) -> tuple[dict, dict]: aggregate_admin anchors share one source of values. """ - from importlib.resources import files - - need = json.loads( - files("microcosm.build.uk") - .joinpath("need_energy_targets.json") - .read_text(encoding="utf-8") - ) + need = _load_need_energy_targets() + rates = need["source"]["ofgem_q2_2026"] + gas_rate = float(rates["gas_gbp_per_kwh"]) + electricity_rate = float(rates["electricity_gbp_per_kwh"]) block = need[margin] if margin == "region": mapping = {name: name for name in block["gas_kwh"]} @@ -646,9 +677,9 @@ def _need_categorical_targets(margin: str) -> tuple[dict, dict]: mapping = dict(block["map"]) targets = { frs_value: { - "gas_consumption": block["gas_kwh"][need_key] * _GAS_RATE, + "gas_consumption": block["gas_kwh"][need_key] * gas_rate, "electricity_consumption": ( - block["electricity_kwh"][need_key] * _ELEC_RATE + block["electricity_kwh"][need_key] * electricity_rate ), } for frs_value, need_key in mapping.items() @@ -659,7 +690,7 @@ def _need_categorical_targets(margin: str) -> tuple[dict, dict]: def _income_band(values: pd.Series) -> pd.Series: income = _numeric(values) result = pd.Series(index=income.index, dtype=object) - for lo, hi, name, _, _ in _NEED_INCOME_BANDS: + for lo, hi, name, _, _ in _need_income_bands(): result[(income >= lo) & (income < hi)] = name return result diff --git a/packages/microcosm-build/tests/test_uk_consumption_resources.py b/packages/microcosm-build/tests/test_uk_consumption_resources.py index d1681862..c528c909 100644 --- a/packages/microcosm-build/tests/test_uk_consumption_resources.py +++ b/packages/microcosm-build/tests/test_uk_consumption_resources.py @@ -20,6 +20,7 @@ def test_need_energy_targets_shape_and_citations() -> None: assert payload["version"] == 1 assert payload["country"] == "uk" assert payload["source"]["chronicle_candidate"] is True + assert payload["source"]["urls"] assert "NEED 2023" in payload["source"]["citation"] assert len(payload["income_bands"]) == 10 assert payload["tenure"]["map"]["OWNED_OUTRIGHT"] == "owner" @@ -33,6 +34,9 @@ def test_policy_anchor_resources_carry_parameter_paths() -> None: services = _load("etb_services_anchors.json") assert lcfs["source"]["chronicle_candidate"] is True + assert lcfs["source"]["urls"] + assert vat["source"]["urls"] + assert services["source"]["urls"] assert lcfs["cpi"]["parameter_path"] assert vat["vat"]["standard_rate"]["parameter_path"] == ( "gov.hmrc.vat.standard_rate" diff --git a/packages/microcosm-build/tests/test_uk_etb_services.py b/packages/microcosm-build/tests/test_uk_etb_services.py index 80fae448..8ace2e2d 100644 --- a/packages/microcosm-build/tests/test_uk_etb_services.py +++ b/packages/microcosm-build/tests/test_uk_etb_services.py @@ -5,14 +5,13 @@ import pytest from microcosm.build.uk_runtime.etb_services import ( - NHS_BUDGET_2025_26, - RAIL_FARE_INDEX_2023, UK_ETB_SERVICES_FIT_NAME, build_nhs_cell_table, clean_etb_services_table, donor_realized_ranges, household_grain_services_predictors, impute_etb_services, + load_etb_services_anchors, parse_nhs_age_bounds, support_clip_to_donor, ) @@ -157,7 +156,8 @@ def test_services_support_clip_ranges_and_rail_ratio() -> None: assert clipped["dfe_education_spending"].tolist() == [520.0, 1040.0] assert donor_realized_ranges(donor)["rail_subsidy_spending"] == (104.0, 208.0) - assert 111.0 / RAIL_FARE_INDEX_2023 == pytest.approx(100.0) + fare_index = load_etb_services_anchors()["rail_fare_index_2023"]["value"] + assert 111.0 / fare_index == pytest.approx(100.0) def _nhs_raw() -> pd.DataFrame: @@ -213,7 +213,7 @@ def test_nhs_age_parsing_and_85_plus_fold_in_uses_full_table_denominator() -> No assert top["Total people"] == 5.0 assert np.isclose( cells["Per-person average spending"].mul(cells["Total people"]).sum(), - NHS_BUDGET_2025_26, + load_etb_services_anchors()["nhs_budget_2025_26"]["value"], ) diff --git a/packages/microcosm-build/tests/test_uk_etb_vat.py b/packages/microcosm-build/tests/test_uk_etb_vat.py index 384f6924..e80da7a4 100644 --- a/packages/microcosm-build/tests/test_uk_etb_vat.py +++ b/packages/microcosm-build/tests/test_uk_etb_vat.py @@ -48,6 +48,14 @@ def test_etb_vat_cleaning_fails_loud_on_missing_rate() -> None: clean_etb_vat_table(_raw_etb(), standard_rate=np.nan) +def test_etb_vat_cleaning_fails_loud_on_zero_denominator() -> None: + raw = _raw_etb() + raw.loc[1, "expdis"] = raw.loc[1, "totvat"] + + with pytest.raises(ValueError, match="zero disposable-expenditure"): + clean_etb_vat_table(raw) + + def test_etb_vat_support_clip_and_ranges() -> None: donor = clean_etb_vat_table(_raw_etb()) draws = pd.DataFrame({"full_rate_vat_expenditure_rate": [-99.0, 99.0]}) diff --git a/packages/microcosm-build/tests/test_uk_nhs_allocation.py b/packages/microcosm-build/tests/test_uk_nhs_allocation.py index eeebe267..d8f1e0e4 100644 --- a/packages/microcosm-build/tests/test_uk_nhs_allocation.py +++ b/packages/microcosm-build/tests/test_uk_nhs_allocation.py @@ -4,9 +4,9 @@ import pandas as pd from microcosm.build.uk_runtime.etb_services import ( - NHS_BUDGET_2025_26, allocate_nhs_by_age_gender, build_nhs_cell_table, + load_etb_services_anchors, parse_nhs_age_bounds, ) @@ -66,7 +66,7 @@ def test_nhs_85_plus_fold_in_and_budget_normalization_use_full_table() -> None: assert top["Total people"] == 3.0 assert np.isclose( cells["Per-person average spending"].mul(cells["Total people"]).sum(), - NHS_BUDGET_2025_26, + load_etb_services_anchors()["nhs_budget_2025_26"]["value"], ) # The real frame's household table has no household_weight column (weights diff --git a/packages/microcosm-build/tests/test_uk_raking.py b/packages/microcosm-build/tests/test_uk_raking.py index 42f5af6a..88be385d 100644 --- a/packages/microcosm-build/tests/test_uk_raking.py +++ b/packages/microcosm-build/tests/test_uk_raking.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd +import pytest from microcosm.build.raking import MarginSpec, iterative_proportional_fit @@ -86,7 +87,7 @@ def test_weighted_and_unweighted_means_use_distinct_denominators() -> None: np.testing.assert_allclose(weighted["value"], [16.0, 48.0]) -def test_zero_empty_and_unmapped_cells_are_left_untouched() -> None: +def test_zero_target_empty_and_unmapped_cells_are_left_untouched() -> None: frame = pd.DataFrame( { "band": ["zero", "empty", "unmapped"], @@ -101,7 +102,7 @@ def test_zero_empty_and_unmapped_cells_are_left_untouched() -> None: MarginSpec( "band", { - "zero": {"value": 10.0}, + "zero": {"value": 0.0}, "absent": {"value": 20.0}, }, ), @@ -112,6 +113,19 @@ def test_zero_empty_and_unmapped_cells_are_left_untouched() -> None: assert raked["value"].tolist() == [0.0, 5.0, 7.0] +def test_positive_target_with_zero_current_mean_fails_closed() -> None: + frame = pd.DataFrame({"band": ["zero"], "value": [0.0]}) + + with pytest.raises(ValueError, match="current mean is zero"): + iterative_proportional_fit( + frame, + columns=("value",), + margins=(MarginSpec("band", {"zero": {"value": 10.0}}),), + iterations=1, + fail_on_unattainable=True, + ) + + def test_margin_sweep_order_is_observable_and_pinned() -> None: frame = pd.DataFrame( { diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index 25f8acc2..a151d80f 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -639,12 +639,12 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: == UK_ETB_SERVICES_ENGINE_VARIABLES ) assert ( - tuple( + set( stages["etb_services"] .operations[1] .parameters["derived_predictors"] ) - == tuple(UK_ETB_SERVICES_EDUCATION_COUNTS) + == set(UK_ETB_SERVICES_EDUCATION_COUNTS) ) assert ( tuple(stages["etb_services"].operations[2].parameters["targets"]) diff --git a/tools/verify_uk_identity_stability.py b/tools/verify_uk_identity_stability.py index 01779003..c816571e 100644 --- a/tools/verify_uk_identity_stability.py +++ b/tools/verify_uk_identity_stability.py @@ -345,8 +345,11 @@ def e6_identity_receipt( """ from microcosm.build.uk_runtime.etb_services import ( - RAIL_FARE_INDEX_2023, allocate_nhs_by_age_gender, + load_etb_services_anchors, + ) + rail_fare_index = float( + load_etb_services_anchors()["rail_fare_index_2023"]["value"] ) def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: @@ -363,7 +366,7 @@ def recompute(person_t, benunit_t, household_t) -> dict[str, pd.DataFrame]: if "rail_subsidy_spending" in household.columns: household_out["rail_usage"] = ( household["rail_subsidy_spending"].to_numpy(dtype=float) - / RAIL_FARE_INDEX_2023 + / rail_fare_index ) if {"has_fuel_consumption", "petrol_spending", "diesel_spending"} <= set( household.columns