Port the LCFS consumption, ETB VAT, and ETB services/NHS layers as declarative source stages

changelog.d/682-uk-lcfs-consumption.added.md

@@ -0,0 +1,6 @@
+UK E6 source stages: `lcfs_consumption` (18-output LCFS QRF chain with the WAS
+has-fuel bridge and NEED energy raking as a generic `iterative_proportional_fit`
+operation), `etb_vat`, and `etb_services` (public-services QRF, NHS age-gender
+person allocation, `rail_usage`), with cited NEED/Ofgem/VAT/NHS anchor
+resources, sha-bound SDC-rounded support bounds, the `uk_aggregate_admin` gate,
+and the carried #717 review fixes (microcosm#682).

packages/microcosm-build/src/microcosm/build/raking.py

@@ -0,0 +1,114 @@
+"""Generic iterative proportional fitting helpers for source stages."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+
+import numpy as np
+import pandas as pd
+
+__all__ = ["MarginSpec", "iterative_proportional_fit"]
+
+
+@dataclass(frozen=True)
+class MarginSpec:
+    """Mean targets for one categorical margin.
+
+    ``targets`` maps category value -> output column -> target mean. The helper
+    deliberately works in means because the UK LCFS/NEED application receives
+    published mean kWh/spend cells, but the utility is country-neutral.
+    """
+
+    column: str
+    targets: Mapping[object, Mapping[str, float]]
+
+
+def iterative_proportional_fit(
+    frame: pd.DataFrame,
+    *,
+    columns: Sequence[str],
+    margins: Sequence[MarginSpec],
+    iterations: int,
+    weight_column: str | None = None,
+    fail_on_unattainable: bool = False,
+) -> pd.DataFrame:
+    """Scale columns in-place-by-copy to match declared cell means.
+
+    Empty cells and categories absent from the declared targets are skipped.
+    Populated zero-current-mean cells with positive targets are recorded in the
+    returned frame's ``raking_zero_current_cells`` evidence attribute. Callers
+    that require fail-closed behavior can set ``fail_on_unattainable``.
+    """
+
+    if iterations < 1:
+        raise ValueError("iterations must be positive")
+    if not columns:
+        raise ValueError("at least one column is required")
+
+    result = frame.copy()
+    for column in columns:
+        if column not in result:
+            raise KeyError(f"frame is missing raked column {column!r}")
+        result[column] = pd.to_numeric(result[column], errors="coerce").fillna(0.0)
+    if weight_column is not None and weight_column not in result:
+        raise KeyError(f"frame is missing weight column {weight_column!r}")
+
+    weights = None
+    if weight_column is not None:
+        weights = pd.to_numeric(result[weight_column], errors="coerce").fillna(0.0)
+        if (weights < 0).any():
+            raise ValueError("raking weights must be nonnegative")
+
+    zero_current_cells: list[dict[str, object]] = []
+    for _ in range(iterations):
+        for margin in margins:
+            if margin.column not in result:
+                raise KeyError(f"frame is missing margin column {margin.column!r}")
+            for category, target_by_column in margin.targets.items():
+                mask = result[margin.column] == category
+                if not bool(mask.any()):
+                    continue
+                for column in columns:
+                    if column not in target_by_column:
+                        continue
+                    current = _cell_mean(
+                        result.loc[mask, column],
+                        None if weights is None else weights.loc[mask],
+                    )
+                    target = float(target_by_column[column])
+                    if not np.isfinite(target) or target < 0:
+                        raise ValueError(
+                            f"target for {margin.column!r}={category!r}, "
+                            f"{column!r} must be finite and nonnegative"
+                        )
+                    if current <= 0 or not np.isfinite(current):
+                        if target > 0:
+                            evidence = {
+                                "margin": margin.column,
+                                "category": category,
+                                "column": column,
+                                "target": target,
+                            }
+                            zero_current_cells.append(evidence)
+                            if fail_on_unattainable:
+                                raise ValueError(
+                                    f"cannot rake {margin.column!r}={category!r} "
+                                    f"for {column!r}: current mean is "
+                                    f"zero/non-finite but target is {target}."
+                                )
+                        continue
+                    result.loc[mask, column] *= target / current
+    result.attrs["raking_zero_current_cells"] = tuple(zero_current_cells)
+    return result
+
+
+def _cell_mean(values: pd.Series, weights: pd.Series | None) -> float:
+    data = pd.to_numeric(values, errors="coerce").fillna(0.0).to_numpy(dtype=float)
+    if weights is None:
+        return float(data.mean()) if len(data) else 0.0
+    w = weights.to_numpy(dtype=float)
+    total = float(w.sum())
+    if total <= 0:
+        return 0.0
+    return float(np.dot(data, w) / total)

packages/microcosm-build/src/microcosm/build/source_manifest.py

@@ -49,11 +49,13 @@
         "assign_clipped_normal",
         "assign_uniform_draw",
         "aggregate_person_to_benunit",
+        "allocate_per_capita_from_cell_table",
         "allocate_within_group_waterfall",
         "allocate_zero_weight_prior_mass",
         "annualize_periodic_amounts",
         "assemble_group_entities",
         "attribute_self_employed_health_premiums",
+        "bridge_donor_column_via_qrf",
         "calibrate_binary_assignment",
         "calibrate_binary_assignment_joint_targets",
         "classify_hmrc_income_facts_with_reviewed_fences",
@@ -112,6 +114,7 @@
         "impute_retirement_distributions_to_puf_support",
         "impute_workers_compensation_to_puf_support",
         "impute_weeks_unemployed_to_puf_support",
+        "iterative_proportional_fit",
         "map_columns",
         "map_coded_amounts",
         "materialize_hmrc_income_bands_fail_closed",

packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json

@@ -5013,6 +5013,12 @@
       },
       "runtime_sha256_required": {
        "type": "boolean"
+      },
+      "format": {
+       "type": "string"
+      },
+      "vintage": {
+       "type": "string"
       }
      },
      "required": [
@@ -5771,6 +5777,140 @@
       }
      }
     },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": [
+      "kind",
+      "role",
+      "resource",
+      "format"
+     ],
+     "properties": {
+      "kind": {
+       "const": "public_parameter_reference"
+      },
+      "role": {
+       "type": "string"
+      },
+      "resource": {
+       "type": "string"
+      },
+      "format": {
+       "type": "string"
+      }
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind"],
+     "properties": {
+      "kind": {
+       "enum": [
+        "allocate_per_capita_from_cell_table",
+        "assign_binary_from_rate",
+        "bridge_donor_column_via_qrf",
+        "compute_ratio",
+        "derive",
+        "iterative_proportional_fit",
+        "zero_when_false"
+       ]
+      },
+      "age_bands": {"type": "string"},
+      "annualization_weeks": {"type": "number"},
+      "budget_resource": {"type": "string"},
+      "categorical_predictors": {"type": "array", "items": {"type": "string"}},
+      "chain_order": {"type": "array", "items": {"type": "string"}},
+      "columns": {"type": "array", "items": {"type": "string"}},
+      "condition": {"type": "string"},
+      "denominator_key": {"type": "string"},
+      "denominator_resource": {"type": "string"},
+      "donor_weight": {"type": "string"},
+      "exempt": {"type": "array", "items": {"type": "string"}},
+      "fail_loud_on_missing_rate": {"type": "boolean"},
+      "iterations": {"type": "integer", "minimum": 1},
+      "lossy_mappings": {"type": "array", "items": {"type": "string"}},
+      "logged_dropna_row_count": {"type": "boolean"},
+      "margins": {"type": "array", "items": {"type": "string"}},
+      "n_estimators": {"type": "integer", "minimum": 1},
+      "numerator": {"type": "string"},
+      "output": {"type": "string"},
+      "predictors": {"type": "array", "items": {"type": "string"}},
+      "rate_key": {"type": "string"},
+      "range": {"type": "string"},
+      "reduced_rate_share": {"type": "number"},
+      "resource": {"type": "string"},
+      "seed": {"type": "integer"},
+      "source": {"type": "string"},
+      "standard_rate": {"type": "number"},
+      "target": {"type": "string"},
+      "targets": {"type": "array", "items": {"type": "string"}},
+      "top_band_fold_in": {"type": "string"},
+      "weights": {"type": "string"},
+      "weighted": {"type": "boolean"},
+      "year": {"type": ["string", "integer"]},
+      "lowercase_columns": {"type": "boolean"}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "predictors", "targets", "seed"],
+     "properties": {
+      "kind": {"const": "fit_weighted_qrf"},
+      "predictors": {"type": "array", "items": {"type": "string"}},
+      "targets": {"type": "array", "items": {"type": "string"}},
+      "weights": {"type": "string"},
+      "n_estimators": {"type": "integer", "minimum": 1},
+      "seed": {"type": "integer"}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "predictors", "targets", "seed"],
+     "properties": {
+      "kind": {"const": "fit_weighted_qrf_chain"},
+      "predictors": {"type": "array", "items": {"type": "string"}},
+      "targets": {"type": "array", "items": {"type": "string"}},
+      "categorical_predictors": {"type": "array", "items": {"type": "string"}},
+      "weights": {"type": "string"},
+      "n_estimators": {"type": "integer", "minimum": 1},
+      "seed": {"type": "integer"}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "range", "exempt"],
+     "properties": {
+      "kind": {"const": "support_clip"},
+      "range": {"type": "string"},
+      "exempt": {"type": "array", "items": {"type": "string"}}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "predictors", "derived_predictors"],
+     "properties": {
+      "kind": {"const": "materialize_rules_engine_predictors"},
+      "predictors": {"type": "array", "items": {"type": "string"}},
+      "derived_predictors": {"type": "object", "additionalProperties": {"type": "string"}}
+     }
+    },
+    {
+     "type": "object",
+     "additionalProperties": false,
+     "required": ["kind", "output", "inputs"],
+     "properties": {
+      "kind": {"const": "fold_into"},
+      "output": {"type": "string"},
+      "inputs": {"type": "array", "items": {"type": "string"}},
+      "drop_inputs": {"type": "boolean"}
+     }
+    },
     {
      "type": "object",
      "additionalProperties": false,
@@ -6446,6 +6586,12 @@
       },
       "runtime_sha256_required": {
        "type": "boolean"
+      },
+      "format": {
+       "type": "string"
+      },
+      "vintage": {
+       "type": "string"
       }
      }
     },

packages/microcosm-build/src/microcosm/build/uk/country_package.json

@@ -77,6 +77,46 @@
       "kind": "legacy_json",
       "schema_id": "legacy_json"
     },
+    {
+      "path": "need_energy_targets.json",
+      "kind": "legacy_json",
+      "schema_id": "legacy_json"
+    },
+    {
+      "path": "lcfs_consumption_anchors.json",
+      "kind": "legacy_json",
+      "schema_id": "legacy_json"
+    },
+    {
+      "path": "etb_policy_anchors.json",
+      "kind": "legacy_json",
+      "schema_id": "legacy_json"
+    },
+    {
+      "path": "etb_services_anchors.json",
+      "kind": "legacy_json",
+      "schema_id": "legacy_json"
+    },
+    {
+      "path": "nhs_consumption_by_age_gender.json",
+      "kind": "legacy_json",
+      "schema_id": "legacy_json"
+    },
+    {
+      "path": "lcfs_consumption_support_bounds.json",
+      "kind": "legacy_json",
+      "schema_id": "legacy_json"
+    },
+    {
+      "path": "etb_vat_support_bounds.json",
+      "kind": "legacy_json",
+      "schema_id": "legacy_json"
+    },
+    {
+      "path": "etb_services_support_bounds.json",
+      "kind": "legacy_json",
+      "schema_id": "legacy_json"
+    },
     {
       "path": "regional_land_values.json",
       "kind": "legacy_json",

packages/microcosm-build/src/microcosm/build/uk/etb_policy_anchors.json

@@ -0,0 +1,27 @@
+{
+  "version": 1,
+  "country": "uk",
+  "source": {
+    "citation": "PolicyEngine UK VAT parameters for 2023; incumbent fallback removed.",
+    "urls": [
+      "https://www.gov.uk/guidance/rates-and-allowances-for-vat",
+      "https://github.com/PolicyEngine/policyengine-uk/tree/main/policyengine_uk/parameters/gov/hmrc/vat"
+    ],
+    "chronicle_candidate": true
+  },
+  "vat": {
+    "standard_rate": {
+      "value": 0.2,
+      "period": 2023,
+      "parameter_path": "gov.hmrc.vat.standard_rate"
+    },
+    "reduced_rate_share": {
+      "value": 0.025,
+      "period": 2023,
+      "parameter_path": "gov.hmrc.vat.reduced_rate_share"
+    }
+  },
+  "chronicle": [
+    "E6 committed VAT anchors for ETB VAT imputation."
+  ]
+}