policyengine-uk 2.40.1__py3-none-any.whl → 2.65.6__py3-none-any.whl

policyengine_uk/system.py

@@ -1,257 +1,3 @@
-from pathlib import Path
-from policyengine_uk.entities import entities
-from policyengine_core.data import Dataset
-from policyengine_core.taxbenefitsystems import TaxBenefitSystem
-from policyengine_core.simulations import (
-    Simulation as CoreSimulation,
-    Microsimulation as CoreMicrosimulation,
-)
-from policyengine_uk.data.dataset_schema import (
-    UKSingleYearDataset,
-    UKMultiYearDataset,
-)
-from policyengine_core.tools.hugging_face import download_huggingface_dataset
-
-import pandas as pd
-from policyengine_uk.utils.parameters import (
-    backdate_parameters,
-    convert_to_fiscal_year_parameters,
-)
-from policyengine_uk.parameters.gov.economic_assumptions.create_economic_assumption_indices import (
-    create_economic_assumption_indices,
-)
-from policyengine_uk.parameters.gov.economic_assumptions.lag_average_earnings import (
-    add_lagged_earnings,
-)
-from policyengine_uk.parameters.gov.economic_assumptions.lag_cpi import (
-    add_lagged_cpi,
-)
-from policyengine_core.reforms import Reform
-from policyengine_uk.reforms import create_structural_reforms_from_parameters
-
-from policyengine_uk.parameters.gov.contrib.create_private_pension_uprating import (
-    add_private_pension_uprating_factor,
-)
-from policyengine_uk.parameters.gov.dwp.state_pension.triple_lock.create_triple_lock import (
-    add_triple_lock,
-)
-from policyengine_core.parameters.operations.homogenize_parameters import (
-    homogenize_parameter_structures,
-)
-from policyengine_core.parameters.operations.interpolate_parameters import (
-    interpolate_parameters,
-)
-from policyengine_core.parameters.operations.propagate_parameter_metadata import (
-    propagate_parameter_metadata,
-)
-from policyengine_core.parameters.operations.uprate_parameters import (
-    uprate_parameters,
-)
-from policyengine_core.reforms import Reform
-
-COUNTRY_DIR = Path(__file__).parent
-
-ENHANCED_FRS = "hf://policyengine/policyengine-uk-data/enhanced_frs_2023_24.h5"
-
-
-class CountryTaxBenefitSystem(TaxBenefitSystem):
-    variables_dir = COUNTRY_DIR / "variables"
-    auto_carry_over_input_variables = True
-    basic_inputs = [
-        "brma",
-        "local_authority",
-        "region",
-        "employment_income",
-        "age",
-    ]
-    modelled_policies = COUNTRY_DIR / "modelled_policies.yaml"
-
-    def process_parameters(self, reform=None):
-        if reform:
-            self.apply_reform_set(reform)
-        self.parameters = add_private_pension_uprating_factor(self.parameters)
-        self.parameters = add_lagged_earnings(self.parameters)
-        self.parameters = add_lagged_cpi(self.parameters)
-        self.parameters = add_triple_lock(self.parameters)
-        self.parameters = create_economic_assumption_indices(self.parameters)
-        self.parameters.add_child("baseline", self.parameters.clone())
-        self.parameters = homogenize_parameter_structures(
-            self.parameters, self.variables
-        )
-        self.parameters = propagate_parameter_metadata(self.parameters)
-        self.parameters = interpolate_parameters(self.parameters)
-        self.parameters = uprate_parameters(self.parameters)
-        self.parameters = propagate_parameter_metadata(self.parameters)
-        self.add_abolition_parameters()
-        self.parameters = backdate_parameters(self.parameters, "2015-01-01")
-
-        self.parameters.gov = convert_to_fiscal_year_parameters(
-            self.parameters.gov
-        )
-
-    def __init__(self, reform=None):
-        super().__init__(entities, reform=reform)
-
-        self.parameters_dir = COUNTRY_DIR / "parameters"
-
-        self.load_parameters(self.parameters_dir)
-
-        self.process_parameters(reform=reform)
-
-
-system = CountryTaxBenefitSystem()
-
-parameters = system.parameters
-variables = system.variables
-
-
-class Simulation(CoreSimulation):
-    default_tax_benefit_system = CountryTaxBenefitSystem
-    default_tax_benefit_system_instance = system
-    default_calculation_period = 2023
-    default_input_period = 2023
-    default_role = "member"
-    max_spiral_loops = 10
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        reform = create_structural_reforms_from_parameters(
-            self.tax_benefit_system.parameters, "2023-01-01"
-        )
-        if reform is not None:
-            self.apply_reform(reform)
-
-        reform_dict = kwargs.get("reform")
-        if reform_dict is not None:
-            if isinstance(reform_dict, type):
-                try:
-                    reform_dict = reform_dict.parameter_values
-                except:
-                    reform_dict = None
-
-        if reform_dict is not None:
-            if any(["obr" in param for param in reform_dict]):
-                self.tax_benefit_system.load_parameters(
-                    self.tax_benefit_system.parameters_dir
-                )
-                Reform.from_dict(reform_dict).apply(self.tax_benefit_system)
-                self.tax_benefit_system.process_parameters()
-
-        # Labor supply responses
-
-        employment_income = self.get_holder("employment_income")
-        for known_period in employment_income.get_known_periods():
-            array = employment_income.get_array(known_period)
-            self.set_input("employment_income_before_lsr", known_period, array)
-            employment_income.delete_arrays(known_period)
-
-        # Capital gains responses
-
-        cg_holder = self.get_holder("capital_gains")
-        for known_period in cg_holder.get_known_periods():
-            array = cg_holder.get_array(known_period)
-            self.set_input(
-                "capital_gains_before_response", known_period, array
-            )
-            employment_income.delete_arrays(known_period)
-
-
-class Microsimulation(CoreMicrosimulation):
-    default_tax_benefit_system = CountryTaxBenefitSystem
-    default_dataset = ENHANCED_FRS
-    default_dataset_year = 2022
-    default_tax_benefit_system_instance = system
-    default_calculation_period = 2025
-    default_input_period = 2025
-    default_role = "member"
-    max_spiral_loops = 10
-
-    def __init__(self, *args, dataset=ENHANCED_FRS, **kwargs):
-        if dataset is not None:
-            if isinstance(dataset, str):
-                if "hf://" in dataset:
-                    owner, repo, filename = dataset.split("/")[-3:]
-                    if "@" in filename:
-                        version = filename.split("@")[-1]
-                        filename = filename.split("@")[0]
-                    else:
-                        version = None
-                    dataset_file_path = download_huggingface_dataset(
-                        repo=f"{owner}/{repo}",
-                        repo_filename=filename,
-                        version=version,
-                    )
-
-                if Path(dataset_file_path).exists():
-                    if dataset_file_path.endswith(".h5"):
-                        try:
-                            UKSingleYearDataset.validate_file_path(
-                                dataset_file_path
-                            )
-                            dataset = UKSingleYearDataset(
-                                file_path=dataset_file_path
-                            )
-                        except:
-                            pass
-
-                        try:
-                            UKMultiYearDataset.validate_file_path(
-                                dataset_file_path
-                            )
-                            dataset = UKMultiYearDataset(
-                                file_path=dataset_file_path
-                            )
-                        except Exception as e:
-                            pass
-
-                        if not isinstance(
-                            dataset, (UKSingleYearDataset, UKMultiYearDataset)
-                        ):
-                            dataset = Dataset.from_file(dataset_file_path)
-
-        super().__init__(*args, dataset=dataset, **kwargs)
-
-        reform = create_structural_reforms_from_parameters(
-            self.tax_benefit_system.parameters, "2023-01-01"
-        )
-        if reform is not None:
-            self.apply_reform(reform)
-
-        reform_dict = kwargs.get("reform")
-        if reform_dict is not None:
-            if isinstance(reform_dict, type):
-                try:
-                    reform_dict = reform_dict.parameter_values
-                except:
-                    reform_dict = None
-
-        if reform_dict is not None:
-            if any(["obr" in param for param in reform_dict]):
-                self.tax_benefit_system.load_parameters(
-                    self.tax_benefit_system.parameters_dir
-                )
-                Reform.from_dict(reform_dict).apply(self.tax_benefit_system)
-                self.tax_benefit_system.process_parameters()
-
-        # Labor supply responses
-
-        for simulation in list(self.branches.values()) + [self]:
-            employment_income = simulation.get_holder("employment_income")
-            for known_period in employment_income.get_known_periods():
-                array = employment_income.get_array(known_period)
-                simulation.set_input(
-                    "employment_income_before_lsr", known_period, array
-                )
-                employment_income.delete_arrays(known_period)
-
-        # Capital gains responses
-
-        for simulation in list(self.branches.values()) + [self]:
-            cg_holder = self.get_holder("capital_gains")
-            for known_period in cg_holder.get_known_periods():
-                array = cg_holder.get_array(known_period)
-                self.set_input(
-                    "capital_gains_before_response", known_period, array
-                )
-                employment_income.delete_arrays(known_period)
+from .tax_benefit_system import *
+from .simulation import *
+from .microsimulation import *

policyengine_uk/tax_benefit_system.py

@@ -0,0 +1,141 @@
+# Standard library imports
+import copy
+from pathlib import Path
+from typing import Any, Dict, List
+
+# PolicyEngine core imports
+from policyengine_core.parameters.operations.propagate_parameter_metadata import (
+    propagate_parameter_metadata,
+)
+from policyengine_core.parameters.operations.uprate_parameters import (
+    uprate_parameters,
+)
+from policyengine_core.taxbenefitsystems import TaxBenefitSystem
+from policyengine_core.variables import Variable
+
+# PolicyEngine UK imports
+from policyengine_uk.entities import BenUnit, Household, Person
+from policyengine_uk.parameters.gov.contrib.create_private_pension_uprating import (
+    add_private_pension_uprating_factor,
+)
+from policyengine_uk.parameters.gov.dwp.state_pension.triple_lock.create_triple_lock import (
+    add_triple_lock,
+)
+from policyengine_uk.parameters.gov.economic_assumptions.create_economic_assumption_indices import (
+    create_economic_assumption_indices,
+)
+from policyengine_uk.parameters.gov.economic_assumptions.lag_average_earnings import (
+    add_lagged_earnings,
+)
+from policyengine_uk.parameters.gov.economic_assumptions.lag_cpi import (
+    add_lagged_cpi,
+)
+from policyengine_uk.utils.parameters import (
+    backdate_parameters,
+    convert_to_fiscal_year_parameters,
+)
+
+# Module constants
+COUNTRY_DIR = Path(__file__).parent
+ENHANCED_FRS = "hf://policyengine/policyengine-uk-data/enhanced_frs_2023_24.h5"
+
+
+class CountryTaxBenefitSystem(TaxBenefitSystem):
+    """UK-specific tax and benefit system implementation.
+
+    This class defines the UK tax-benefit system with all relevant
+    variables, parameters, and entities (Person, BenUnit, Household).
+    """
+
+    basic_inputs: List[str] = [
+        "brma",
+        "local_authority",
+        "region",
+        "employment_income",
+        "age",
+    ]
+    modelled_policies = COUNTRY_DIR / "modelled_policies.yaml"
+    auto_carry_over_input_variables: bool = True
+
+    variables: Dict[str, Variable]
+
+    def reset_parameter_caches(self):
+        """Reset all caches in the tax-benefit system."""
+        self._parameters_at_instant_cache = {}
+        for parameter in self.parameters.get_descendants():
+            parameter._at_instant_cache = {}
+        self.parameters._at_instant_cache = {}
+
+    def reset_parameters(self) -> None:
+        """Reset parameters by reloading from the parameters directory."""
+        self._parameters_at_instant_cache = {}
+        self.load_parameters(self.parameters_dir)
+
+    def process_parameters(self) -> None:
+        """Process and transform parameters with UK-specific adjustments.
+
+        Applies various parameter transformations including:
+        - Private pension uprating factors
+        - Lagged earnings and CPI indices
+        - Triple lock calculations for state pensions
+        - Economic assumption indices
+        - Parameter uprating and backdating
+        - Conversion to fiscal year parameters
+        """
+        self._parameters_at_instant_cache = {}
+        # Add various UK-specific parameter adjustments
+        self.parameters = add_private_pension_uprating_factor(self.parameters)
+        self.parameters = add_lagged_earnings(self.parameters)
+        self.parameters = add_lagged_cpi(self.parameters)
+        self.parameters = add_triple_lock(self.parameters)
+        self.parameters = create_economic_assumption_indices(self.parameters)
+
+        # Create baseline parameters for reform comparisons
+        self.parameters.add_child("baseline", self.parameters.clone())
+
+        # Apply general parameter operations
+        self.parameters = propagate_parameter_metadata(self.parameters)
+        self.parameters = uprate_parameters(self.parameters)
+        self.parameters = backdate_parameters(self.parameters, "2015-01-01")
+        self.parameters.gov = convert_to_fiscal_year_parameters(
+            self.parameters.gov
+        )
+        self.reset_parameter_caches()
+
+    def __init__(self):
+        """Initialize the UK tax-benefit system with entities and parameters."""
+        self._parameters_at_instant_cache: Dict[str, Any] = {}
+        self.variables = {}
+
+        # Create copies of entity classes to avoid modifying originals
+        person, benunit, household = (
+            copy.copy(Person),
+            copy.copy(BenUnit),
+            copy.copy(Household),
+        )
+
+        # Set up entities
+        self.entities = [person, benunit, household]
+        self.person_entity = person
+        self.group_entities = [benunit, household]
+        self.group_entity_keys = [entity.key for entity in self.group_entities]
+
+        # Link entities to this tax-benefit system
+        for entity in self.entities:
+            entity.set_tax_benefit_system(self)
+
+        self.variable_module_metadata = {}
+
+        # Load all variables from the variables directory
+        self.add_variables_from_directory(COUNTRY_DIR / "variables")
+
+        # Set up and process parameters
+        self.parameters_dir = COUNTRY_DIR / "parameters"
+        self.reset_parameters()
+        self.process_parameters()
+
+
+# Create system instance for module-level access
+system = CountryTaxBenefitSystem()
+parameters = system.parameters
+variables = system.variables

policyengine_uk/tests/behavioral_responses/test_labor_supply_responses.yaml

@@ -0,0 +1,183 @@
+# Test 1: OBR parameter enabled - Verifies behavioral response system can be activated
+# This test ensures that:
+# - No more simulation state corruption from sim.reset_calculations()
+# - Income changes are calculated before any state modifications
+# - The system correctly returns 0 FTE responses when there's no actual policy reform
+# - Proper NaN handling prevents calculation errors
+- name: Test basic behavioral response mechanism works
+  period: 2025
+  input:
+    people:
+      parent:
+        age: 30
+        employment_income: 20_000
+        hours_worked: 1500
+      child1:
+        age: 5
+      child2:
+        age: 3
+    benunits:
+      benunit:
+        members: [parent, child1, child2]
+    households:
+      household:
+        members: [parent, child1, child2]
+  reforms:
+    gov.dynamic.obr_labour_supply_assumptions:
+      "2025": true
+  output:
+    employment_income:
+      parent: 20_000  # Should have behavioral response capability
+
+# Test 2: OBR parameter enabled with married couple - Verifies different gender behavioral responses
+# This test ensures that:
+# - No more simulation state corruption from sim.reset_calculations()
+# - Income changes are calculated before any state modifications
+# - The system correctly returns 0 FTE responses when there's no actual policy reform
+# - Proper NaN handling prevents calculation errors
+- name: Test married couple behavioral response
+  period: 2025
+  input:
+    people:
+      adult1:
+        age: 35
+        employment_income: 25_000
+        hours_worked: 1800
+        gender: MALE
+      adult2:
+        age: 33
+        employment_income: 18_000
+        hours_worked: 1200
+        gender: FEMALE
+      child1:
+        age: 6
+      child2:
+        age: 4
+    benunits:
+      benunit:
+        members: [adult1, adult2, child1, child2]
+        is_married: true
+    households:
+      household:
+        members: [adult1, adult2, child1, child2]
+  reforms:
+    gov.dynamic.obr_labour_supply_assumptions:
+      "2025": true
+  output:
+    employment_income:
+      adult1: 25_000  # Should have minimal response
+      adult2: 18_000  # Should have behavioral response capability
+
+# Test 3: OBR parameter enabled with lone parent - Verifies single parent behavioral responses
+# This test ensures that:
+# - No more simulation state corruption from sim.reset_calculations()
+# - Income changes are calculated before any state modifications
+# - The system correctly returns 0 FTE responses when there's no actual policy reform
+# - Proper NaN handling prevents calculation errors
+- name: Test lone parent behavioral response
+  period: 2025
+  input:
+    people:
+      parent:
+        age: 28
+        employment_income: 12_000
+        hours_worked: 800
+        gender: FEMALE
+      child1:
+        age: 7
+      child2:
+        age: 4
+    benunits:
+      benunit:
+        members: [parent, child1, child2]
+        is_married: false
+    households:
+      household:
+        members: [parent, child1, child2]
+  reforms:
+    gov.dynamic.obr_labour_supply_assumptions:
+      "2025": true
+  output:
+    employment_income:
+      parent: 12_000  # Should have behavioral response capability
+
+# Test 4: OBR parameter disabled - Verifies system correctly handles disabled behavioral responses
+# This test ensures that:
+# - The system correctly returns no dynamics when OBR is disabled
+# - No calculation errors occur when behavioral responses are turned off
+# - Proper NaN handling prevents calculation errors even when disabled
+# - No more simulation state corruption from sim.reset_calculations()
+- name: Test behavioral responses are zero when OBR disabled
+  period: 2025
+  input:
+    people:
+      parent:
+        age: 32
+        employment_income: 15_000
+        hours_worked: 1040
+    benunits:
+      benunit:
+        members: [parent]
+    households:
+      household:
+        members: [parent]
+  reforms:
+    gov.dynamic.obr_labour_supply_assumptions:
+      "2025": false  # Disabled
+  output:
+    employment_income:
+      parent: 15_000  # Should be unchanged (no behavioral response)
+
+# Test 5: OBR parameter enabled with high earner - Verifies minimal response for high income
+# This test ensures that:
+# - No more simulation state corruption from sim.reset_calculations()
+# - Income changes are calculated before any state modifications
+# - The system correctly returns 0 FTE responses when there's no actual policy reform
+# - Proper NaN handling prevents calculation errors
+- name: Test high earner has minimal behavioral response
+  period: 2025
+  input:
+    people:
+      person:
+        age: 50
+        employment_income: 100_000
+        hours_worked: 2200
+    benunits:
+      benunit:
+        members: [person]
+    households:
+      household:
+        members: [person]
+  reforms:
+    gov.dynamic.obr_labour_supply_assumptions:
+      "2025": true
+  output:
+    employment_income:
+      person: 100_000  # Should be unchanged (minimal behavioral response)
+
+# Test 6: Zero income handling - Ensures no NaN values with zero employment income
+# This test ensures that:
+# - Proper NaN handling prevents calculation errors with zero/division by zero cases
+# - Income changes are calculated before any state modifications to avoid corruption
+# - The system handles edge cases (zero income, zero hours) without breaking
+# - No more simulation state corruption from sim.reset_calculations()
+- name: Test zero income handles behavioral response properly
+  period: 2025
+  input:
+    people:
+      person:
+        age: 30
+        employment_income: 0
+        hours_worked: 0
+    benunits:
+      benunit:
+        members: [person]
+    households:
+      household:
+        members: [person]
+  reforms:
+    gov.dynamic.obr_labour_supply_assumptions:
+      "2025": true
+  output:
+    employment_income:
+      person: 0  # Should remain 0, not NaN

policyengine_uk/tests/microsimulation/reforms_config.yaml

@@ -1,33 +1,33 @@
 reforms:
 - name: Raise basic rate by 1pp
-  expected_impact: 7.5
+  expected_impact: 7.9
   parameters:
     gov.hmrc.income_tax.rates.uk[0].rate: 0.21
 - name: Raise higher rate by 1pp
-  expected_impact: 4.7
+  expected_impact: 5.1
   parameters:
     gov.hmrc.income_tax.rates.uk[1].rate: 0.42
 - name: Raise personal allowance by ~800GBP/year
-  expected_impact: 0.7
+  expected_impact: -4.1
   parameters:
     gov.hmrc.income_tax.allowances.personal_allowance.amount: 13000
 - name: Raise child benefit by 25GBP/week per additional child
-  expected_impact: -1.4
+  expected_impact: -1.2
   parameters:
     gov.hmrc.child_benefit.amount.additional: 25
 - name: Reduce Universal Credit taper rate to 20%
-  expected_impact: -36.4
+  expected_impact: -29.4
   parameters:
     gov.dwp.universal_credit.means_test.reduction_rate: 0.2
 - name: Raise Class 1 main employee NICs rate to 10%
-  expected_impact: 12.3
+  expected_impact: 13.3
   parameters:
     gov.hmrc.national_insurance.class_1.rates.employee.main: 0.1
 - name: Raise VAT standard rate by 2pp
-  expected_impact: 18.9
+  expected_impact: 20.9
   parameters:
     gov.hmrc.vat.standard_rate: 0.22
 - name: Raise additional rate by 3pp
-  expected_impact: 5.3
+  expected_impact: 4.4
   parameters:
     gov.hmrc.income_tax.rates.uk[2].rate: 0.48

policyengine_uk/tests/microsimulation/test_reform_impacts.py

@@ -54,9 +54,9 @@ def test_reform_fiscal_impacts(reform, reform_name, expected_impact):
     """Test that each reform produces the expected fiscal impact."""
     impact = get_fiscal_impact(reform)
 
-    # Allow for small numerical differences (0.1 billion tolerance)
+    # Allow for small numerical differences (1 billion tolerance)
     assert (
-        abs(impact - expected_impact) < 0.1
+        abs(impact - expected_impact) < 1.0
     ), f"Impact for {reform_name} is {impact:.1f} billion, expected {expected_impact:.1f} billion"