policyengine-uk 2.39.3__tar.gz → 2.40.0__tar.gz

{policyengine_uk-2.39.3 → policyengine_uk-2.40.0}/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.40.0] - 2025-07-21 13:23:31
+
+### Added
+
+- UKMultiYearDataset class to handle multiple fiscal years.
+- Uprating of datasets using the `uprate` method.
+
 ## [2.39.3] - 2025-07-17 12:45:26
 
 ### Fixed
@@ -1967,6 +1974,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 
+[2.40.0]: https://github.com/PolicyEngine/openfisca-uk/compare/2.39.3...2.40.0
 [2.39.3]: https://github.com/PolicyEngine/openfisca-uk/compare/2.39.2...2.39.3
 [2.39.2]: https://github.com/PolicyEngine/openfisca-uk/compare/2.39.1...2.39.2
 [2.39.1]: https://github.com/PolicyEngine/openfisca-uk/compare/2.39.0...2.39.1

{policyengine_uk-2.39.3 → policyengine_uk-2.40.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: policyengine-uk
-Version: 2.39.3
+Version: 2.40.0
 Summary: PolicyEngine tax and benefit system for the UK
 Project-URL: Homepage, https://github.com/PolicyEngine/policyengine-uk
 Project-URL: Repository, https://github.com/PolicyEngine/policyengine-uk
@@ -15,7 +15,7 @@ Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: >=3.10
-Requires-Dist: microdf-python
+Requires-Dist: microdf-python==0.4.4
 Requires-Dist: policyengine-core>=3.6.4
 Provides-Extra: dev
 Requires-Dist: black; extra == 'dev'

{policyengine_uk-2.39.3 → policyengine_uk-2.40.0}/policyengine_uk/__init__.py

@@ -13,9 +13,5 @@ from policyengine_uk.system import (
 from pathlib import Path
 import os
 from policyengine_core.taxbenefitsystems import TaxBenefitSystem
-from policyengine_uk.data.economic_assumptions import (
-    BASELINE_GROWFACTORS,
-    apply_growth_factors,
-)
 
 REPO = Path(__file__).parent

policyengine_uk-2.40.0/policyengine_uk/data/__init__.py

@@ -0,0 +1,4 @@
+from policyengine_uk.data.dataset_schema import (
+    UKMultiYearDataset,
+    UKSingleYearDataset,
+)

policyengine_uk-2.40.0/policyengine_uk/data/dataset_schema.py

@@ -0,0 +1,231 @@
+import pandas as pd
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from policyengine_uk import Microsimulation
+
+from pathlib import Path
+import h5py
+
+
+class UKSingleYearDataset:
+    person: pd.DataFrame
+    benunit: pd.DataFrame
+    household: pd.DataFrame
+
+    @staticmethod
+    def validate_file_path(file_path: str):
+        if not file_path.endswith(".h5"):
+            raise ValueError("File path must end with '.h5' for UKDataset.")
+        if not Path(file_path).exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        # Check if the file contains time_period, person, benunit, and household datasets
+        with h5py.File(file_path, "r") as f:
+            required_datasets = [
+                "time_period",
+                "person",
+                "benunit",
+                "household",
+            ]
+            for dataset in required_datasets:
+                if dataset not in f:
+                    raise ValueError(
+                        f"Dataset '{dataset}' not found in the file: {file_path}"
+                    )
+
+    def __init__(
+        self,
+        file_path: str = None,
+        person: pd.DataFrame = None,
+        benunit: pd.DataFrame = None,
+        household: pd.DataFrame = None,
+        fiscal_year: int = 2025,
+    ):
+        if file_path is not None:
+            self.validate_file_path(file_path)
+            with pd.HDFStore(file_path) as f:
+                self.person = f["person"]
+                self.benunit = f["benunit"]
+                self.household = f["household"]
+                self.time_period = str(f["time_period"].iloc[0])
+        else:
+            if person is None or benunit is None or household is None:
+                raise ValueError(
+                    "Must provide either a file path or all three DataFrames (person, benunit, household)."
+                )
+            self.person = person
+            self.benunit = benunit
+            self.household = household
+            self.time_period = str(fiscal_year)
+
+        self.data_format = "arrays"
+        self.tables = (self.person, self.benunit, self.household)
+        self.table_names = ("person", "benunit", "household")
+
+    def save(self, file_path: str):
+        with pd.HDFStore(file_path) as f:
+            f.put("person", self.person, format="table", data_columns=True)
+            f.put("benunit", self.benunit, format="table", data_columns=True)
+            f.put(
+                "household", self.household, format="table", data_columns=True
+            )
+            f.put("time_period", pd.Series([self.time_period]), format="table")
+
+    def load(self):
+        data = {}
+        for df in (self.person, self.benunit, self.household):
+            for col in df.columns:
+                data[col] = df[col].values
+
+        return data
+
+    def copy(self):
+        return UKSingleYearDataset(
+            person=self.person.copy(),
+            benunit=self.benunit.copy(),
+            household=self.household.copy(),
+            fiscal_year=self.time_period,
+        )
+
+    def validate(self):
+        # Check for NaNs in the tables
+        for df in self.tables:
+            for col in df.columns:
+                if df[col].isna().any():
+                    raise ValueError(f"Column '{col}' contains NaN values.")
+
+    @staticmethod
+    def from_simulation(
+        simulation: "Microsimulation", fiscal_year: int = 2025
+    ):
+        entity_dfs = {}
+
+        for entity in ["person", "benunit", "household"]:
+            input_variables = [
+                variable
+                for variable in simulation.input_variables
+                if simulation.tax_benefit_system.variables[variable].entity.key
+                == entity
+            ]
+            entity_dfs[entity] = simulation.calculate_dataframe(
+                input_variables, period=fiscal_year
+            )
+
+        return UKSingleYearDataset(
+            person=entity_dfs["person"],
+            benunit=entity_dfs["benunit"],
+            household=entity_dfs["household"],
+            fiscal_year=fiscal_year,
+        )
+
+
+class UKMultiYearDataset:
+    def __init__(
+        self,
+        file_path: str = None,
+        datasets: list[UKSingleYearDataset] | None = None,
+    ):
+        if datasets is not None:
+            self.datasets = {}
+            for dataset in datasets:
+                if not isinstance(dataset, UKSingleYearDataset):
+                    raise TypeError(
+                        "All items in datasets must be of type UKSingleYearDataset."
+                    )
+                year = int(dataset.time_period[:4])
+                self.datasets[year] = dataset
+
+        if file_path is not None:
+            UKSingleYearDataset.validate_file_path(file_path)
+            with pd.HDFStore(file_path) as f:
+                self.datasets = {}
+                for year in f.keys():
+                    if year.startswith("/person/"):
+                        fiscal_year = int(year.split("/")[2])
+                        person_df = f[year]
+                        benunit_df = f[f"/benunit/{fiscal_year}"]
+                        household_df = f[f"/household/{fiscal_year}"]
+                        self.datasets[fiscal_year] = UKSingleYearDataset(
+                            person=person_df,
+                            benunit=benunit_df,
+                            household=household_df,
+                            fiscal_year=fiscal_year,
+                        )
+
+        self.data_format = "time_period_arrays"
+        self.time_period = list(sorted(self.datasets.keys()))[0]
+
+    def get_year(self, fiscal_year: int) -> UKSingleYearDataset:
+        if fiscal_year in self.datasets:
+            return self.datasets[fiscal_year]
+        else:
+            raise ValueError(f"No dataset found for year {fiscal_year}.")
+
+    def __getitem__(self, fiscal_year: int):
+        return self.get_year(fiscal_year)
+
+    def save(self, file_path: str):
+        Path(file_path).unlink(
+            missing_ok=True
+        )  # Remove existing file if it exists
+        with pd.HDFStore(file_path) as f:
+            for year, dataset in self.datasets.items():
+                f.put(
+                    f"person/{year}",
+                    dataset.person,
+                    format="table",
+                    data_columns=True,
+                )
+                f.put(
+                    f"benunit/{year}",
+                    dataset.benunit,
+                    format="table",
+                    data_columns=True,
+                )
+                f.put(
+                    f"household/{year}",
+                    dataset.household,
+                    format="table",
+                    data_columns=True,
+                )
+                f.put(
+                    f"time_period/{year}",
+                    pd.Series([year]),
+                    format="table",
+                    data_columns=True,
+                )
+
+    def copy(self):
+        new_datasets = {
+            year: dataset.copy() for year, dataset in self.datasets.items()
+        }
+        return UKMultiYearDataset(datasets=list(new_datasets.values()))
+
+    @staticmethod
+    def validate_file_path(file_path: str):
+        if not file_path.endswith(".h5"):
+            raise ValueError(
+                "File path must end with '.h5' for UKMultiYearDataset."
+            )
+        if not Path(file_path).exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        # Check if the file contains datasets for multiple years
+        with h5py.File(file_path, "r") as f:
+            if not any(key.startswith("/person/") for key in f.keys()):
+                raise ValueError("No person dataset found in the file.")
+            if not any(key.startswith("/benunit/") for key in f.keys()):
+                raise ValueError("No benunit dataset found in the file.")
+            if not any(key.startswith("/household/") for key in f.keys()):
+                raise ValueError("No household dataset found in the file.")
+
+    def load(self):
+        data = {}
+        for year, dataset in self.datasets.items():
+            for df in (dataset.person, dataset.benunit, dataset.household):
+                for col in df.columns:
+                    if col not in data:
+                        data[col] = {}
+                    data[col][year] = df[col].values
+        return data

policyengine_uk-2.40.0/policyengine_uk/data/economic_assumptions.py

@@ -0,0 +1,184 @@
+from policyengine_uk.data import UKMultiYearDataset, UKSingleYearDataset
+from policyengine_uk.system import system
+import yaml
+from policyengine_core.parameters import ParameterNode
+from pathlib import Path
+import numpy as np
+
+
+def apply_uprating(
+    dataset: UKMultiYearDataset,
+):
+    # Apply uprating to the dataset.
+    dataset = dataset.copy()
+
+    if not isinstance(dataset, UKMultiYearDataset):
+        raise TypeError("dataset must be of type UKMultiYearDataset.")
+
+    for year in dataset.datasets.keys():
+        if year == min(dataset.datasets.keys()):
+            continue  # Don't uprate the first year
+        current_year = dataset.datasets[year]
+        prev_year = dataset.datasets[year - 1]
+        apply_single_year_uprating(current_year, prev_year, system.parameters)
+
+    return dataset
+
+
+def apply_single_year_uprating(
+    current_year: UKSingleYearDataset,
+    previous_year: UKSingleYearDataset,
+    parameters: ParameterNode,
+):
+    # Apply uprating to a single year dataset.
+
+    # First, apply standard variable-YoY growth based uprating.
+
+    with open(Path(__file__).parent / "uprating_indices.yaml", "r") as f:
+        uprating = yaml.safe_load(f)
+    for index_name, variables in uprating.items():
+        index_rel_change = parameters.get_child(index_name)(
+            current_year.time_period
+        )
+        for variable in variables:
+            for table_name, df in zip(
+                current_year.table_names, current_year.tables
+            ):
+                if variable in df.columns:
+                    prev_year_value = getattr(previous_year, table_name)[
+                        variable
+                    ]
+                    current_year_value = prev_year_value * index_rel_change
+                    getattr(current_year, table_name)[
+                        variable
+                    ] = current_year_value
+
+    # Next, apply custom uprating.
+
+    # Council Tax is uprated by OBR forecasts/outturns by country.
+
+    current_year = uprate_council_tax(current_year, previous_year, parameters)
+
+    # Rent is uprated by OBR forecasts/outturns by region.
+
+    current_year = uprate_rent(current_year, previous_year, parameters)
+
+    current_year.validate()
+
+    return current_year
+
+
+def uprate_council_tax(
+    current_year: UKSingleYearDataset,
+    previous_year: UKSingleYearDataset,
+    parameters: ParameterNode,
+):
+    # Uprate council tax for a single year dataset.
+
+    council_tax = (
+        parameters.gov.economic_assumptions.yoy_growth.obr.council_tax
+    )
+    region = current_year.household["region"]
+    country = np.select(
+        [
+            region == "WALES",
+            region == "SCOTLAND",
+            region == "NORTHERN IRELAND",
+        ],
+        [
+            "WALES",
+            "SCOTLAND",
+            "NORTHERN IRELAND",
+        ],
+        default="ENGLAND",
+    )
+    growth_rates = np.select(
+        [
+            country == "ENGLAND",
+            country == "WALES",
+            country == "SCOTLAND",
+        ],
+        [
+            council_tax.england(current_year.time_period),
+            council_tax.wales(current_year.time_period),
+            council_tax.scotland(current_year.time_period),
+        ],
+        default=0,
+    )
+
+    current_year.household["council_tax"] = previous_year.household[
+        "council_tax"
+    ] * (1 + growth_rates)
+    return current_year
+
+
+def uprate_rent(
+    current_year: UKSingleYearDataset,
+    previous_year: UKSingleYearDataset,
+    parameters: ParameterNode,
+):
+    # Uprate rent for a single year dataset.
+    is_private_rented = (
+        current_year.household["tenure_type"] == "RENT_PRIVATELY"
+    )
+    region = current_year.household["region"]
+    prev_rent = previous_year.household["rent"]
+    growth = parameters.gov.economic_assumptions.yoy_growth
+    year = int(current_year.time_period)
+    social_rent_growth = growth.obr.social_rent(year)
+
+    if year < 2022:
+        raise ValueError(
+            "Rent uprating is not supported for years before 2022."
+        )
+    elif year < 2025:
+        # We have regional growth rates for private rent.
+        regional_growth_rate = growth.ons.private_rental_prices(year)[
+            region.values
+        ]
+        current_year.household["rent"] = np.where(
+            is_private_rented,
+            prev_rent * (1 + regional_growth_rate),
+            prev_rent * (1 + social_rent_growth),
+        )
+    elif year >= 2025:
+        # Back out private rent growth from the aggregate
+        # from latest English Housing Survey data
+        PRIVATE_RENTAL_HOUSEHOLDS = 0.188
+        SOCIAL_RENTAL_HOUSEHOLDS = 0.164
+
+        total_rental_households = (
+            PRIVATE_RENTAL_HOUSEHOLDS + SOCIAL_RENTAL_HOUSEHOLDS
+        )
+
+        private_weight = PRIVATE_RENTAL_HOUSEHOLDS / total_rental_households
+        social_weight = SOCIAL_RENTAL_HOUSEHOLDS / total_rental_households
+
+        aggregate_growth = growth.obr.rent(year)
+        private_rent_growth = (
+            aggregate_growth - social_weight * social_rent_growth
+        ) / private_weight
+        print(
+            f"Backed out private rent growth: {private_rent_growth:.1%} in {year}"
+        )
+        print(f"OBR aggregate rent growth: {aggregate_growth:.1%} in {year}")
+        print(f"Social rent growth: {social_rent_growth:.1%} in {year}")
+
+        current_year.household["rent"] = np.where(
+            is_private_rented,
+            prev_rent * (1 + private_rent_growth),
+            prev_rent * (1 + social_rent_growth),
+        )
+
+    return current_year
+
+
+def reset_uprating(
+    dataset: UKMultiYearDataset,
+):
+    # Remove all uprating from the dataset.
+
+    first_year = min(dataset.datasets.keys())
+    for year in dataset.datasets:
+        if year != first_year:
+            dataset.datasets[year] = dataset.datasets[first_year].copy()

policyengine_uk-2.40.0/policyengine_uk/data/uprating_indices.yaml

@@ -0,0 +1,87 @@
+gov.economic_assumptions.yoy_growth.obr.average_earnings:
+- employee_pension_contributions
+- employer_pension_contributions
+- employment_income
+- employment_income_before_lsr
+- personal_pension_contributions
+- student_loan_repayments
+gov.economic_assumptions.yoy_growth.obr.consumer_price_index:
+- afcs_reported
+- alcohol_and_tobacco_consumption
+- attendance_allowance_reported
+- bsp_reported
+- carers_allowance_reported
+- child_benefit_reported
+- child_tax_credit_reported
+- childcare_expenses
+- clothing_and_footwear_consumption
+- communication_consumption
+- diesel_spending
+- dla_m_reported
+- dla_sc_reported
+- domestic_energy_consumption
+- education_consumption
+- esa_contrib_reported
+- esa_income_reported
+- food_and_non_alcoholic_beverages_consumption
+- free_school_fruit_veg
+- free_school_meals
+- free_school_milk
+- health_consumption
+- household_furnishings_consumption
+- housing_benefit_reported
+- housing_water_and_electricity_consumption
+- iidb_reported
+- incapacity_benefit_reported
+- income_support_reported
+- jsa_contrib_reported
+- jsa_income_reported
+- maintenance_expenses
+- maternity_allowance_reported
+- miscellaneous_consumption
+- pension_credit_reported
+- petrol_spending
+- pip_dl_reported
+- pip_m_reported
+- recreation_consumption
+- restaurants_and_hotels_consumption
+- sda_reported
+- state_pension
+- state_pension_reported
+- statutory_maternity_pay
+- statutory_paternity_pay
+- statutory_sick_pay
+- transport_consumption
+- universal_credit_reported
+- winter_fuel_allowance_reported
+- working_tax_credit_reported
+gov.economic_assumptions.yoy_growth.obr.mortgage_interest:
+- mortgage_interest_repayment
+gov.economic_assumptions.yoy_growth.obr.per_capita.gdp:
+- capital_gains
+- capital_gains_before_response
+- corporate_wealth
+- dividend_income
+- gross_financial_wealth
+- lump_sum_income
+- main_residence_value
+- maintenance_income
+- miscellaneous_income
+- mortgage_capital_repayment
+- net_financial_wealth
+- non_residential_property_value
+- other_investment_income
+- other_residential_property_value
+- owned_land
+- pension_income
+- private_transfer_income
+- property_income
+- savings
+- savings_interest_income
+- sublet_income
+gov.economic_assumptions.yoy_growth.obr.per_capita.mixed_income:
+- self_employment_income
+gov.economic_assumptions.yoy_growth.obr.private_pension_index:
+- private_pension_income
+gov.economic_assumptions.yoy_growth.ons.population:
+- household_weight