policyengine-core 3.0.0__tar.gz → 3.2.0__tar.gz

{policyengine_core-3.0.0/policyengine_core.egg-info → policyengine_core-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: policyengine-core
-Version: 3.0.0
+Version: 3.2.0
 Summary: Core microsimulation engine enabling country-specific policy models.
 Home-page: https://github.com/policyengine/policyengine-core
 Author: PolicyEngine

{policyengine_core-3.0.0 → policyengine_core-3.2.0}/policyengine_core/country_template/data/datasets/country_template_dataset.py

@@ -29,6 +29,7 @@ class CountryTemplateDataset(Dataset):
             "person_household_id": {ETERNITY: person_household_id},
             "person_household_role": {ETERNITY: person_household_role},
             "salary": {salary_time_period: salary},
+            "accommodation_size": {salary_time_period: [200, 300]},
             "household_weight": {weight_time_period: weight},
         }
         self.save_dataset(data)

{policyengine_core-3.0.0 → policyengine_core-3.2.0}/policyengine_core/data/dataset.py

@@ -30,6 +30,7 @@ class Dataset:
     TABLES = "tables"
     ARRAYS = "arrays"
     TIME_PERIOD_ARRAYS = "time_period_arrays"
+    FLAT_FILE = "flat_file"
 
     _table_cache: Dict[str, pd.DataFrame] = None
 
@@ -55,17 +56,11 @@ class Dataset:
             Dataset.TABLES,
             Dataset.ARRAYS,
             Dataset.TIME_PERIOD_ARRAYS,
+            Dataset.FLAT_FILE,
         ], f"You tried to instantiate a Dataset object, but your data_format attribute is invalid ({self.data_format})."
 
         self._table_cache = {}
 
-        if (
-            self.data_format != Dataset.TIME_PERIOD_ARRAYS
-        ) and self.time_period is None:
-            raise ValueError(
-                "You tried to instantiate a Dataset object, but no time_period has been provided."
-            )
-
         if not self.exists and require:
             if self.url is not None:
                 self.download()
@@ -107,6 +102,13 @@ class Dataset:
                     values = f[key]
                 self._table_cache[key] = values
                 return values
+        elif self.data_format == Dataset.FLAT_FILE:
+            if key is None:
+                return pd.read_csv(file)
+            else:
+                raise ValueError(
+                    "You tried to load a key from a flat file dataset, but flat file datasets do not support keys."
+                )
         else:
             raise ValueError(
                 f"Invalid data format {self.data_format} for dataset {self.label}."
@@ -130,6 +132,8 @@ class Dataset:
             with pd.HDFStore(file, "a") as f:
                 f.put(key, values)
             self._table_cache = {}
+        elif self.data_format == Dataset.FLAT_FILE:
+            values.to_csv(file, index=False)
         else:
             raise ValueError(
                 f"Invalid data format {self.data_format} for dataset {self.label}."
@@ -242,6 +246,8 @@ class Dataset:
         elif self.data_format in (Dataset.ARRAYS, Dataset.TIME_PERIOD_ARRAYS):
             with h5py.File(self.file_path, "r") as f:
                 return list(f.keys())
+        elif self.data_format == Dataset.FLAT_FILE:
+            return pd.read_csv(self.file_path, nrows=0).columns.tolist()
         else:
             raise ValueError(
                 f"Invalid data format {self.data_format} for dataset {self.label}."
@@ -332,3 +338,51 @@ class Dataset:
         """Removes the dataset from disk."""
         if self.exists:
             self.file_path.unlink()
+
+    @staticmethod
+    def from_file(file_path: str, time_period: str = None):
+        """Creates a dataset from a file.
+
+        Args:
+            file_path (str): The file path to create the dataset from.
+
+        Returns:
+            Dataset: The dataset.
+        """
+        file_path = Path(file_path)
+        dataset = type(
+            "Dataset",
+            (Dataset,),
+            {
+                "name": file_path.stem,
+                "label": file_path.stem,
+                "data_format": Dataset.FLAT_FILE,
+                "file_path": file_path,
+                "time_period": time_period,
+            },
+        )()
+
+        return dataset
+
+    @staticmethod
+    def from_dataframe(dataframe: pd.DataFrame, time_period: str = None):
+        """Creates a dataset from a DataFrame.
+
+        Returns:
+            Dataset: The dataset.
+        """
+        file_path = Path(file_path)
+        dataset = type(
+            "Dataset",
+            (Dataset,),
+            {
+                "name": file_path.stem,
+                "label": file_path.stem,
+                "data_format": Dataset.FLAT_FILE,
+                "file_path": "dataframe",
+                "time_period": time_period,
+                "load": lambda: dataframe,
+            },
+        )()
+
+        return dataset

{policyengine_core-3.0.0 → policyengine_core-3.2.0}/policyengine_core/parameters/parameter.py

@@ -137,7 +137,7 @@ class Parameter(AtInstantLike):
         ]
         return clone
 
-    def update(self, period=None, start=None, stop=None, value=None):
+    def update(self, value=None, period=None, start=None, stop=None):
         """
         Change the value for a given period.
 
@@ -156,7 +156,7 @@ class Parameter(AtInstantLike):
             start = period.start
             stop = period.stop
         if start is None:
-            raise ValueError("You must provide either a start or a period")
+            start = "0000-01-01"
         start_str = str(start)
         stop_str = str(stop.offset(1, "day")) if stop else None
 

{policyengine_core-3.0.0 → policyengine_core-3.2.0}/policyengine_core/parameters/parameter_node.py

@@ -253,3 +253,24 @@ class ParameterNode(AtInstantLike):
         self.modified = True
         if self.parent is not None:
             self.parent.mark_as_modified()
+
+    def get_child(self, path: str) -> "ParameterNode":
+        node = self
+        for name in path.split("."):
+            try:
+                if "[" not in name:
+                    node = node.children[name]
+                else:
+                    try:
+                        name, index = name.split("[")
+                        index = int(index[:-1])
+                        node = node.children[name].brackets[index]
+                    except:
+                        raise ValueError(
+                            "Invalid bracket syntax (should be e.g. tax.brackets[3].rate"
+                        )
+            except:
+                raise ValueError(
+                    f"Could not find the parameter (failed at {name})."
+                )
+        return node

{policyengine_core-3.0.0 → policyengine_core-3.2.0}/policyengine_core/reforms/reform.py

@@ -1,10 +1,13 @@
 from __future__ import annotations
 
 import copy
-from typing import Callable, Union
+from typing import Callable, Union, TYPE_CHECKING
 
 from policyengine_core.parameters import ParameterNode, Parameter
 from policyengine_core.taxbenefitsystems import TaxBenefitSystem
+
+if TYPE_CHECKING:
+    from policyengine_core.simulations import Simulation
 from policyengine_core.periods import (
     period as period_,
     instant as instant_,
@@ -60,6 +63,8 @@ class Reform(TaxBenefitSystem):
     parameter_values: dict = None
     """The parameter values of the reform. This is used to inform any calls to the PolicyEngine API."""
 
+    simulation: "Simulation" = None
+
     def __init__(self, baseline: TaxBenefitSystem):
         """
         :param baseline: Baseline TaxBenefitSystem.
@@ -134,25 +139,22 @@ class Reform(TaxBenefitSystem):
         class reform(Reform):
             def apply(self):
                 for path, period_values in parameter_values.items():
-                    for period, value in period_values.items():
-                        if "." in period:
-                            start, stop = period.split(".")
-                            self.modify_parameters(
-                                set_parameter(
-                                    path,
-                                    value,
-                                    period=None,
-                                    start=start,
-                                    stop=stop,
-                                    return_modifier=True,
+                    parameter = self.parameters.get_child(path)
+                    if not isinstance(period_values, dict):
+                        parameter.update(
+                            start="0000-01-01", value=period_values
+                        )
+                    else:
+                        for period, value in period_values.items():
+                            if "." in period:
+                                start, stop = period.split(".")
+                                parameter.update(
+                                    start=start, stop=stop, value=value
                                 )
-                            )
-                        else:
-                            self.modify_parameters(
-                                set_parameter(
-                                    path, value, period, return_modifier=True
+                            else:
+                                parameter = parameter.update(
+                                    period=period, value=value
                                 )
-                            )
 
         reform.country_id = country_id
         reform.parameter_values = parameter_values

{policyengine_core-3.0.0 → policyengine_core-3.2.0}/policyengine_core/simulations/simulation.py

@@ -5,6 +5,7 @@ import numpy
 import numpy as np
 import pandas as pd
 from numpy.typing import ArrayLike
+import logging
 
 from policyengine_core import commons, periods
 from policyengine_core.data.dataset import Dataset
@@ -30,7 +31,7 @@ if TYPE_CHECKING:
     from policyengine_core.taxbenefitsystems import TaxBenefitSystem
 
 from policyengine_core.experimental import MemoryConfig
-from policyengine_core.populations import Population
+from policyengine_core.populations import Population, GroupPopulation
 from policyengine_core.tracers import SimpleTracer
 from policyengine_core.variables import Variable, QuantityType
 from policyengine_core.reforms.reform import Reform
@@ -84,7 +85,6 @@ class Simulation:
         reform: Reform = None,
         trace: bool = False,
     ):
-        reform_applied_after = False
         if tax_benefit_system is None:
             if (
                 self.default_tax_benefit_system_instance is not None
@@ -93,20 +93,11 @@ class Simulation:
                 tax_benefit_system = self.default_tax_benefit_system_instance
             else:
                 # If reform is taken as an arg, pass it
-                try:
-                    tax_benefit_system = self.default_tax_benefit_system(
-                        reform=reform
-                    )
-                except:
-                    tax_benefit_system = self.default_tax_benefit_system()
-                    reform_applied_after = True
+                tax_benefit_system = self.default_tax_benefit_system()
             self.tax_benefit_system = tax_benefit_system
 
         self.reform = reform
         self.tax_benefit_system = tax_benefit_system
-
-        if reform_applied_after and reform is not None:
-            self.apply_reform(reform)
         self.branch_name = "default"
 
         if dataset is None:
@@ -156,17 +147,23 @@ class Simulation:
                 datasets_by_name = {
                     dataset.name: dataset for dataset in self.datasets
                 }
-                if dataset not in datasets_by_name:
-                    raise ValueError(
-                        f"Dataset {dataset} not found. Available datasets: {list(datasets_by_name.keys())}"
+                if dataset in datasets_by_name:
+                    dataset = datasets_by_name.get(dataset)
+                elif Path(dataset).exists():
+                    dataset = Dataset.from_file(
+                        dataset, self.default_input_period
                     )
-                dataset = datasets_by_name[dataset]
             if isinstance(dataset, type):
                 self.dataset: Dataset = dataset(require=True)
             else:
                 self.dataset = dataset
             self.build_from_dataset()
 
+        self.tax_benefit_system.simulation = self
+
+        if self.reform is not None:
+            self.tax_benefit_system.apply_reform_set(self.reform)
+
         # Backwards compatibility methods
         self.calc = self.calculate
         self.df = self.calculate_dataframe
@@ -247,23 +244,31 @@ class Simulation:
 
         person_entity = self.tax_benefit_system.person_entity
         entity_id_field = f"{person_entity.key}_id"
-        assert (
-            entity_id_field in data
-        ), f"Missing {entity_id_field} column in the dataset. Each person entity must have an ID array defined for ETERNITY."
-
-        get_eternity_array = lambda ds: (
-            ds[list(ds.keys())[0]]
-            if self.dataset.data_format == Dataset.TIME_PERIOD_ARRAYS
-            else ds
-        )
+        if self.dataset.data_format != Dataset.FLAT_FILE:
+            assert (
+                entity_id_field in data
+            ), f"Missing {entity_id_field} column in the dataset. Each person entity must have an ID array defined for ETERNITY."
+        elif entity_id_field not in data:
+            data[entity_id_field] = np.arange(len(data))
+        if self.dataset.data_format != Dataset.FLAT_FILE:
+            get_eternity_array = lambda ds: (
+                ds[list(ds.keys())[0]]
+                if self.dataset.data_format == Dataset.TIME_PERIOD_ARRAYS
+                else ds
+            )
+        else:
+            get_eternity_array = lambda ds: ds
         entity_ids = get_eternity_array(data[entity_id_field])
         builder.declare_person_entity(person_entity.key, entity_ids)
 
         for group_entity in self.tax_benefit_system.group_entities:
             entity_id_field = f"{group_entity.key}_id"
-            assert (
-                entity_id_field in data
-            ), f"Missing {entity_id_field} column in the dataset. Each group entity must have an ID array defined for ETERNITY."
+            if self.dataset.data_format != Dataset.FLAT_FILE:
+                assert (
+                    entity_id_field in data
+                ), f"Missing {entity_id_field} column in the dataset. Each group entity must have an ID array defined for ETERNITY."
+            elif entity_id_field not in data:
+                data[entity_id_field] = np.arange(len(data))
 
             entity_ids = get_eternity_array(data[entity_id_field])
             builder.declare_entity(group_entity.key, entity_ids)
@@ -271,9 +276,12 @@ class Simulation:
             person_membership_id_field = (
                 f"{person_entity.key}_{group_entity.key}_id"
             )
-            assert (
-                person_membership_id_field in data
-            ), f"Missing {person_membership_id_field} column in the dataset. Each group entity must have a person membership array defined for ETERNITY."
+            if self.dataset.data_format != Dataset.FLAT_FILE:
+                assert (
+                    person_membership_id_field in data
+                ), f"Missing {person_membership_id_field} column in the dataset. Each group entity must have a person membership array defined for ETERNITY."
+            elif person_membership_id_field not in data:
+                data[person_membership_id_field] = np.arange(len(data))
             person_membership_ids = get_eternity_array(
                 data[person_membership_id_field]
             )
@@ -297,20 +305,55 @@ class Simulation:
 
         self.build_from_populations(builder.populations)
 
-        for variable in data:
-            if variable in self.tax_benefit_system.variables:
-                if self.dataset.data_format == Dataset.TIME_PERIOD_ARRAYS:
-                    for time_period in data[variable]:
+        if self.dataset.data_format != Dataset.FLAT_FILE:
+            for variable in data:
+                if variable in self.tax_benefit_system.variables:
+                    if self.dataset.data_format == Dataset.TIME_PERIOD_ARRAYS:
+                        for time_period in data[variable]:
+                            self.set_input(
+                                variable,
+                                time_period,
+                                data[variable][time_period],
+                            )
+                    else:
                         self.set_input(
-                            variable, time_period, data[variable][time_period]
+                            variable, self.dataset.time_period, data[variable]
                         )
                 else:
-                    self.set_input(
-                        variable, self.dataset.time_period, data[variable]
+                    # Silently skip.
+                    pass
+        else:
+            for variable in data:
+                if "__" in variable:
+                    variable_name, time_period = variable.split("__")
+                else:
+                    variable_name = variable
+                    time_period = (
+                        self.dataset.time_period or self.default_input_period
                     )
-            else:
-                # Silently skip.
-                pass
+
+                if variable_name not in self.tax_benefit_system.variables:
+                    logging.warn(
+                        f"Variable {variable_name} not found. Skipping."
+                    )
+                    continue
+
+                variable_meta = self.tax_benefit_system.get_variable(
+                    variable_name
+                )
+                entity = variable_meta.entity
+                population = self.get_population(entity.plural)
+
+                # All data should be person level
+                if len(data[variable]) != len(population.ids):
+                    population: GroupPopulation
+                    entity_level_data = population.value_from_first_person(
+                        data[variable]
+                    )
+                else:
+                    entity_level_data = data[variable]
+
+                self.set_input(variable, time_period, entity_level_data)
 
         self.default_calculation_period = self.dataset.time_period
 

policyengine_core-3.2.0/policyengine_core/weighting/__init__.py

@@ -0,0 +1 @@
+from .microdf import MicroDataFrame, MicroSeries