policyengine-core 3.5.3__tar.gz → 3.6.2__tar.gz

{policyengine_core-3.5.3/policyengine_core.egg-info → policyengine_core-3.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: policyengine-core
-Version: 3.5.3
+Version: 3.6.2
 Summary: Core microsimulation engine enabling country-specific policy models.
 Home-page: https://github.com/policyengine/policyengine-core
 Author: PolicyEngine

{policyengine_core-3.5.3 → policyengine_core-3.6.2}/policyengine_core/data/dataset.py

@@ -187,7 +187,7 @@ class Dataset:
                             f"Could not save {variable} to {file}. The value is {value}."
                         )
         elif self.data_format == Dataset.FLAT_FILE:
-            values.to_csv(file, index=False)
+            data.to_csv(file, index=False)
 
     def load_dataset(
         self,

policyengine_core-3.6.2/policyengine_core/simulations/sim_macro_cache.py

@@ -0,0 +1,84 @@
+import shutil
+from pathlib import Path
+import h5py
+from numpy.typing import ArrayLike
+import importlib.metadata
+
+from policyengine_core.taxbenefitsystems import TaxBenefitSystem
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(
+                *args, **kwargs
+            )
+        return cls._instances[cls]
+
+
+class SimulationMacroCache(metaclass=Singleton):
+    def __init__(self, tax_benefit_system: TaxBenefitSystem):
+        self.core_version = importlib.metadata.version("policyengine-core")
+        self.country_package_metadata = (
+            tax_benefit_system.get_package_metadata()
+        )
+        self.country_version = self.country_package_metadata["version"]
+        self.cache_folder_path = None
+        self.cache_file_path = None
+
+    def set_cache_path(
+        self,
+        parent_path: [Path, str],
+        dataset_name: str,
+        variable_name: str,
+        period: str,
+        branch_name: str,
+    ):
+        storage_folder = Path(parent_path) / f"{dataset_name}_variable_cache"
+        self.cache_folder_path = storage_folder
+        storage_folder.mkdir(exist_ok=True)
+        self.cache_file_path = (
+            storage_folder / f"{variable_name}_{period}_{branch_name}.h5"
+        )
+
+    def set_cache_value(self, cache_file_path: Path, value: ArrayLike):
+        with h5py.File(cache_file_path, "w") as f:
+            f.create_dataset(
+                "metadata:core_version",
+                data=self.core_version,
+            )
+            f.create_dataset(
+                "metadata:country_version",
+                data=self.country_version,
+            )
+            f.create_dataset("values", data=value)
+
+    def get_cache_path(self):
+        return self.cache_file_path
+
+    def get_cache_value(self, cache_file_path: Path):
+        with h5py.File(cache_file_path, "r") as f:
+            # Validate both core version and country package metadata are up-to-date, otherwise flush the cache
+            if (
+                "metadata:core_version" in f
+                and "metadata:country_version" in f
+            ):
+                if (
+                    f["metadata:core_version"][()].decode("utf-8")
+                    != self.core_version
+                    or f["metadata:country_version"][()].decode("utf-8")
+                    != self.country_version
+                ):
+                    f.close()
+                    self.clear_cache(self.cache_folder_path)
+                    return None
+            else:
+                f.close()
+                self.clear_cache(self.cache_folder_path)
+                return None
+            return f["values"][()]
+
+    def clear_cache(self, cache_folder_path: Path):
+        shutil.rmtree(cache_folder_path)

{policyengine_core-3.5.3 → policyengine_core-3.6.2}/policyengine_core/simulations/simulation.py

@@ -6,6 +6,7 @@ import numpy as np
 import pandas as pd
 from numpy.typing import ArrayLike
 import logging
+from pathlib import Path
 
 from policyengine_core import commons, periods
 from policyengine_core.data.dataset import Dataset
@@ -21,9 +22,6 @@ from policyengine_core.tracers import (
     SimpleTracer,
     TracingParameterNodeAtInstant,
 )
-import h5py
-from pathlib import Path
-import shutil
 
 import json
 
@@ -36,6 +34,7 @@ from policyengine_core.tracers import SimpleTracer
 from policyengine_core.variables import Variable, QuantityType
 from policyengine_core.reforms.reform import Reform
 from policyengine_core.parameters import get_parameter
+from policyengine_core.simulations.sim_macro_cache import SimulationMacroCache
 
 
 class Simulation:
@@ -606,11 +605,30 @@ class Simulation:
         if cached_array is not None:
             return cached_array
 
-        cache_path = self._get_macro_cache(variable_name, str(period))
-        if cache_path and cache_path.exists():
-            value = self._get_macro_cache_value(cache_path)
-            if value is not None:
-                return self._get_macro_cache_value(cache_path)
+        smc = SimulationMacroCache(self.tax_benefit_system)
+
+        # Check if cache could be used, if available, check if path exists
+        is_cache_available = self.check_macro_cache(variable_name, str(period))
+        if is_cache_available:
+            smc.set_cache_path(
+                self.dataset.file_path.parent,
+                self.dataset.name,
+                variable_name,
+                str(period),
+                self.branch_name,
+            )
+            cache_path = smc.get_cache_path()
+            if cache_path.exists():
+                if (
+                    not self.macro_cache_read
+                    or self.tax_benefit_system.data_modified
+                ):
+                    value = None
+                else:
+                    value = smc.get_cache_value(cache_path)
+
+                if value is not None:
+                    return value
 
         if variable.requires_computation_after is not None:
             if variable.requires_computation_after not in [
@@ -639,8 +657,8 @@ class Simulation:
                 values = self.calculate_divide(variable_name, period)
 
         if alternate_period_handling:
-            if cache_path is not None:
-                self._set_macro_cache_value(cache_path, values)
+            if is_cache_available:
+                smc.set_cache_value(cache_path, values)
             return values
 
         self._check_period_consistency(period, variable)
@@ -738,8 +756,8 @@ class Simulation:
                 f"RecursionError while calculating {variable_name} for period {period}. The full computation stack is:\n{stack_formatted}"
             )
 
-        if cache_path is not None:
-            self._set_macro_cache_value(cache_path, array)
+        if is_cache_available:
+            smc.set_cache_value(cache_path, array)
 
         return array
 
@@ -1396,77 +1414,33 @@ class Simulation:
 
         return json.loads(json.dumps(situation, cls=NpEncoder))
 
-    def _get_macro_cache(
-        self,
-        variable_name: str,
-        period: str,
-    ):
+    def check_macro_cache(self, variable_name: str, period: str) -> bool:
         """
-        Get the cache location of a variable for a given period, if it exists.
+        Check if the variable is able to have cached value
         """
-        if not self.is_over_dataset:
-            return None
+        if (
+            hasattr(self, "dataset")
+            and self.dataset.data_format == Dataset.FLAT_FILE
+        ):
+            return False
+
+        if self.is_over_dataset:
+            return True
 
         variable = self.tax_benefit_system.get_variable(variable_name)
         parameter_deps = variable.exhaustive_parameter_dependencies
 
         if parameter_deps is None:
-            return None
+            return False
 
         for parameter in parameter_deps:
             param = get_parameter(
                 self.tax_benefit_system.parameters, parameter
             )
             if param.modified:
-                return None
-
-        storage_folder = (
-            self.dataset.file_path.parent
-            / f"{self.dataset.name}_variable_cache"
-        )
-        storage_folder.mkdir(exist_ok=True)
+                return False
 
-        cache_file_path = (
-            storage_folder / f"{variable_name}_{period}_{self.branch_name}.h5"
-        )
-
-        return cache_file_path
-
-    def clear_macro_cache(self):
-        """
-        Clear the cache of all variables.
-        """
-        storage_folder = (
-            self.dataset.file_path.parent
-            / f"{self.dataset.name}_variable_cache"
-        )
-        if storage_folder.exists():
-            shutil.rmtree(storage_folder)
-
-    def _get_macro_cache_value(
-        self,
-        cache_file_path: Path,
-    ):
-        """
-        Get the value of a variable from a cache file.
-        """
-        if not self.macro_cache_read or self.tax_benefit_system.data_modified:
-            return None
-        with h5py.File(cache_file_path, "r") as f:
-            return f["values"][()]
-
-    def _set_macro_cache_value(
-        self,
-        cache_file_path: Path,
-        value: ArrayLike,
-    ):
-        """
-        Set the value of a variable in a cache file.
-        """
-        if not self.macro_cache_write or self.tax_benefit_system.data_modified:
-            return None
-        with h5py.File(cache_file_path, "w") as f:
-            f.create_dataset("values", data=value)
+        return True
 
     def to_input_dataframe(
         self,

{policyengine_core-3.5.3 → policyengine_core-3.6.2}/policyengine_core/taxbenefitsystems/tax_benefit_system.py

@@ -598,7 +598,8 @@ class TaxBenefitSystem:
 
         fallback_metadata = {
             "name": self.__class__.__name__,
-            "version": "",
+            # For testing purposes
+            "version": "0.0.0",
             "repository_url": "",
             "location": "",
         }

{policyengine_core-3.5.3 → policyengine_core-3.6.2}/policyengine_core/types/data_types/arrays.py

@@ -3,7 +3,7 @@ from numpy.typing import NDArray as ArrayType
 
 T = TypeVar("T", bool, bytes, float, int, object, str)
 
-ArrayLike = ArrayType[Any] | Sequence[T]
+ArrayLike = Sequence[T]
 
 """:obj:`typing.Generic`: Type of any castable to :class:`numpy.ndarray`.
 

{policyengine_core-3.5.3 → policyengine_core-3.6.2/policyengine_core.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: policyengine-core
-Version: 3.5.3
+Version: 3.6.2
 Summary: Core microsimulation engine enabling country-specific policy models.
 Home-page: https://github.com/policyengine/policyengine-core
 Author: PolicyEngine

{policyengine_core-3.5.3 → policyengine_core-3.6.2}/policyengine_core.egg-info/SOURCES.txt

@@ -152,6 +152,7 @@ policyengine_core/simulations/__init__.py
 policyengine_core/simulations/helpers.py
 policyengine_core/simulations/individual_sim.py
 policyengine_core/simulations/microsimulation.py
+policyengine_core/simulations/sim_macro_cache.py
 policyengine_core/simulations/simulation.py
 policyengine_core/simulations/simulation_builder.py
 policyengine_core/taxbenefitsystems/__init__.py

{policyengine_core-3.5.3 → policyengine_core-3.6.2}/setup.py

@@ -47,7 +47,7 @@ dev_requirements = [
 
 setup(
     name="policyengine-core",
-    version="3.5.3",
+    version="3.6.2",
     author="PolicyEngine",
     author_email="hello@policyengine.org",
     classifiers=[