policyengine-core 3.26.2__tar.gz → 3.26.4__tar.gz

{policyengine_core-3.26.2 → policyengine_core-3.26.4}/CHANGELOG.md

@@ -1,3 +1,18 @@
+## [3.26.4] - 2026-05-14
+
+### Fixed
+
+- Use safe YAML loaders and remove dynamic eval from parameter breakdown handling.
+
+
+## [3.26.3] - 2026-05-14
+
+### Fixed
+
+- Preserve values passed through dispatched `set_input` handlers across cache invalidation.
+- Apply chained parameter uprating dependencies in deterministic dependency order.
+
+
 ## [3.26.2] - 2026-05-14
 
 ### Fixed

{policyengine_core-3.26.2 → policyengine_core-3.26.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: policyengine-core
-Version: 3.26.2
+Version: 3.26.4
 Summary: Core microsimulation engine enabling country-specific policy models.
 Project-URL: Homepage, https://github.com/policyengine/policyengine-core
 Author-email: PolicyEngine <hello@policyengine.org>

{policyengine_core-3.26.2 → policyengine_core-3.26.4}/policyengine_core/data_structures/parameter_node_metadata.py

@@ -37,8 +37,8 @@ class ParameterNodeMetadata:
         metadata:
             breakdown:
                 - region # If `region` is an Enum-type variable with possible values `[ENGLAND, WALES]` then these children will be added.
-                - range(1, 7) # This code is `eval`uated to produce the list `[1, 2, 3, 4, 5, 6]` which are then added as grandchildren.
-                - [True, False] # This list is added as great-grandchildren (using the same `eval` method as above).
+                - range(1, 7) # Safe dynamic form producing `[1, 2, 3, 4, 5, 6]`, which are then added as grandchildren.
+                - [True, False] # Literal collection form added as great-grandchildren.
     
     """
 

{policyengine_core-3.26.2 → policyengine_core-3.26.4}/policyengine_core/holders/helpers.py

@@ -10,6 +10,18 @@ from policyengine_core.periods import Period
 log = logging.getLogger(__name__)
 
 
+def get_input_branch(holder: Holder) -> str:
+    simulation = getattr(holder, "simulation", None)
+    user_input_contexts = getattr(simulation, "_user_input_contexts", None)
+    if user_input_contexts:
+        return user_input_contexts[-1]
+    return "default"
+
+
+def get_stored_array(holder: Holder, period: Period, branch_name: str) -> ArrayLike:
+    return holder._get_array_from_storage(period, branch_name)
+
+
 def set_input_dispatch_by_period(holder: Holder, period: Period, array: ArrayLike):
     """
     This function can be declared as a ``set_input`` attribute of a variable.
@@ -35,11 +47,12 @@ def set_input_dispatch_by_period(holder: Holder, period: Period, array: ArrayLik
     after_instant = period.start.offset(period_size, period_unit)
 
     # Cache the input data, skipping the existing cached months
+    branch_name = get_input_branch(holder)
     sub_period = period.start.period(cached_period_unit)
     while sub_period.start < after_instant:
-        existing_array = holder.get_array(sub_period)
+        existing_array = get_stored_array(holder, sub_period, branch_name)
         if existing_array is None:
-            holder._set(sub_period, array)
+            holder._set(sub_period, array, branch_name)
         else:
             # The array of the current sub-period is reused for the next ones.
             # TODO: refactor or document this behavior
@@ -72,11 +85,12 @@ def set_input_divide_by_period(holder: Holder, period: Period, array: ArrayLike)
     after_instant = period.start.offset(period_size, period_unit)
 
     # Count the number of elementary periods to change, and the difference with what is already known.
+    branch_name = get_input_branch(holder)
     remaining_array = array.copy()
     sub_period = period.start.period(cached_period_unit)
     sub_periods_count = 0
     while sub_period.start < after_instant:
-        existing_array = holder.get_array(sub_period)
+        existing_array = get_stored_array(holder, sub_period, branch_name)
         if existing_array is not None:
             remaining_array -= existing_array
         else:
@@ -88,8 +102,8 @@ def set_input_divide_by_period(holder: Holder, period: Period, array: ArrayLike)
         divided_array = remaining_array / sub_periods_count
         sub_period = period.start.period(cached_period_unit)
         while sub_period.start < after_instant:
-            if holder.get_array(sub_period) is None:
-                holder._set(sub_period, divided_array)
+            if get_stored_array(holder, sub_period, branch_name) is None:
+                holder._set(sub_period, divided_array, branch_name)
             sub_period = sub_period.offset(1)
     elif not (remaining_array == 0).all():
         raise ValueError(

{policyengine_core-3.26.2 → policyengine_core-3.26.4}/policyengine_core/holders/holder.py

@@ -94,6 +94,14 @@ class Holder:
         if self._disk_storage:
             self._disk_storage.delete(period, branch_name)
 
+    def _get_array_from_storage(
+        self, period: Period, branch_name: str = "default"
+    ) -> ArrayLike:
+        value = self._memory_storage.get(period, branch_name)
+        if value is None and self._disk_storage:
+            value = self._disk_storage.get(period, branch_name)
+        return value
+
     def get_array(self, period: Period, branch_name: str = "default") -> ArrayLike:
         """
         Get the value of the variable for the given period.
@@ -102,7 +110,9 @@ class Holder:
         """
         if self.variable.is_neutralized:
             return self.default_array()
-        value = self._memory_storage.get(period, branch_name)
+        value = self._get_array_from_storage(period, branch_name)
+        if value is not None:
+            return value
         if value is None and branch_name != "default":
             # Walk up ``simulation.parent_branch`` so nested branches inherit
             # values from their parent (e.g. a ``no_salt`` branch cloned
@@ -121,17 +131,16 @@ class Holder:
                 else None
             )
             while parent is not None:
-                ancestor_value = self._memory_storage.get(period, parent.branch_name)
+                ancestor_value = self._get_array_from_storage(
+                    period,
+                    parent.branch_name,
+                )
                 if ancestor_value is not None:
                     return ancestor_value
                 parent = getattr(parent, "parent_branch", None)
-            default_value = self._memory_storage.get(period, "default")
+            default_value = self._get_array_from_storage(period, "default")
             if default_value is not None:
                 return default_value
-        if value is not None:
-            return value
-        if self._disk_storage:
-            return self._disk_storage.get(period, branch_name)
 
     def get_memory_usage(self) -> dict:
         """
@@ -241,21 +250,23 @@ class Holder:
             return warnings.warn(warning_message, Warning)
         if self.variable.value_type in (float, int) and isinstance(array, str):
             array = tools.eval_expression(array)
-        # Track user-provided inputs on the simulation so
-        # ``Simulation._invalidate_all_caches`` can preserve them across
-        # ``apply_reform``. ``Simulation.set_input`` also records this, but
-        # ``SimulationBuilder.finalize_variables_init`` (the situation-dict
-        # path) and country-package dataset loaders call
-        # ``holder.set_input`` directly, bypassing the simulation-level hook.
-        # Recording here covers both paths.
         simulation = getattr(self, "simulation", None)
         if simulation is not None:
             if not hasattr(simulation, "_user_input_keys"):
                 simulation._user_input_keys = set()
-            simulation._user_input_keys.add((self.variable.name, branch_name, period))
-        if self.variable.set_input and period.unit != self.variable.definition_period:
-            return self.variable.set_input(self, period, array)
-        return self._set(period, array, branch_name)
+            if not hasattr(simulation, "_user_input_contexts"):
+                simulation._user_input_contexts = []
+            simulation._user_input_contexts.append(branch_name)
+        try:
+            if (
+                self.variable.set_input
+                and period.unit != self.variable.definition_period
+            ):
+                return self.variable.set_input(self, period, array)
+            return self._set(period, array, branch_name)
+        finally:
+            if simulation is not None:
+                simulation._user_input_contexts.pop()
 
     def _to_array(self, value: Any) -> ArrayLike:
         if not isinstance(value, numpy.ndarray):
@@ -295,6 +306,10 @@ class Holder:
     def _set(
         self, period: Period, value: ArrayLike, branch_name: str = "default"
     ) -> None:
+        simulation = getattr(self, "simulation", None)
+        user_input_contexts = getattr(simulation, "_user_input_contexts", None)
+        if user_input_contexts and branch_name == "default":
+            branch_name = user_input_contexts[-1]
         value = self._to_array(value)
         if self.variable.definition_period != periods.ETERNITY:
             if period is None:
@@ -313,6 +328,10 @@ class Holder:
             self._disk_storage.put(value, period, branch_name)
         else:
             self._memory_storage.put(value, period, branch_name)
+        if user_input_contexts:
+            if not hasattr(simulation, "_user_input_keys"):
+                simulation._user_input_keys = set()
+            simulation._user_input_keys.add((self.variable.name, branch_name, period))
 
     def put_in_cache(
         self, value: ArrayLike, period: Period, branch_name: str = "default"

{policyengine_core-3.26.2 → policyengine_core-3.26.4}/policyengine_core/parameters/config.py

@@ -7,7 +7,7 @@ import yaml
 from policyengine_core.warnings import LibYAMLWarning
 
 try:
-    from yaml import CLoader as Loader
+    from yaml import CSafeLoader as Loader
 except ImportError:
     message = [
         "libyaml is not installed in your environment.",
@@ -17,7 +17,7 @@ except ImportError:
     ]
     warnings.warn(" ".join(message), LibYAMLWarning)
     from yaml import (
-        Loader,
+        SafeLoader as Loader,
     )  # type: ignore # (see https://github.com/python/mypy/issues/1153#issuecomment-455802270)
 
 ALLOWED_PARAM_TYPES = (float, int, bool, type(None), typing.List)
@@ -33,15 +33,30 @@ yaml.add_constructor("tag:yaml.org,2002:timestamp", date_constructor, Loader=Loa
 
 
 def dict_no_duplicate_constructor(loader, node, deep=False):
-    keys = [key.value for key, value in node.value]
+    explicit_keys = {}
+    for key_node, _value_node in node.value:
+        if key_node.tag == "tag:yaml.org,2002:merge":
+            continue
+        key = loader.construct_object(key_node, deep=deep)
+        try:
+            if key in explicit_keys:
+                raise yaml.parser.ParserError(
+                    "", node.start_mark, f"Found duplicate key '{key}'"
+                )
+        except TypeError as exc:
+            raise yaml.constructor.ConstructorError(
+                "", node.start_mark, f"Found unhashable key '{key}'"
+            ) from exc
+        explicit_keys[key] = True
 
-    if len(keys) != len(set(keys)):
-        duplicate = next((key for key in keys if keys.count(key) > 1))
-        raise yaml.parser.ParserError(
-            "", node.start_mark, f"Found duplicate key '{duplicate}'"
-        )
+    loader.flatten_mapping(node)
+    pairs = loader.construct_pairs(node, deep=deep)
+    mapping = {}
 
-    return loader.construct_mapping(node, deep)
+    for key, value in pairs:
+        mapping[key] = value
+
+    return mapping
 
 
 yaml.add_constructor(

{policyengine_core-3.26.2 → policyengine_core-3.26.4}/policyengine_core/parameters/operations/homogenize_parameters.py

@@ -1,3 +1,4 @@
+import ast
 import logging
 from typing import Any, Dict, List, Type
 
@@ -6,6 +7,8 @@ from policyengine_core.parameters.parameter import Parameter
 from policyengine_core.parameters.parameter_node import ParameterNode
 from policyengine_core.variables import Variable
 
+MAX_DYNAMIC_BREAKDOWN_VALUES = 10_000
+
 
 def homogenize_parameter_structures(
     root: ParameterNode, variables: Dict[str, Variable], default_value: Any = 0
@@ -43,6 +46,11 @@ def get_breakdown_variables(node: ParameterNode) -> List[str]:
                 f"Invalid breakdown metadata for parameter {node.name}: {type(breakdown)}"
             )
             return None
+        if len(breakdown) == 0:
+            logging.warning(
+                f"Invalid breakdown metadata for parameter {node.name}: empty list"
+            )
+            return None
         return breakdown
     else:
         return None
@@ -71,8 +79,7 @@ def homogenize_parameter_node(
         elif dtype == bool:
             possible_values = [True, False]
     else:
-        # Try to execute the breakdown as Python code
-        possible_values = list(eval(first_breakdown))
+        possible_values = evaluate_dynamic_breakdown(first_breakdown)
     if not hasattr(node, "children"):
         node = ParameterNode(
             node.name,
@@ -119,3 +126,84 @@ def homogenize_parameter_node(
                 node.children[child], breakdown[1:], variables, default_value
             )
     return node
+
+
+def evaluate_dynamic_breakdown(expression: str) -> List[Any]:
+    """Safely evaluate a dynamic breakdown expression.
+
+    The parameter metadata only needs literal collections and the documented
+    ``range(...)`` / ``list(range(...))`` forms. Anything else is rejected.
+    """
+
+    parsed = ast.parse(expression, mode="eval")
+    evaluated = evaluate_dynamic_breakdown_node(parsed.body)
+    if isinstance(evaluated, range):
+        validate_dynamic_breakdown_range_cardinality(evaluated, expression)
+        return list(evaluated)
+    if isinstance(evaluated, (list, tuple)):
+        validate_dynamic_breakdown_cardinality(len(evaluated), expression)
+        return list(evaluated)
+    if isinstance(evaluated, set):
+        validate_dynamic_breakdown_cardinality(len(evaluated), expression)
+        return list(evaluated)
+    raise ValueError(
+        f"Invalid dynamic breakdown expression '{expression}'. "
+        "Only literal collections and range() calls are allowed."
+    )
+
+
+def validate_dynamic_breakdown_cardinality(count: int, expression: str) -> None:
+    if count > MAX_DYNAMIC_BREAKDOWN_VALUES:
+        raise ValueError(
+            f"Dynamic breakdown expression '{expression}' produces {count} values, "
+            f"which exceeds the maximum of {MAX_DYNAMIC_BREAKDOWN_VALUES}."
+        )
+
+
+def validate_dynamic_breakdown_range_cardinality(
+    values: range, expression: str
+) -> None:
+    try:
+        count = len(values)
+    except OverflowError as exc:
+        raise ValueError(
+            f"Dynamic breakdown expression '{expression}' produces too many values."
+        ) from exc
+    validate_dynamic_breakdown_cardinality(count, expression)
+
+
+def evaluate_dynamic_breakdown_node(node: ast.AST) -> Any:
+    if isinstance(node, ast.Constant):
+        return node.value
+    if isinstance(node, ast.List):
+        validate_dynamic_breakdown_cardinality(len(node.elts), ast.unparse(node))
+        return [evaluate_dynamic_breakdown_node(element) for element in node.elts]
+    if isinstance(node, ast.Tuple):
+        validate_dynamic_breakdown_cardinality(len(node.elts), ast.unparse(node))
+        return tuple(evaluate_dynamic_breakdown_node(element) for element in node.elts)
+    if isinstance(node, ast.Set):
+        validate_dynamic_breakdown_cardinality(len(node.elts), ast.unparse(node))
+        return {evaluate_dynamic_breakdown_node(element) for element in node.elts}
+    if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)):
+        operand = evaluate_dynamic_breakdown_node(node.operand)
+        return operand if isinstance(node.op, ast.UAdd) else -operand
+    if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
+        if node.func.id == "range":
+            args = [evaluate_dynamic_breakdown_node(arg) for arg in node.args]
+            if node.keywords:
+                raise ValueError("range() keyword arguments are not allowed")
+            result = range(*args)
+            validate_dynamic_breakdown_range_cardinality(result, ast.unparse(node))
+            return result
+        if node.func.id == "list":
+            if len(node.args) != 1 or node.keywords:
+                raise ValueError("list() must contain a single positional argument")
+            evaluated = evaluate_dynamic_breakdown_node(node.args[0])
+            if isinstance(evaluated, (range, list, tuple, set)):
+                return evaluated
+            raise ValueError(
+                "list() only supports range() and literal collection expressions"
+            )
+    raise ValueError(
+        f"Unsupported dynamic breakdown expression: {ast.unparse(node) if hasattr(ast, 'unparse') else type(node).__name__}"
+    )