calkit-python 0.27.1__tar.gz → 0.27.3__tar.gz

{calkit_python-0.27.1 → calkit_python-0.27.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.27.1
+Version: 0.27.3
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.27.1 → calkit_python-0.27.3}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.27.1"
+__version__ = "0.27.3"
 
 from .core import *  # noqa: F403, I001
 from . import git  # noqa: F401

{calkit_python-0.27.1 → calkit_python-0.27.3}/calkit/cli/main.py

@@ -620,9 +620,22 @@ def pull(
         list[str],
         typer.Option("--dvc-arg", help="Additional DVC args."),
     ] = [],
+    force: Annotated[
+        bool,
+        typer.Option(
+            "--force",
+            "-f",
+            help="Force pull, potentially overwriting local changes.",
+        ),
+    ] = False,
 ):
     """Pull with both Git and DVC."""
     typer.echo("Git pulling")
+    if force:
+        if "-f" not in git_args and "--force" not in git_args:
+            git_args.append("-f")
+        if "-f" not in dvc_args and "--force" not in dvc_args:
+            dvc_args.append("-f")
     try:
         subprocess.check_call(["git", "pull"] + git_args)
     except subprocess.CalledProcessError:

{calkit_python-0.27.1 → calkit_python-0.27.3}/calkit/models/iteration.py

@@ -16,10 +16,19 @@ class RangeIteration(BaseModel):
 
     @property
     def values(self) -> list[int | float]:
+        # Determine precision from inputs
+        def get_decimal_places(num):
+            return len(str(num).split(".")[-1]) if "." in str(num) else 0
+
+        max_precision = max(
+            get_decimal_places(self.range.start),
+            get_decimal_places(self.range.stop),
+            get_decimal_places(self.range.step),
+        )
         vals = []
         current = self.range.start
         while current < self.range.stop:
-            vals.append(current)
+            vals.append(round(current, max_precision))
             current += self.range.step
         return vals
 

{calkit_python-0.27.1 → calkit_python-0.27.3}/calkit/models/pipeline.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from pathlib import PurePosixPath
 from typing import Any, Literal
 
-from pydantic import BaseModel, ConfigDict, Discriminator
+from pydantic import BaseModel, ConfigDict, Discriminator, field_validator
 from typing_extensions import Annotated
 
 from calkit.models.io import InputsFromStageOutputs, PathOutput
@@ -21,18 +21,79 @@ from calkit.notebooks import (
 
 
 class StageIteration(BaseModel):
-    arg_name: str
-    values: list[int | float | str | RangeIteration | ParameterIteration]
+    """A model for the ``iterate_over`` key in a stage definition.
 
-    def expand_values(self, params: ParametersType) -> list[int | float | str]:
+    If ``arg_name`` is a list, ``values`` also must be a list of lists with
+    each sublist the length of ``arg_name``.
+    """
+
+    arg_name: str | list[str]
+    values: list[
+        int
+        | float
+        | str
+        | RangeIteration
+        | ParameterIteration
+        | list[int | float | str]
+    ]
+
+    @field_validator("values")
+    @classmethod
+    def validate_values_structure(cls, v, info):
+        """Validate that values are structured correctly based on arg_name."""
+        arg_name = info.data.get("arg_name")
+        # If arg_name is a list, check that values contains lists of the
+        # correct length
+        if isinstance(arg_name, list):
+            expected_length = len(arg_name)
+            for i, value in enumerate(v):
+                # TODO: Support RangeIteration and ParameterIteration
+                if isinstance(value, (RangeIteration, ParameterIteration)):
+                    raise ValueError(
+                        "RangeIteration and ParameterIteration are not "
+                        "allowed when arg_name is a list"
+                    )
+                # Check if the value is a list and has the correct length
+                if not isinstance(value, list):
+                    raise ValueError(
+                        f"When arg_name is a list, all values must be lists; "
+                        f"Value at index {i} is {type(value).__name__}"
+                    )
+                if len(value) != expected_length:
+                    raise ValueError(
+                        f"When arg_name has {expected_length} elements, "
+                        f"each value list must have {expected_length} "
+                        f"elements;  Value at index {i} has {len(value)} "
+                        "elements"
+                    )
+        return v
+
+    def expand_values(
+        self, params: ParametersType
+    ) -> list[int | float | str | dict[str, int | float | str]]:
         vals = []
-        for vals_i in self.values:
-            if isinstance(vals_i, ParameterIteration):
-                vals += vals_i.values_from_params(params)
-            elif isinstance(vals_i, RangeIteration):
-                vals += vals_i.values
-            else:
-                vals.append(vals_i)
+        if isinstance(self.arg_name, list):
+            # Expand into a list of dictionaries, in which case the DVC arg
+            # name must be auto-generated
+            for vals_list in self.values:
+                if not isinstance(vals_list, list):
+                    raise ValueError(
+                        "Expected a list for vals_list, got "
+                        f"{type(vals_list).__name__}"
+                    )
+                v = {}
+                for n, name in enumerate(self.arg_name):
+                    v[name] = vals_list[n]
+                vals.append(v)
+        else:
+            # arg_name is a string
+            for vals_i in self.values:
+                if isinstance(vals_i, ParameterIteration):
+                    vals += vals_i.values_from_params(params)
+                elif isinstance(vals_i, RangeIteration):
+                    vals += vals_i.values
+                else:
+                    vals.append(vals_i)
         return vals
 
 

{calkit_python-0.27.1 → calkit_python-0.27.3}/calkit/pipeline.py

@@ -28,7 +28,23 @@ def _expand_matrix(input_dict: dict[str, list]) -> list[dict]:
     list_of_dicts = []
     for combination in combinations:
         list_of_dicts.append(dict(zip(keys, combination)))
-    return list_of_dicts
+    # After expanding the matrix, flatten any nested dictionaries in the result
+    # This handles cases where list-of-lists iterations produce dictionaries as
+    # values, by concatenating parent and child keys (e.g., "parent.child")
+    # so all permutations are represented as flat dictionaries
+    final_list = []
+    for item in list_of_dicts:
+        keys = list(item.keys())
+        vals = list(item.values())
+        vd = {}
+        for key, val in zip(keys, vals):
+            if isinstance(val, dict):
+                for k, v in val.items():
+                    vd[f"{key}.{k}"] = v
+            else:
+                vd[key] = val
+        final_list.append(vd)
+    return final_list
 
 
 def to_dvc(
@@ -90,16 +106,24 @@ def to_dvc(
         # stage
         if stage.iterate_over is not None:
             # Process a list of iterations into a DVC matrix stage
+            # Initialize a DVC matrix
             dvc_matrix = {}
+            # Initialize a dict for doing string formatting on the DVC stage
             format_dict = {}
-            for iteration in stage.iterate_over:
+            for n, iteration in enumerate(stage.iterate_over):
                 arg_name = iteration.arg_name
-                dvc_matrix[arg_name] = iteration.expand_values(
+                exp_vals = iteration.expand_values(
                     params=ck_info.get("parameters", {})
                 )
-                # Now replace arg name in cmd, deps, and outs with
-                # ${item.{arg_name}}
-                format_dict[arg_name] = f"${{item.{arg_name}}}"
+                if isinstance(arg_name, list):
+                    dvc_arg_name = f"_arg{n}"
+                    for arg_name_i in arg_name:
+                        item_string = f"${{item.{dvc_arg_name}.{arg_name_i}}}"
+                        format_dict[arg_name_i] = item_string
+                else:
+                    dvc_arg_name = arg_name
+                    format_dict[arg_name] = f"${{item.{arg_name}}}"
+                dvc_matrix[dvc_arg_name] = exp_vals
             try:
                 cmd = dvc_stage["cmd"]
                 cmd = cmd.format(**format_dict)

calkit_python-0.27.3/calkit/tests/models/test_iteration.py

@@ -0,0 +1,16 @@
+"""Tests for ``calkit.models.iteration``."""
+
+from calkit.models.iteration import RangeIteration, RangeIterationParams
+
+
+def test_rangeiteration():
+    r = RangeIteration(range=RangeIterationParams(start=0, stop=10, step=2))
+    assert r.values == [0, 2, 4, 6, 8]
+    r = RangeIteration(
+        range=RangeIterationParams(start=0.05, stop=0.25, step=0.1)
+    )
+    assert r.values == [0.05, 0.15]
+    r = RangeIteration(range=RangeIterationParams(start=0.5, stop=1, step=0.2))
+    assert r.values == [0.5, 0.7, 0.9]
+    r = RangeIteration(range=RangeIterationParams(start=0, stop=1, step=0.2))
+    assert r.values == [0.0, 0.2, 0.4, 0.6, 0.8]

{calkit_python-0.27.1 → calkit_python-0.27.3}/calkit/tests/models/test_pipeline.py

@@ -1,9 +1,13 @@
 """Tests for ``calkit.models.pipeline``."""
 
+import pytest
+from pydantic import ValidationError
+
 from calkit.models.pipeline import (
     JupyterNotebookStage,
     LatexStage,
     PythonScriptStage,
+    StageIteration,
     WordToPdfStage,
 )
 
@@ -102,3 +106,16 @@ def test_jupyternotebookstage():
     assert "html" not in dvc_stage["cmd"]
     assert " -p param1=value1 " in dvc_stage["cmd"]
     assert " -p param2=value2 " in dvc_stage["cmd"]
+
+
+def test_stageiteration():
+    StageIteration(
+        arg_name="param1",
+        values=[1, 2, 3],
+    )
+    with pytest.raises(ValidationError):
+        StageIteration(arg_name=["param1", "param2"], values=[1, 2, 3])
+    i = StageIteration(arg_name=["param1", "param2"], values=[[1, 2], [3, 4]])
+    i.values
+    exp_vals = i.expand_values(params={})
+    assert exp_vals == [{"param1": 1, "param2": 2}, {"param1": 3, "param2": 4}]

{calkit_python-0.27.1 → calkit_python-0.27.3}/calkit/tests/test_pipeline.py

@@ -277,6 +277,60 @@ def test_to_dvc_notebook_stage():
     assert "something" in stage["matrix"]["param3"]
 
 
+def test_to_dvc_list_of_list_iteration():
+    # Test that we can define iteration over a list of lists
+    ck_info = {
+        "pipeline": {
+            "stages": {
+                "get-data": {
+                    "kind": "python-script",
+                    "environment": "something",
+                    "script_path": "something/my-cool-script.py",
+                    "args": ["--a1={var1}", "--a2={var2}"],
+                    "iterate_over": [
+                        {
+                            "arg_name": ["var1", "var2"],
+                            "values": [[1, "a"], [2, "b"], [3, "c"]],
+                        },
+                    ],
+                    "outputs": [
+                        "out-{var1}-{var2}.out",
+                        "out2-{var1}-{var2}.out",
+                    ],
+                },
+                "process-data": {
+                    "kind": "python-script",
+                    "script_path": "something.py",
+                    "environment": "py",
+                    "inputs": [
+                        {"from_stage_outputs": "get-data"},
+                        "something.else.txt",
+                    ],
+                },
+            }
+        }
+    }
+    dvc_stages = calkit.pipeline.to_dvc(ck_info=ck_info, write=False)
+    stage = dvc_stages["get-data"]
+    assert "--a1=${item._arg0.var1} --a2=${item._arg0.var2}" in stage["cmd"]
+    matrix = stage["matrix"]
+    assert matrix == {
+        "_arg0": [
+            {"var1": 1, "var2": "a"},
+            {"var1": 2, "var2": "b"},
+            {"var1": 3, "var2": "c"},
+        ]
+    }
+    assert stage["outs"] == [
+        "out-${item._arg0.var1}-${item._arg0.var2}.out",
+        "out2-${item._arg0.var1}-${item._arg0.var2}.out",
+    ]
+    print(dvc_stages)
+    stage2 = dvc_stages["process-data"]
+    assert "out-1-a.out" in stage2["deps"]
+    assert "out2-2-b.out" in stage2["deps"]
+
+
 def test_remove_stage(tmp_dir):
     subprocess.check_call(["calkit", "init"])
     # Test that we can remove a stage from the pipeline

{calkit_python-0.27.1 → calkit_python-0.27.3}/docs/pipeline/index.md

@@ -105,6 +105,28 @@ pipeline:
         - models/{var}.h5
 ```
 
+### Over a table (or list of lists)
+
+```yaml
+pipeline:
+  stages:
+    my-iter-stage:
+      kind: python-script
+      script_path: scripts/my-script.py
+      args:
+        - "--model={var1}"
+        - "--n_estimators={var2}"
+      iterate_over:
+        - arg_name: [var1, var2]
+          values:
+            - [some-model, 5]
+            - [some-other-model, 7]
+      inputs:
+        - data/raw
+      outputs:
+        - models/{var1}-{var2}.h5
+```
+
 ### Over ranges of numbers
 
 ```yaml