calkit-python 0.28.2__tar.gz → 0.28.4__tar.gz

{calkit_python-0.28.2 → calkit_python-0.28.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.28.2
+Version: 0.28.4
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.28.2 → calkit_python-0.28.4}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.28.2"
+__version__ = "0.28.4"
 
 from .core import *  # noqa: F403, I001
 from . import git  # noqa: F401

{calkit_python-0.28.2 → calkit_python-0.28.4}/calkit/cli/new.py

@@ -157,10 +157,16 @@ def new_project(
                 repo.git.commit(["-m", "Initialize DVC"])
     ck_info_fpath = os.path.join(abs_path, "calkit.yaml")
     if os.path.isfile(ck_info_fpath) and not overwrite:
-        raise_error(
+        ck_info = calkit.load_calkit_info(wdir=abs_path)
+        name = ck_info.get("name", name)
+        title = ck_info.get("title", title)
+        description = ck_info.get("description", description)
+        typer.echo(
             "Destination is already a Calkit project; "
-            "use --overwrite to continue"
+            "will use existing project info where possible"
         )
+    else:
+        ck_info = {}
     if os.path.isdir(abs_path) and os.listdir(abs_path) and repo is None:
         warn(f"{abs_path} is not empty")
     if name is None:
@@ -224,7 +230,29 @@ def new_project(
                     )
                 repo.git.add("calkit.yaml")
                 if not no_commit:
-                    repo.git.commit(["-m", "Create calkit.yaml"])
+                    repo.git.commit(
+                        ["calkit.yaml", "-m", "Create calkit.yaml"]
+                    )
+            else:
+                # Merge with existing project info in calkit.yaml
+                typer.echo("Updating existing calkit.yaml file")
+                ck_info = (
+                    dict(
+                        owner=resp["owner_account_name"],
+                        name=resp["name"],
+                        title=resp["title"],
+                        description=resp["description"],
+                        git_repo_url=resp["git_repo_url"],
+                    )
+                    | ck_info
+                )
+                with open(calkit_fpath, "w") as f:
+                    ryaml.dump(ck_info, f)
+                repo.git.add("calkit.yaml")
+                if not no_commit and repo.git.diff("--staged"):
+                    repo.git.commit(
+                        ["calkit.yaml", "-m", "Update calkit.yaml"]
+                    )
         try:
             calkit.dvc.configure_remote(wdir=abs_path)
         except Exception:

{calkit_python-0.28.2 → calkit_python-0.28.4}/calkit/cli/notebooks.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import base64
+import json
 import os
 import subprocess
 import sys
@@ -140,6 +142,27 @@ def execute_notebook(
             help="Parameter to pass to the notebook in key=value format.",
         ),
     ] = [],
+    params_json: Annotated[
+        str | None,
+        typer.Option(
+            "--params-json",
+            "-j",
+            help=(
+                "JSON string to parse as parameters to pass to the notebook."
+            ),
+        ),
+    ] = None,
+    params_base64: Annotated[
+        str | None,
+        typer.Option(
+            "--params-base64",
+            "-b",
+            help=(
+                "Base64-encoded JSON string to parse as parameters to pass to "
+                "the notebook."
+            ),
+        ),
+    ] = None,
     verbose: Annotated[
         bool, typer.Option("--verbose", "-v", help="Print verbose output.")
     ] = False,
@@ -165,6 +188,17 @@ def execute_notebook(
             raise_error(str(e))
     else:
         parsed_params = {}
+    # Parse JSON parameters
+    if params_json is not None:
+        parsed_params_json = json.loads(params_json)
+        parsed_params |= parsed_params_json
+    # Parse base64 parameters
+    if params_base64 is not None:
+        try:
+            decoded_json = base64.b64decode(params_base64).decode("utf-8")
+            parsed_params |= json.loads(decoded_json)
+        except Exception as e:
+            raise_error(f"Failed to parse base64 parameters: {e}")
     # Next, always execute the notebook and save as ipynb
     fpath_out_exec = calkit.notebooks.get_executed_notebook_path(
         notebook_path=path,
@@ -175,6 +209,11 @@ def execute_notebook(
     folder = os.path.dirname(fpath_out_exec)
     os.makedirs(folder, exist_ok=True)
     notebook_dir = os.path.dirname(path) or None
+    if verbose:
+        typer.echo(f"Executing notebook {path} with params: {parsed_params}")
+        typer.echo(f"Using kernel: {kernel_name}")
+        typer.echo(f"Running with cwd: {notebook_dir}")
+        typer.echo(f"Output will be saved to: {fpath_out_exec}")
     papermill.execute_notebook(
         input_path=path,
         output_path=fpath_out_exec,

{calkit_python-0.28.2 → calkit_python-0.28.4}/calkit/models/iteration.py

@@ -60,11 +60,15 @@ ParametersType = dict[
     int | float | str | list[int | float | str | RangeIteration],
 ]
 
+ExpandedParametersType = dict[str, int | float | str | list[int | float | str]]
+
 
 class ParameterIteration(BaseModel):
     parameter: str
 
-    def values_from_params(self, params: ParametersType) -> list:
+    def values_from_params(
+        self, params: ParametersType | ExpandedParametersType
+    ) -> list:
         """Convert parameters from calkit.yaml into a list of values."""
         if self.parameter not in params:
             raise ValueError(f"'{self.parameter}' not found in parameters")
@@ -79,3 +83,23 @@ class ParameterIteration(BaseModel):
             else:
                 vals.append(val)
         return vals
+
+
+def expand_project_parameters(
+    params: ParametersType,
+) -> ExpandedParametersType:
+    """Expand any range iterations in project parameters."""
+    expanded = {}
+    for key, value in params.items():
+        if isinstance(value, list):
+            expanded_list = []
+            for item in value:
+                try:
+                    range_iter = RangeIteration.model_validate(item)
+                    expanded_list.extend(range_iter.values)
+                except Exception:
+                    expanded_list.append(item)
+            expanded[key] = expanded_list
+        else:
+            expanded[key] = value
+    return expanded

{calkit_python-0.28.2 → calkit_python-0.28.4}/calkit/models/pipeline.py

@@ -2,14 +2,23 @@
 
 from __future__ import annotations
 
+import base64
+import json
 from pathlib import PurePosixPath
 from typing import Any, Literal
 
-from pydantic import BaseModel, ConfigDict, Discriminator, field_validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Discriminator,
+    ValidationError,
+    field_validator,
+)
 from typing_extensions import Annotated
 
 from calkit.models.io import InputsFromStageOutputs, PathOutput
 from calkit.models.iteration import (
+    ExpandedParametersType,
     ParameterIteration,
     ParametersType,
     RangeIteration,
@@ -69,7 +78,7 @@ class StageIteration(BaseModel):
         return v
 
     def expand_values(
-        self, params: ParametersType
+        self, params: ParametersType | ExpandedParametersType
     ) -> list[int | float | str | dict[str, int | float | str]]:
         vals = []
         if isinstance(self.arg_name, list):
@@ -365,6 +374,34 @@ class JupyterNotebookStage(Stage):
     html_storage: Literal["git", "dvc"] | None = "dvc"
     parameters: dict[str, Any] = {}
 
+    def update_parameters(self, params: dict) -> None:
+        """If we have any templated parameters, update those, e.g., from
+        project-level parameters.
+
+        This needs to happen before writing a DVC stage, so we can properly
+        create JSON for the notebook.
+        """
+        updated_params = {}
+        for k, v in self.parameters.items():
+            # If we have something like {var_name} in v, replace it with the
+            # value from params
+            if isinstance(v, str) and v.startswith("{") and v.endswith("}"):
+                var_name = v[1:-1]
+                if var_name in params:
+                    updated_params[k] = params[var_name]
+                else:
+                    updated_params[k] = v
+            else:
+                updated_params[k] = v
+            # Try parsing as a RangeIteration and expanding
+            try:
+                updated_params[k] = RangeIteration.model_validate(
+                    updated_params[k]
+                ).values
+            except ValidationError:
+                pass
+        self.parameters = updated_params
+
     @property
     def cleaned_notebook_path(self) -> str:
         return get_cleaned_notebook_path(self.notebook_path, as_posix=True)
@@ -396,8 +433,15 @@ class JupyterNotebookStage(Stage):
         cmd = f"calkit nb execute --environment {self.environment} --no-check"
         if self.html_storage:
             cmd += " --to html"
-        for k, v in self.parameters.items():
-            cmd += f" -p {k}={v}"
+        if self.parameters:
+            # If we have parameters, we need to pass them as JSON, escaping
+            # double quotes
+            params_json = json.dumps(self.parameters)
+            # Now base64 encode
+            params_base64 = base64.b64encode(
+                params_json.encode("utf-8")
+            ).decode("utf-8")
+            cmd += f' --params-base64 "{params_base64}"'
         cmd += f' "{self.notebook_path}"'
         return cmd
 

{calkit_python-0.28.2 → calkit_python-0.28.4}/calkit/notebooks.py

@@ -1,5 +1,7 @@
 """Functionality for working with notebooks."""
 
+import hashlib
+import json
 import os
 from pathlib import PurePosixPath
 from typing import Any, Literal
@@ -21,7 +23,25 @@ def get_executed_notebook_path(
     nb_fname = os.path.basename(notebook_path)
     # If we have any parameters, add these to the notebook name
     if parameters:
-        params_txt = "-".join(f"{k}-{v}" for k, v in parameters.items())
+        parameters_cleaned = {}
+        for k, v in parameters.items():
+            if isinstance(v, list):
+                if len(v) == 0:
+                    v = "empty"
+                elif len(v) < 5:
+                    v = ",".join(map(str, v))
+                else:
+                    v = hashlib.md5(
+                        json.dumps(v, sort_keys=True).encode()
+                    ).hexdigest()[:7]
+            elif isinstance(v, dict):
+                v = hashlib.md5(
+                    json.dumps(v, sort_keys=True).encode()
+                ).hexdigest()[:7]
+            parameters_cleaned[k] = v
+        params_txt = "-".join(
+            f"{k}-{v}" for k, v in parameters_cleaned.items()
+        )
         nb_fname = f"{nb_fname.removesuffix('.ipynb')}-{params_txt}.ipynb"
     if to == "html":
         fname_out = nb_fname.removesuffix(".ipynb") + ".html"

{calkit_python-0.28.2 → calkit_python-0.28.4}/calkit/pipeline.py

@@ -8,6 +8,7 @@ import typer
 
 import calkit
 from calkit.environments import get_env_lock_fpath
+from calkit.models.iteration import expand_project_parameters
 from calkit.models.pipeline import (
     InputsFromStageOutputs,
     PathOutput,
@@ -92,8 +93,13 @@ def to_dvc(
         )
         dvc_stages[f"_check-env-{env_name}"] = stage
         env_lock_fpaths[env_name] = lock_fpath
+    project_params = expand_project_parameters(ck_info.get("parameters", {}))
     # Now convert Calkit stages into DVC stages
     for stage_name, stage in pipeline.stages.items():
+        # If this stage is a Jupyter notebook stage, we need to update its
+        # parameters if any reference project-level parameters
+        if stage.kind == "jupyter-notebook":
+            stage.update_parameters(params=project_params)
         dvc_stage = stage.to_dvc()
         # Add environment lock file to deps
         env_lock_fpath = env_lock_fpaths.get(stage.environment)
@@ -112,9 +118,7 @@ def to_dvc(
             format_dict = {}
             for n, iteration in enumerate(stage.iterate_over):
                 arg_name = iteration.arg_name
-                exp_vals = iteration.expand_values(
-                    params=ck_info.get("parameters", {})
-                )
+                exp_vals = iteration.expand_values(params=project_params)
                 if isinstance(arg_name, list):
                     dvc_arg_name = f"_arg{n}"
                     for arg_name_i in arg_name:
@@ -143,21 +147,31 @@ def to_dvc(
                 except Exception as e:
                     raise ValueError(
                         (
-                            f"Failed to format dep '{dep}': "
+                            f"Failed to format dep '{dep}' with "
+                            f"'{format_dict}': "
                             f"{e.__class__.__name__}: {e}"
                         )
                     )
             for out in dvc_stage.get("outs", []):
-                if isinstance(out, dict):
-                    formatted_outs.append(
-                        {
-                            str(list(out.keys())[0]).format(
-                                **format_dict
-                            ): dict(list(out.values())[0])
-                        }
+                try:
+                    if isinstance(out, dict):
+                        formatted_outs.append(
+                            {
+                                str(list(out.keys())[0]).format(
+                                    **format_dict
+                                ): dict(list(out.values())[0])
+                            }
+                        )
+                    else:
+                        formatted_outs.append(out.format(**format_dict))
+                except Exception as e:
+                    raise ValueError(
+                        (
+                            f"Failed to format out '{out}' with "
+                            f"'{format_dict}': "
+                            f"{e.__class__.__name__}: {e}"
+                        )
                     )
-                else:
-                    formatted_outs.append(out.format(**format_dict))
             dvc_stage["deps"] = formatted_deps
             dvc_stage["outs"] = formatted_outs
             dvc_stage["matrix"] = dvc_matrix

calkit_python-0.28.4/calkit/tests/cli/test_notebooks.py

@@ -0,0 +1,137 @@
+"""Tests for ``calkit.cli.notebooks``."""
+
+import base64
+import json
+import os
+import shutil
+import subprocess
+
+import calkit
+
+
+def test_clean_notebook_outputs(tmp_dir):
+    # Copy in a test notebook and clean it
+    nb_fpath = os.path.join(
+        os.path.dirname(__file__), "..", "..", "..", "test", "pipeline.ipynb"
+    )
+    shutil.copy(nb_fpath, "notebook.ipynb")
+    subprocess.check_call(["calkit", "nb", "clean", "notebook.ipynb"])
+
+
+def test_execute_notebook(tmp_dir):
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "project",
+            ".",
+            "-n",
+            "cool-project",
+            "--title",
+            "Cool project",
+        ]
+    )
+    subprocess.check_call(
+        ["calkit", "new", "uv-venv", "-n", "main", "ipykernel"]
+    )
+    nb_fpath = os.path.join(
+        os.path.dirname(__file__), "..", "..", "..", "test", "nb-subdir.ipynb"
+    )
+    os.makedirs("notebooks/results")
+    shutil.copy(nb_fpath, "notebooks/main.ipynb")
+    subprocess.check_call(
+        ["calkit", "nb", "execute", "notebooks/main.ipynb", "-e", "main"]
+    )
+    assert os.path.isfile("notebooks/results/something.txt")
+    os.makedirs("results")
+    shutil.copy("notebooks/main.ipynb", "main.ipynb")
+    subprocess.check_call(
+        ["calkit", "nb", "execute", "main.ipynb", "-e", "main"]
+    )
+    assert os.path.isfile("results/something.txt")
+    # Test we can execute with parameters
+    nb_fpath = os.path.join(
+        os.path.dirname(__file__), "..", "..", "..", "test", "nb-params.ipynb"
+    )
+    shutil.copy(nb_fpath, "nb-params.ipynb")
+    params1 = {
+        "my_value": 5,
+        "my_list": [1, 2, 3],
+    }
+    params2 = {
+        "my_dict": {"something": 55.5, "else": "b"},
+    }
+    subprocess.check_call(
+        [
+            "calkit",
+            "nb",
+            "execute",
+            "nb-params.ipynb",
+            "-e",
+            "main",
+            "--params-json",
+            json.dumps(params1),
+            "--params-base64",
+            base64.b64encode(json.dumps(params2).encode("utf-8")).decode(
+                "utf-8"
+            ),
+            "--verbose",
+        ]
+    )
+    params = params1 | params2
+    with open("params-out.json") as f:
+        params_out = json.load(f)
+    assert params_out["my_value"] == params["my_value"]
+    assert params_out["my_list"] == params["my_list"]
+    assert params_out["my_dict"] == params["my_dict"]
+    # Test we can execute in the pipeline with params
+    pipeline = {
+        "stages": {
+            "nb-params": {
+                "kind": "jupyter-notebook",
+                "notebook_path": "nb-params.ipynb",
+                "environment": "main",
+                "parameters": params,
+                "html_storage": None,
+            }
+        }
+    }
+    ck_info = calkit.load_calkit_info()
+    ck_info["pipeline"] = pipeline
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    subprocess.check_call(["calkit", "run"])
+    with open("params-out.json") as f:
+        params_out = json.load(f)
+    assert params_out["my_value"] == params["my_value"]
+    assert params_out["my_list"] == params["my_list"]
+    assert params_out["my_dict"] == params["my_dict"]
+    # Test we can patch in a range iteration from project parameters
+    project_params = {
+        "my_range": {"range": {"start": 1, "stop": 6, "step": 1}},
+        "my_project_value": 77,
+    }
+    pipeline = {
+        "stages": {
+            "nb-params": {
+                "kind": "jupyter-notebook",
+                "notebook_path": "nb-params.ipynb",
+                "environment": "main",
+                "parameters": params
+                | {"my_list": "{my_range}", "my_value": "{my_project_value}"},
+                "html_storage": None,
+            }
+        }
+    }
+    ck_info = calkit.load_calkit_info()
+    ck_info["parameters"] = project_params
+    ck_info["pipeline"] = pipeline
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    os.remove("params-out.json")
+    subprocess.check_call(["calkit", "run"])
+    with open("params-out.json") as f:
+        params_out = json.load(f)
+    assert params_out["my_value"] == project_params["my_project_value"]
+    assert params_out["my_list"] == list(range(1, 6))
+    assert params_out["my_dict"] == params["my_dict"]

{calkit_python-0.28.2 → calkit_python-0.28.4}/calkit/tests/models/test_pipeline.py

@@ -105,8 +105,10 @@ def test_jupyternotebookstage():
     assert s.html_path not in outs
     assert s.executed_notebook_path in outs
     assert "html" not in dvc_stage["cmd"]
-    assert " -p param1=value1 " in dvc_stage["cmd"]
-    assert " -p param2=value2 " in dvc_stage["cmd"]
+    assert (
+        " --params-base64 "
+        '"eyJwYXJhbTEiOiAidmFsdWUxIiwgInBhcmFtMiI6ICJ2YWx1ZTIifQ==" '
+    ) in dvc_stage["cmd"]
 
 
 def test_stageiteration():

{calkit_python-0.28.2 → calkit_python-0.28.4}/calkit/tests/test_pipeline.py

@@ -269,9 +269,7 @@ def test_to_dvc_notebook_stage():
     print(dvc_stages)
     stage = dvc_stages["notebook-1"]
     assert stage["cmd"].startswith("calkit nb execute")
-    assert " -p param1=${item.param1} " in stage["cmd"]
-    assert " -p param2=${item.param2} " in stage["cmd"]
-    assert " -p param3=${item.param3} " in stage["cmd"]
+    assert "--params-base64" in stage["cmd"]
     assert stage["matrix"]["param1"][0] == 5.0
     assert "m" in stage["matrix"]["param2"]
     assert "something" in stage["matrix"]["param3"]

{calkit_python-0.28.2 → calkit_python-0.28.4}/docs/notebooks.md

@@ -322,4 +322,4 @@ pipeline:
         - results/param1={param1}/param2={param2}/results.csv
 ```
 
-For more information, see [pipeline iteration](/pipeline#iteration).
+For more information, see [pipeline iteration](pipeline/index.md#iteration).

{calkit_python-0.28.2 → calkit_python-0.28.4}/docs/quickstart.md

@@ -25,7 +25,7 @@ If you're using Conda for environment management,
 e.g., with an `environment.yml` file,
 you can use the `calkit new conda-env` command.
 
-Next, we can start building our [pipeline](pipeline.md).
+Next, we can start building our [pipeline](pipeline/index.md).
 Let's say we have a Jupyter notebook called `collect-data.ipynb`
 that produces raw data at `data/raw.h5`.
 We can add a pipeline stage to run this notebook in the `main` environment

calkit_python-0.28.4/docs/tutorials/github-actions.md

@@ -0,0 +1,83 @@
+# Running Calkit in GitHub Actions
+
+A project can be set up to automatically run the pipeline every time a
+change is pushed, either to the main branch or on a pull request,
+using GitHub Actions.
+The latter allows for inspection of outputs before merging into main.
+
+To get started, generate a DVC token and set it in your GitHub Actions
+secrets as `CALKIT_DVC_TOKEN`,
+either at your account or project level.
+On calkit.io, there are shortcuts on the project page for managing both
+Calkit tokens and GitHub Actions secrets:
+
+![Project quick actions links](img/quick-actions.png)
+
+![DVC token](img/latex-codespaces/new-token.png)
+
+![Actions repo secrets](img/actions-repo-secrets.png)
+
+Next, add a workflow to the project in the `.github/workflows` folder.
+For example, we can put the content below into `.github/workflows/run.yml`:
+
+```yaml
+name: Run pipeline
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: write
+
+# Make sure we only ever run one per branch so we don't have issues pushing
+# after running the pipeline
+concurrency:
+  group: calkit-run-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  main:
+    name: Run
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # For PRs, checkout the head ref to avoid detached HEAD
+          ref: ${{ github.head_ref || github.ref_name }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Configure Git credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+      - name: Install Calkit
+        run: uv tool install calkit-python
+      - name: Run Calkit
+        uses: calkit/run-action@v1
+        with:
+          dvc_token: ${{ secrets.CALKIT_DVC_TOKEN }}
+```
+
+This particular example installs Calkit with uv,
+meaning `uv` and `uv-venv` environment types will work without any additional
+configuration.
+Docker is also installed by default on the `ubuntu-latest` machine.
+If other environment types are used in the project,
+setup steps may be necessary for those, e.g.,
+[`setup-miniconda`](https://github.com/marketplace/actions/setup-miniconda)
+or [`install-juliaup`](https://github.com/marketplace/actions/install-juliaup).
+
+By default, the Calkit GitHub Action will run the pipeline and save results.
+This is why the workflow needs write permissions and configures Git
+credentials to act as the GitHub Actions bot.
+The workflow also limits concurrency so multiple jobs don't attempt to push
+to the same branch at the same time.
+
+It's possible to configure the action to not save results, e.g., if you
+just want to check that the pipeline can run without errors.
+See the [documentation](https://github.com/marketplace/actions/run-calkit)
+for all available options.

{calkit_python-0.28.2 → calkit_python-0.28.4}/docs/tutorials/index.md

@@ -10,3 +10,4 @@
 - [Adding a new LaTeX-based publication with its own Docker build environment](adding-latex-pub-docker.md)
 - [A reproducible workflow using Microsoft Office (Word and Excel)](office.md)
 - [Using Calkit with MATLAB](matlab.md)
+- [Automation with GitHub Actions](github-actions.md)

{calkit_python-0.28.2 → calkit_python-0.28.4}/mkdocs.yml

@@ -67,6 +67,7 @@ nav:
       - tutorials/conda-envs.md
       - tutorials/office.md
       - tutorials/matlab.md
+      - tutorials/github-actions.md
   - CLI reference: cli-reference.md
   - Help and support: help.md
 markdown_extensions: