calkit-python 0.26.11__tar.gz → 0.27.0__tar.gz

{calkit_python-0.26.11 → calkit_python-0.27.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.26.11
+Version: 0.27.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -18,8 +18,10 @@ Requires-Dist: docx2pdf
 Requires-Dist: dvc==3.61.0
 Requires-Dist: fastapi
 Requires-Dist: gitpython
+Requires-Dist: jupyterlab>=4.4.5
 Requires-Dist: keyring
 Requires-Dist: nbconvert
+Requires-Dist: papermill>=2.6.0
 Requires-Dist: pillow
 Requires-Dist: psutil>=7.0.0
 Requires-Dist: pydantic-settings

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.26.11"
+__version__ = "0.27.0"
 
 from .core import *  # noqa: F403, I001
 from . import git  # noqa: F401

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/cli/check.py

@@ -544,6 +544,26 @@ def check_matlab_env(
     )
 
 
+@check_app.command(name="dependencies")
+@check_app.command(name="deps")
+def check_dependencies(
+    verbose: Annotated[
+        bool, typer.Option("--verbose", "-v", help="Print verbose output")
+    ] = False,
+):
+    """Check that a project's system-level dependencies are set up
+    correctly.
+    """
+    typer.echo("Checking project dependencies")
+    dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
+    try:
+        calkit.check_system_deps()
+    except Exception as e:
+        raise_error(str(e))
+    message = "✅ All set!"
+    typer.echo(message.encode("utf-8", errors="replace"))
+
+
 @check_app.command(name="env-vars")
 def check_env_vars(
     verbose: Annotated[

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/cli/main.py

@@ -380,7 +380,12 @@ def add(
         else:
             commit_message = f"Add {paths[0]}"
     if to is not None:
-        subprocess.call([to, "add"] + paths)
+        if to == "git":
+            subprocess.call(["git", "add"] + paths)
+        elif to == "dvc":
+            subprocess.call([sys.executable, "-m", "dvc", "add"] + paths)
+        else:
+            raise_error(f"Invalid option for 'to': {to}")
     else:
         if "." in paths:
             paths.remove(".")
@@ -388,8 +393,14 @@ def add(
             for dvc_uncommitted in dvc_status["uncommitted"].get(
                 "modified", []
             ):
-                typer.echo(f"Adding {dvc_uncommitted} to DVC")
-                dvc_repo.commit(dvc_uncommitted, force=True)
+                if os.path.exists(dvc_uncommitted):
+                    typer.echo(f"Adding {dvc_uncommitted} to DVC")
+                    dvc_repo.commit(dvc_uncommitted, force=True)
+                else:
+                    warn(
+                        f"DVC uncommitted '{dvc_uncommitted}' does not exist; "
+                        "skipping"
+                    )
             if not disable_auto_ignore:
                 for untracked_file in untracked_git_files:
                     if (
@@ -1779,3 +1790,20 @@ def call_dvc(
     """
     process = subprocess.run([sys.executable, "-m", "dvc"] + sys.argv[2:])
     sys.exit(process.returncode)
+
+
+@app.command(
+    name="jupyter",
+    add_help_option=False,
+    context_settings={
+        "ignore_unknown_options": True,
+        "allow_extra_args": True,
+    },
+)
+def run_jupyter(
+    ctx: typer.Context,
+    help: Annotated[bool, typer.Option("-h", "--help")] = False,
+):
+    """Run a command with the Jupyter CLI."""
+    process = subprocess.run([sys.executable, "-m", "jupyter"] + sys.argv[2:])
+    sys.exit(process.returncode)

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/cli/notebooks.py

@@ -6,7 +6,9 @@ import os
 import subprocess
 import sys
 from pathlib import PurePosixPath
+from typing import Any
 
+import papermill
 import typer
 from typing_extensions import Annotated
 
@@ -45,6 +47,70 @@ def clean_notebook_outputs(path: str):
     )
 
 
+def _parse_params(params: list[str]) -> dict[str, Any]:
+    """Parse parameters from command line arguments."""
+    parameters = {}
+    for param in params:
+        if "=" not in param:
+            raise ValueError(f"Parameter must be in key=value format: {param}")
+        key, value = param.split("=", 1)
+        # Try to convert to appropriate types
+        try:
+            if "." in value:
+                parameters[key] = float(value)
+            elif value.isdigit() or (
+                value.startswith("-") and value[1:].isdigit()
+            ):
+                parameters[key] = int(value)
+            elif value.lower() in ("true", "false"):
+                parameters[key] = value.lower() == "true"
+            else:
+                parameters[key] = value
+        except ValueError:
+            parameters[key] = value
+    return parameters
+
+
+@notebooks_app.command("check-env-kernel")
+def check_env_kernel(
+    env_name: Annotated[
+        str,
+        typer.Option(
+            "--environment",
+            "-e",
+            help="Environment name in which to run the notebook.",
+        ),
+    ],
+    no_check: Annotated[
+        bool,
+        typer.Option(
+            "--no-check", help="Do not check environment before executing."
+        ),
+    ] = False,
+    verbose: Annotated[
+        bool, typer.Option("--verbose", "-v", help="Print verbose output.")
+    ] = False,
+):
+    """Check that an environment has a registered kernel."""
+    from calkit.cli.main import run_in_env
+
+    project_name = calkit.detect_project_name(prepend_owner=False)
+    kernel_name = calkit.to_kebab_case(f"{project_name}-{env_name}")
+    cmd = [
+        "python",
+        "-m",
+        "ipykernel",
+        "install",
+        "--user",
+        "--name",
+        kernel_name,
+        "--display-name",
+        f"{project_name}: {env_name}",
+    ]
+    run_in_env(cmd=cmd, env_name=env_name, no_check=no_check, verbose=verbose)
+    return kernel_name
+
+
 @notebooks_app.command("execute")
 def execute_notebook(
     path: str,
@@ -66,6 +132,14 @@ def execute_notebook(
             "--no-check", help="Do not check environment before executing."
         ),
     ] = False,
+    params: Annotated[
+        list[str],
+        typer.Option(
+            "--param",
+            "-p",
+            help="Parameter to pass to the notebook in key=value format.",
+        ),
+    ] = [],
     verbose: Annotated[
         bool, typer.Option("--verbose", "-v", help="Print verbose output.")
     ] = False,
@@ -79,33 +153,41 @@ def execute_notebook(
 
     if os.path.isabs(path):
         raise ValueError("Path must be relative")
-    # First, always execute the notebook and save as ipynb
+    # First, ensure the specified environment has a kernel we can use
+    kernel_name = check_env_kernel(
+        env_name=env_name, no_check=no_check, verbose=verbose
+    )
+    # Parse parameters
+    if params:
+        try:
+            parsed_params = _parse_params(params)
+        except ValueError as e:
+            raise_error(str(e))
+    else:
+        parsed_params = {}
+    # Next, always execute the notebook and save as ipynb
     fpath_out_exec = calkit.notebooks.get_executed_notebook_path(
-        notebook_path=path, to="notebook", as_posix=True
+        notebook_path=path,
+        to="notebook",
+        as_posix=True,
+        parameters=parsed_params,
     )
     folder = os.path.dirname(fpath_out_exec)
     os.makedirs(folder, exist_ok=True)
-    fname = os.path.basename(fpath_out_exec)
-    fpath_out_exec = PurePosixPath(fpath_out_exec).as_posix()
-    cmd = [
-        "jupyter",
-        "nbconvert",
-        path,
-        "--execute",
-        "--to",
-        "notebook",
-        "--output-dir",
-        PurePosixPath(folder).as_posix(),
-        "--output",
-        fname,
-    ]
-    run_in_env(cmd=cmd, env_name=env_name, no_check=no_check, verbose=verbose)
+    papermill.execute_notebook(
+        input_path=path,
+        output_path=fpath_out_exec,
+        kernel_name=kernel_name,
+        log_output=True,
+        parameters=parsed_params,
+    )
     for to_fmt in to:
         if to_fmt != "notebook":
             try:
                 fpath_out = calkit.notebooks.get_executed_notebook_path(
                     notebook_path=path,
                     to=to_fmt,  # type: ignore
+                    parameters=parsed_params,
                 )
             except ValueError:
                 raise_error(f"Invalid output format: '{to}'")
@@ -114,6 +196,8 @@ def execute_notebook(
             fname_out = os.path.basename(fpath_out)
             # Now convert without executing or checking the environment
             cmd = [
+                sys.executable,
+                "-m",
                 "jupyter",
                 "nbconvert",
                 fpath_out_exec,

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/core.py

@@ -318,11 +318,12 @@ def check_system_deps(
                 raise ValueError(f"Malformed dependency: {dep}")
             if len(keys) == 1:
                 dep_name = keys[0]
+                dep_kind = dep[dep_name].get("kind", "app")
             elif "name" in keys:
                 dep_name = dep["name"]
+                dep_kind = dep.get("kind", "app")
             else:
                 raise ValueError(f"Malformed dependency: {dep}")
-            dep_kind = dep[dep_name].get("kind", "app")
         else:
             dep_name = re.split("[=<>]", dep)[0]
             dep_kind = "app"
@@ -478,12 +479,16 @@ def get_latest_project_status(wdir: str | None = None) -> ProjectStatus | None:
         return statuses[-1]  # type: ignore
 
 
-def detect_project_name(wdir: str | None = None) -> str:
+def detect_project_name(
+    wdir: str | None = None, prepend_owner: bool = True
+) -> str:
     """Detect a Calkit project owner and name."""
     ck_info = load_calkit_info(wdir=wdir)
     name = ck_info.get("name")
+    if name is not None and not prepend_owner:
+        return name
     owner = ck_info.get("owner")
-    if name is None or owner is None:
+    if name is None and owner is None:
         try:
             url = Repo(path=wdir).remote().url
         except ValueError:
@@ -494,7 +499,9 @@ def detect_project_name(wdir: str | None = None) -> str:
         name = project_name
     if owner is None:
         owner = owner_name
-    return f"{owner}/{name}"
+    if prepend_owner:
+        return f"{owner}/{name}"
+    return name
 
 
 def get_dep_version(dep_name: str) -> str | None:

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/models/pipeline.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from pathlib import PurePosixPath
-from typing import Literal
+from typing import Any, Literal
 
 from pydantic import BaseModel, ConfigDict, Discriminator
 from typing_extensions import Annotated
@@ -273,6 +273,7 @@ class JupyterNotebookStage(Stage):
     cleaned_ipynb_storage: Literal["git", "dvc"] | None = "git"
     executed_ipynb_storage: Literal["git", "dvc"] | None = "dvc"
     html_storage: Literal["git", "dvc"] | None = "dvc"
+    parameters: dict[str, Any] = {}
 
     @property
     def cleaned_notebook_path(self) -> str:
@@ -281,13 +282,19 @@ class JupyterNotebookStage(Stage):
     @property
     def executed_notebook_path(self) -> str:
         return get_executed_notebook_path(
-            self.notebook_path, to="notebook", as_posix=True
+            self.notebook_path,
+            to="notebook",
+            as_posix=True,
+            parameters=self.parameters,
         )
 
     @property
     def html_path(self) -> str:
         return get_executed_notebook_path(
-            self.notebook_path, to="html", as_posix=True
+            self.notebook_path,
+            to="html",
+            as_posix=True,
+            parameters=self.parameters,
         )
 
     @property
@@ -299,6 +306,8 @@ class JupyterNotebookStage(Stage):
         cmd = f"calkit nb execute --environment {self.environment} --no-check"
         if self.html_storage:
             cmd += " --to html"
+        for k, v in self.parameters.items():
+            cmd += f" -p {k}={v}"
         cmd += f' "{self.notebook_path}"'
         return cmd
 

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/notebooks.py

@@ -2,7 +2,7 @@
 
 import os
 from pathlib import PurePosixPath
-from typing import Literal
+from typing import Any, Literal
 
 import git
 
@@ -11,16 +11,22 @@ from calkit.models.io import InputsFromStageOutputs, PathOutput
 
 
 def get_executed_notebook_path(
-    notebook_path: str, to: Literal["html", "notebook"], as_posix: bool = True
+    notebook_path: str,
+    to: Literal["html", "notebook"],
+    as_posix: bool = True,
+    parameters: dict[str, Any] | None = None,
 ) -> str:
     """Return the path of an executed notebook."""
     nb_dir = os.path.dirname(notebook_path)
     nb_fname = os.path.basename(notebook_path)
+    # If we have any parameters, add these to the notebook name
+    if parameters:
+        params_txt = "-".join(f"{k}-{v}" for k, v in parameters.items())
+        nb_fname = f"{nb_fname.removesuffix('.ipynb')}-{params_txt}.ipynb"
     if to == "html":
         fname_out = nb_fname.removesuffix(".ipynb") + ".html"
     else:
         fname_out = nb_fname
-    # Different output types go to different subdirectories
     subdirs = {"html": "html", "notebook": "executed"}
     p = os.path.join(".calkit", "notebooks", subdirs[to], nb_dir, fname_out)
     if as_posix:

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/tests/cli/test_main.py

@@ -322,6 +322,7 @@ def test_add(tmp_dir):
         f.write(os.urandom(2_000_000))
     subprocess.check_call(["calkit", "add", "data2", "-M"])
     assert repo.head.commit.message.strip() == "Add data2"
+    subprocess.check_call(["calkit", "add", "--to", "dvc", "large.bin"])
 
 
 def test_status(tmp_dir):

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/tests/models/test_pipeline.py

@@ -87,3 +87,18 @@ def test_jupyternotebookstage():
     assert s.html_path not in outs
     assert s.executed_notebook_path in outs
     assert "html" not in dvc_stage["cmd"]
+    # Test with parameters
+    s = JupyterNotebookStage(
+        environment="main",
+        notebook_path="something.ipynb",
+        inputs=["file.txt"],
+        html_storage=None,
+        parameters={"param1": "value1", "param2": "value2"},
+    )
+    dvc_stage = s.to_dvc()
+    outs = dvc_outs_to_str_list(dvc_stage)
+    assert s.html_path not in outs
+    assert s.executed_notebook_path in outs
+    assert "html" not in dvc_stage["cmd"]
+    assert " -p param1=value1 " in dvc_stage["cmd"]
+    assert " -p param2=value2 " in dvc_stage["cmd"]

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/tests/test_core.py

@@ -4,6 +4,7 @@ import os
 import subprocess
 
 import git
+import pytest
 
 import calkit
 
@@ -78,3 +79,21 @@ def test_get_env_var_dep_names():
         "MY_ENV_VAR",
         "MY_OTHER_ENV_VAR",
     ]
+
+
+def test_check_system_deps(tmp_dir):
+    ck_info = {
+        "dependencies": [
+            "uv",
+            {"kind": "env-var", "name": "MY_ENV_VAR"},
+            {"MY_ENV_VAR2": {"kind": "env-var"}},
+        ]
+    }
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    subprocess.check_call(
+        ["calkit", "check", "dependencies"],
+        env=os.environ.copy() | {"MY_ENV_VAR": "5", "MY_ENV_VAR2": "55"},
+    )
+    with pytest.raises(ValueError):
+        calkit.check_system_deps()

{calkit_python-0.26.11 → calkit_python-0.27.0}/calkit/tests/test_pipeline.py

@@ -219,6 +219,62 @@ def test_to_dvc_notebook_stage():
             found_html = True
             assert out[p]["cache"]
     assert found_html
+    # Test we can use parameters and iterations in notebook stages
+    ck_info = {
+        "parameters": {
+            "param1": [{"range": {"start": 5, "stop": 23, "step": 2}}],
+            "param2": ["s", "m", "l"],
+            "param3": [
+                {"range": {"start": 0.1, "stop": 1.1, "step": 0.11}},
+                55,
+                "something",
+            ],
+        },
+        "pipeline": {
+            "stages": {
+                "notebook-1": {
+                    "kind": "jupyter-notebook",
+                    "environment": "something",
+                    "notebook_path": nb_path,
+                    "outputs": [
+                        "figures/fig1.png",
+                    ],
+                    "html_storage": "dvc",
+                    "cleaned_ipynb_storage": "git",
+                    "executed_ipynb_storage": None,
+                    "parameters": {
+                        "param1": "{param1}",
+                        "param2": "{param2}",
+                        "param3": "{param3}",
+                    },
+                    "iterate_over": [
+                        {
+                            "arg_name": "param1",
+                            "values": [{"parameter": "param1"}],
+                        },
+                        {
+                            "arg_name": "param2",
+                            "values": [{"parameter": "param2"}],
+                        },
+                        {
+                            "arg_name": "param3",
+                            "values": [{"parameter": "param3"}],
+                        },
+                    ],
+                },
+            }
+        },
+    }
+    dvc_stages = calkit.pipeline.to_dvc(ck_info=ck_info, write=False)
+    print(dvc_stages)
+    stage = dvc_stages["notebook-1"]
+    assert stage["cmd"].startswith("calkit nb execute")
+    assert " -p param1=${item.param1} " in stage["cmd"]
+    assert " -p param2=${item.param2} " in stage["cmd"]
+    assert " -p param3=${item.param3} " in stage["cmd"]
+    assert stage["matrix"]["param1"][0] == 5.0
+    assert "m" in stage["matrix"]["param2"]
+    assert "something" in stage["matrix"]["param3"]
 
 
 def test_remove_stage(tmp_dir):

{calkit_python-0.26.11 → calkit_python-0.27.0}/docs/calkit-yaml.md

@@ -3,7 +3,7 @@
 The `calkit.yaml` file serves as a small "database"
 for the project's important metadata, which includes its:
 
-- Global or system-level dependencies
+- Global or system-level [dependencies](dependencies.md)
   (applications, libraries, environmental variables)
 - Questions the project seeks to answer
 - [Environments](environments.md)

{calkit_python-0.26.11 → calkit_python-0.27.0}/docs/dependencies.md

@@ -40,14 +40,14 @@ changing anything.
 ```yaml
 dependencies:
   - docker
-  - STRAVA_CLIENT_ID:
-      kind: env-var
-      notes: >
-        The STRAVA_CLIENT_ID and STRAVA_CLIENT_SECRET environmental
-        variables can be set in the .env file after creating a Strava
-        application at https://www.strava.com/settings/api
-  - STRAVA_CLIENT_SECRET:
-      kind: env-var
+  - name: STRAVA_CLIENT_ID
+    kind: env-var
+    notes: >
+      The STRAVA_CLIENT_ID and STRAVA_CLIENT_SECRET environmental
+      variables can be set in the .env file after creating a Strava
+      application at https://www.strava.com/settings/api
+  - name: STRAVA_CLIENT_SECRET
+    kind: env-var
 ```
 
 As we can see in the notes for `STRAVA_CLIENT_ID`,

{calkit_python-0.26.11 → calkit_python-0.27.0}/docs/notebooks.md

@@ -236,3 +236,90 @@ about that cell's code or environment has changed.
 
 For a more in-depth look at using the `%%stage` cell magic,
 see [this tutorial](tutorials/notebook-pipeline.md).
+
+## Parameterizing notebooks
+
+Thanks to [Papermill](https://github.com/nteract/papermill),
+Calkit can run notebooks that have been parameterized,
+and executed versions will be saved with parameters in their names.
+To parameterize a notebook,
+first add a cell with the "parameters" tag and add your parameters there
+as variable declarations, e.g.,
+
+```python
+param1 = 5
+param2 = "something"
+```
+
+In JupyterLab, you can use the property inspector to edit cell tags:
+
+![JupyterLab property inspector](img/jupyterlab-params.png)
+
+In VS Code, the cell context menu can be used to mark the cell as parameters:
+
+![VS Code context menu for marking parameters cell](img/vscode-nb-params.png)
+
+Then, in the Calkit pipeline, add `parameters` to the notebook stage:
+
+```yaml
+# In calkit.yaml
+environments:
+  main:
+    kind: uv-venv
+    path: requirements.txt
+    prefix: .venv
+    python: "3.13"
+pipeline:
+  stages:
+    notebook-56-a:
+      kind: jupyter-notebook
+      notebook_path: notebook.ipynb
+      environment: main
+      parameters:
+        param1: 56
+        param2: a
+    notebook-58-b:
+      kind: jupyter-notebook
+      notebook_path: notebook.ipynb
+      environment: main
+      parameters:
+        param1: 58
+        param2: b
+```
+
+### Iterating over parameterized notebooks
+
+The example below shows project-level parameters used to iterate over a
+notebook stage:
+
+```yaml
+parameters:
+  param1:
+    - range:
+        start: 0
+        stop: 10
+        step: 2
+  param2:
+    - random-forest
+    - lightgbm
+pipeline:
+  stages:
+    my-notebook-with-params:
+      kind: jupyter-notebook
+      environment: my-env
+      notebook_path: notebook.ipynb
+      iterate_over:
+        - arg_name: param1
+          values:
+            - parameter: param1 # Args and params can be named differently
+        - arg_name: param2
+          values:
+            - parameter: param2
+      parameters:
+        param1: "{param1}" # Notebook params can also be named differently
+        param2: "{param2}"
+      outputs:
+        - results/param1={param1}/param2={param2}/results.csv
+```
+
+For more information, see [pipeline iteration](/pipeline#iteration).

{calkit_python-0.26.11 → calkit_python-0.27.0}/pyproject.toml

@@ -32,6 +32,8 @@ dependencies = [
   "uvicorn",
   "tqdm>=4.67.1",
   "psutil>=7.0.0",
+  "papermill>=2.6.0",
+  "jupyterlab>=4.4.5",
 ]
 description = "Reproducibility simplified."
 dynamic = ["version"]
@@ -94,7 +96,7 @@ show_error_codes = true
 target-version = "py310"
 line-length = 79
 fix = true
-extend-select = ["I"]
+lint.extend-select = ["I"]
 
 [tool.pytest.ini_options]
 env = ["CALKIT_ENV = test"]