calkit-python 0.23.0__tar.gz → 0.24.0__tar.gz

{calkit_python-0.23.0 → calkit_python-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.23.0
+Version: 0.24.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.23.0 → calkit_python-0.24.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.23.0"
+__version__ = "0.24.0"
 
 from .core import *  # noqa: F403, I001
 from . import git  # noqa: F401

{calkit_python-0.23.0 → calkit_python-0.24.0}/calkit/cli/check.py

@@ -11,6 +11,8 @@ import subprocess
 import warnings
 from typing import Annotated
 
+from calkit.environments import get_env_lock_fpath
+
 # See https://github.com/calkit/calkit/issues/346
 with warnings.catch_warnings():
     warnings.filterwarnings("ignore", category=UserWarning)
@@ -40,28 +42,101 @@ def check_repro(
     typer.echo(res.to_pretty().encode("utf-8", errors="replace"))
 
 
-@check_app.command(name="call")
-def check_call(
-    cmd: Annotated[str, typer.Argument(help="Command to check.")],
-    if_error: Annotated[
+@check_app.command(
+    name="env",
+    help="Check that an environment is up-to-date (alias for 'environment').",
+)
+@check_app.command(name="environment")
+def check_environment(
+    env_name: Annotated[
         str,
-        typer.Option(
-            "--if-error", help="Command to run if there is an error."
-        ),
+        typer.Option("--name", "-n", help="Name of the environment to check."),
     ],
+    verbose: Annotated[
+        bool, typer.Option("--verbose", help="Print verbose output.")
+    ] = False,
 ):
-    """Check that a command succeeds and run an alternate if not."""
-    try:
-        subprocess.check_call(cmd, shell=True)
-        typer.echo("Command succeeded")
-    except subprocess.CalledProcessError:
-        typer.echo("Command failed")
+    """Check that an environment is up-to-date."""
+    dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
+    ck_info = calkit.load_calkit_info(process_includes="environments")
+    envs = ck_info.get("environments", {})
+    if not envs:
+        raise_error("No environments defined in calkit.yaml")
+    if isinstance(envs, list):
+        raise_error("Error: Environments should be a dict, not a list")
+    assert isinstance(envs, dict)
+    if env_name not in envs:
+        raise_error(f"Environment '{env_name}' does not exist")
+    env = envs[env_name]
+    if env["kind"] == "docker":
+        if "image" not in env:
+            raise_error("Image must be defined for Docker environments")
+        if "path" in env:
+            check_docker_env(
+                tag=env["image"],
+                fpath=env["path"],
+                lock_fpath=get_env_lock_fpath(
+                    env=env, env_name=env_name, as_posix=False
+                ),
+                platform=env.get("platform"),
+                deps=env.get("deps", []),
+                quiet=not verbose,
+            )
+    elif env["kind"] == "conda":
+        check_conda_env(
+            env_fpath=env["path"],
+            output_fpath=get_env_lock_fpath(
+                env=env, env_name=env_name, as_posix=False
+            ),
+            relaxed=True,  # TODO: Add option?
+            quiet=not verbose,
+        )
+    elif env["kind"] in ["pixi", "uv"]:
+        cmd = [env["kind"], "lock"]
+        if verbose:
+            typer.echo(f"Running command: {cmd}")
         try:
-            typer.echo("Attempting fallback call")
-            subprocess.check_call(if_error, shell=True)
-            typer.echo("Fallback call succeeded")
+            subprocess.check_call(cmd)
         except subprocess.CalledProcessError:
-            raise_error("Fallback call failed")
+            raise_error(f"Failed to check {env['kind']} environment")
+    elif (kind := env["kind"]) in ["uv-venv", "venv"]:
+        if "prefix" not in env:
+            raise_error("venv environments require a prefix")
+        if "path" not in env:
+            raise_error("venv environments require a path")
+        prefix = env["prefix"]
+        path = env["path"]
+        # Check environment
+        check_venv(
+            path=path,
+            prefix=prefix,
+            use_uv=kind == "uv-venv",
+            python=env.get("python"),
+            lock_fpath=get_env_lock_fpath(
+                env=env, env_name=env_name, as_posix=False
+            ),
+            verbose=verbose,
+        )
+    elif env["kind"] == "ssh":
+        # TODO: How to check SSH environments?
+        # Maybe just check that we can connect
+        raise_error(
+            "Environment checking not implemented for SSH environments"
+        )
+    elif env["kind"] == "renv":
+        try:
+            subprocess.check_call(["Rscript", "-e", "'renv::restore()'"])
+        except subprocess.CalledProcessError:
+            raise_error("Failed to check renv")
+    elif env["kind"] == "matlab":
+        check_matlab_env(
+            env_name=env_name,
+            output_fpath=get_env_lock_fpath(
+                env=env, env_name=env_name, as_posix=False
+            ),  # type: ignore
+        )
+    else:
+        raise_error(f"Environment kind '{env['kind']}' not supported")
 
 
 @check_app.command(name="docker-env")
@@ -273,6 +348,9 @@ def check_venv(
             "--python", help="Python version to specify if using uv."
         ),
     ] = None,
+    quiet: Annotated[
+        bool, typer.Option("--quiet", help="Do not print any output")
+    ] = False,
     verbose: Annotated[
         bool, typer.Option("--verbose", help="Print verbose output.")
     ] = False,
@@ -283,7 +361,7 @@ def check_venv(
         ["uv", "venv"] if kind == "uv-venv" else ["python", "-m", "venv"]
     )
     pip_cmd = "pip" if kind == "venv" else "uv pip"
-    pip_install_args = "-q"
+    pip_install_args = "-q" if quiet else ""
     if python is not None and not use_uv:
         raise_error("Python version cannot be specified if not using uv")
     if python is not None and use_uv:
@@ -319,7 +397,7 @@ def check_venv(
     try:
         if verbose:
             typer.echo(f"Running command: {check_cmd}")
-        subprocess.check_output(
+        subprocess.check_call(
             check_cmd,
             shell=True,
             cwd=wdir,
@@ -374,7 +452,11 @@ def check_matlab_env(
 
 
 @check_app.command(name="env-vars")
-def check_env_vars():
+def check_env_vars(
+    verbose: Annotated[
+        bool, typer.Option("--verbose", "-v", help="Print verbose output")
+    ] = False,
+):
     """Check that the project's required environmental variables exist."""
     typer.echo("Checking project environmental variables")
     dotenv.load_dotenv(dotenv_path=".env")
@@ -383,11 +465,20 @@ def check_env_vars():
     env_var_deps = {}
     for d in deps:
         if isinstance(d, dict):
-            name = list(d.keys())[0]
+            keys = list(d.keys())
+            if len(keys) > 1:
+                raise_error(
+                    f"Malformed dependency: {d}\n"
+                    "Dependencies with attributes should have a single key "
+                    "(their name)"
+                )
+            name = keys[0]
             attrs = list(d.values())[0]
             if attrs.get("kind") == "env-var":
                 env_var_deps[name] = attrs
     for name, attrs in env_var_deps.items():
+        if verbose:
+            typer.echo(f"Checking for environmental variable '{name}'")
         if name not in os.environ:
             typer.echo(f"Missing env var '{name}'")
             if "default" in attrs:
@@ -446,3 +537,27 @@ def check_pipeline(
             raise_error(
                 f"Failed to compile pipeline: {e.__class__.__name__}: {e}"
             )
+
+
+@check_app.command(name="call")
+def check_call(
+    cmd: Annotated[str, typer.Argument(help="Command to check.")],
+    if_error: Annotated[
+        str,
+        typer.Option(
+            "--if-error", help="Command to run if there is an error."
+        ),
+    ],
+):
+    """Check that a command succeeds and run an alternate if not."""
+    try:
+        subprocess.check_call(cmd, shell=True)
+        typer.echo("Command succeeded")
+    except subprocess.CalledProcessError:
+        typer.echo("Command failed")
+        try:
+            typer.echo("Attempting fallback call")
+            subprocess.check_call(if_error, shell=True)
+            typer.echo("Fallback call succeeded")
+        except subprocess.CalledProcessError:
+            raise_error("Fallback call failed")

{calkit_python-0.23.0 → calkit_python-0.24.0}/calkit/cli/main.py

@@ -989,6 +989,7 @@ def run_in_env(
                     env=env, env_name=env_name, as_posix=False
                 ),
                 wdir=wdir,
+                quiet=True,
                 verbose=verbose,
             )
         # Now run the command

calkit_python-0.24.0/calkit/cli/notebooks.py

@@ -0,0 +1,116 @@
+"""Notebooks CLI."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+
+import typer
+from typing_extensions import Annotated
+
+import calkit.notebooks
+from calkit.cli.core import raise_error
+
+notebooks_app = typer.Typer(no_args_is_help=True)
+
+
+@notebooks_app.command("clean")
+def clean_notebook_outputs(path: str):
+    """Clean notebook and place a copy in the cleaned notebooks directory.
+
+    This can be useful to use as a preprocessing DVC stage to use a clean
+    notebook as a dependency for a stage that caches and executed notebook.
+    """
+    if os.path.isabs(path):
+        raise ValueError("Path must be relative")
+    fpath_out = calkit.notebooks.get_cleaned_notebook_path(path)
+    folder = os.path.dirname(fpath_out)
+    os.makedirs(folder, exist_ok=True)
+    fpath_out = os.path.abspath(fpath_out)
+    subprocess.call(
+        [
+            "jupyter",
+            "nbconvert",
+            path,
+            "--clear-output",
+            "--to",
+            "notebook",
+            "--output",
+            fpath_out,
+        ]
+    )
+
+
+@notebooks_app.command("execute")
+def execute_notebook(
+    path: str,
+    env_name: Annotated[
+        str,
+        typer.Option(
+            "--environment",
+            "-e",
+            help="Environment name in which to run the notebook.",
+        ),
+    ],
+    to: Annotated[
+        list[str],
+        typer.Option("--to", help="Output format ('html' or 'notebook')."),
+    ] = ["notebook"],
+    no_check: Annotated[
+        bool,
+        typer.Option(
+            "--no-check", help="Do not check environment before executing."
+        ),
+    ] = False,
+):
+    """Execute notebook and place a copy in the relevant directory.
+
+    This can be useful to use as a preprocessing DVC stage to use a clean
+    notebook as a dependency for a stage that caches and executed notebook.
+    """
+    from calkit.cli.main import run_in_env
+
+    if os.path.isabs(path):
+        raise ValueError("Path must be relative")
+    # First, always execute the notebook and save as ipynb
+    fpath_out_exec = calkit.notebooks.get_executed_notebook_path(
+        notebook_path=path, to="notebook", as_posix=True
+    )
+    folder = os.path.dirname(fpath_out_exec)
+    os.makedirs(folder, exist_ok=True)
+    fpath_out_exec = os.path.abspath(fpath_out_exec)
+    cmd = [
+        "jupyter",
+        "nbconvert",
+        path,
+        "--execute",
+        "--to",
+        "notebook",
+        "--output",
+        fpath_out_exec,
+    ]
+    run_in_env(cmd=cmd, env_name=env_name, no_check=no_check)
+    for to_fmt in to:
+        if to_fmt != "notebook":
+            try:
+                fpath_out = calkit.notebooks.get_executed_notebook_path(
+                    notebook_path=path,
+                    to=to_fmt,  # type: ignore
+                )
+            except ValueError:
+                raise_error(f"Invalid output format: '{to}'")
+            folder = os.path.dirname(fpath_out)
+            os.makedirs(folder, exist_ok=True)
+            fpath_out = os.path.abspath(fpath_out)
+            # Now convert without executing or checking the environment
+            cmd = [
+                "jupyter",
+                "nbconvert",
+                fpath_out_exec,
+                "--to",
+                to_fmt,
+                "--output",
+                fpath_out,
+            ]
+            typer.echo(f"Exporting {to_fmt}")
+            run_in_env(cmd=cmd, env_name=env_name, no_check=True)

{calkit_python-0.23.0 → calkit_python-0.24.0}/calkit/cli/overleaf.py

@@ -141,7 +141,7 @@ def import_publication(
         pdf_path = sync_paths[0].removesuffix(".tex") + ".pdf"
         typer.echo(f"Using PDF path: {pdf_path}")
     tex_path = pdf_path.removesuffix(".pdf") + ".tex"
-    pub_path = os.path.join(dest_dir, pdf_path)
+    pub_path = PurePosixPath(dest_dir, pdf_path).as_posix()
     pub_paths = [pub.get("path") for pub in pubs]
     if not overwrite and pub_path in pub_paths:
         raise_error(

{calkit_python-0.23.0 → calkit_python-0.24.0}/calkit/dvc.py

@@ -155,7 +155,7 @@ def read_pipeline(wdir: str = ".") -> dict:
         return calkit.ryaml.load(f)
 
 
-def get_remotes(wdir: str = None) -> dict[str, str]:
+def get_remotes(wdir: str | None = None) -> dict[str, str]:
     """Get a dictionary of DVC remotes, keyed by name, with URL as the
     value.
     """
@@ -169,13 +169,16 @@ def get_remotes(wdir: str = None) -> dict[str, str]:
     if not out:
         return {}
     resp = {}
-    for line in out.split("\n"):
-        split_line = line.split()
-        if len(split_line) >= 2:
-            # This is a remote
-            name = split_line[0]
-            url = split_line[1]
+    out = out.replace("(default)", "").strip()
+    is_name = True
+    for token in out.split():
+        token = token.strip()
+        if is_name:
+            name = token
+        else:
+            url = token
             resp[name] = url
+        is_name = not is_name
     return resp
 
 

{calkit_python-0.23.0 → calkit_python-0.24.0}/calkit/magics.py

@@ -38,12 +38,11 @@ class Calkit(Magics):
     )
     @magic_arguments.argument(
         "--env",
-        nargs="?",
-        const=True,
-        help=(
-            "Whether or not this cell should be run in an environment. "
-            "If no environment name is provided, the default will be used."
-        ),
+        "--environment",
+        "-e",
+        required=True,
+        help=("Environment with which to run this cell."),
+        type=_parse_string_arg,
     )
     @magic_arguments.argument(
         "--dep",
@@ -112,6 +111,19 @@ class Calkit(Magics):
         help="Description for Calkit output object.",
         type=_parse_string_arg,
     )
+    @magic_arguments.argument(
+        "--out-storage",
+        choices=["dvc", "git", "none"],
+        help="Which version control system to use to store the output.",
+        default="dvc",
+        type=_parse_string_arg,
+    )
+    @magic_arguments.argument(
+        "--verbose",
+        help="Print verbose output.",
+        default=False,
+        action="store_true",
+    )
     @cell_magic
     def stage(self, line, cell):
         """Turn a notebook cell into a DVC pipeline stage.
@@ -222,21 +234,22 @@ class Calkit(Magics):
         with open(script_fpath, "w") as f:
             f.write(script_txt)
         # Create a DVC stage that runs the script, defining the appropriate
-        # dependencies and outputs, and run it
+        # dependencies and outputs
+        # TODO: Insert this into dvc.yaml directly, since DVC reformats
         cmd = [
             sys.executable,
             "-m",
             "dvc",
             "stage",
             "add",
-            "-q",
             "-n",
             args.name,
-            "--run",
             "--force",
             "-d",
             _posix_path(script_fpath),
         ]
+        if not args.verbose:
+            cmd.append("-q")
         if args.dep:
             for dep in args.dep:
                 dep_split = dep.split(":")
@@ -259,8 +272,13 @@ class Calkit(Magics):
                 kws = dict(stage_name=args.name, out_name=out_name)
                 if len(out_split) > 1:
                     kws["fmt"] = out_split[1]
+                # Figure out DVC caching
+                if args.out_storage == "dvc":
+                    out_flag = "-o"
+                else:
+                    out_flag = "--outs-no-cache"
                 cmd += [
-                    "-o",
+                    out_flag,
                     _posix_path(calkit.get_notebook_stage_out_path(**kws)),
                 ]
         if args.out_path:
@@ -274,7 +292,18 @@ class Calkit(Magics):
             stage_cmd = xenv + " -- " + stage_cmd
         cmd.append(stage_cmd)
         try:
-            subprocess.run(cmd, check=True, capture_output=True, text=True)
+            subprocess.run(
+                cmd, check=True, capture_output=not args.verbose, text=True
+            )
+        except subprocess.CalledProcessError as e:
+            print(f"Error: {e.stderr}")
+            raise e
+        # Now run the stage
+        run_cmd = [sys.executable, "-m", "dvc", "repro", args.name]
+        try:
+            subprocess.run(
+                run_cmd, check=True, capture_output=False, text=True
+            )
         except subprocess.CalledProcessError as e:
             print(f"Error: {e.stderr}")
             raise e

{calkit_python-0.23.0 → calkit_python-0.24.0}/calkit/models/pipeline.py

@@ -13,6 +13,10 @@ from calkit.models.iteration import (
     ParametersType,
     RangeIteration,
 )
+from calkit.notebooks import (
+    get_cleaned_notebook_path,
+    get_executed_notebook_path,
+)
 
 
 class Input(BaseModel):
@@ -156,7 +160,7 @@ class Stage(BaseModel):
 
 
 class PythonScriptStage(Stage):
-    kind: Literal["python-script"]
+    kind: Literal["python-script"] = "python-script"
     script_path: str
     args: list[str] = []
 
@@ -292,6 +296,9 @@ class JupyterNotebookStage(Stage):
     2. Notebook running, depending on the cleaned notebook, and optionally
        producing HTML output.
 
+    Alternatively, we could force the use of ``nbstripout`` so the cleaned
+    notebook is saved at the notebook path.
+
     TODO: Can/should we do something like Papermill and let users modify
     parameters in the notebook?
 
@@ -299,15 +306,115 @@ class JupyterNotebookStage(Stage):
     needing to be run from top to bottom every time they change.
     """
 
-    kind: Literal["jupyter-notebook"]
+    kind: Literal["jupyter-notebook"] = "jupyter-notebook"
     notebook_path: str
-    store_cleaned_with: Literal["git", "dvc"] | None = "git"
-    store_executed_ipynb_with: Literal["git", "dvc"] | None = "dvc"
-    store_executed_html_with: Literal["git", "dvc"] | None = "dvc"
+    cleaned_ipynb_storage: Literal["git", "dvc"] | None = "git"
+    executed_ipynb_storage: Literal["git", "dvc"] | None = "dvc"
+    html_storage: Literal["git", "dvc"] | None = "dvc"
+
+    @property
+    def cleaned_notebook_path(self) -> str:
+        return get_cleaned_notebook_path(self.notebook_path, as_posix=True)
+
+    @property
+    def executed_notebook_path(self) -> str:
+        return get_executed_notebook_path(
+            self.notebook_path, to="notebook", as_posix=True
+        )
+
+    @property
+    def html_path(self) -> str:
+        return get_executed_notebook_path(
+            self.notebook_path, to="html", as_posix=True
+        )
 
     @property
     def dvc_deps(self) -> list[str]:
-        return [self.notebook_path] + super().dvc_deps
+        return [self.cleaned_notebook_path] + super().dvc_deps
+
+    @property
+    def dvc_cmd(self) -> str:
+        cmd = f"calkit nb execute --environment {self.environment} --no-check"
+        if self.html_storage:
+            cmd += " --to html"
+        cmd += f' "{self.notebook_path}"'
+        return cmd
+
+    @property
+    def dvc_outs(self) -> list[str | dict]:
+        outs = super().dvc_outs
+        # TODO: This should also export HTML?
+        exec_nb_path = self.executed_notebook_path
+        exec_nb_out = {
+            exec_nb_path: {"cache": self.executed_ipynb_storage == "dvc"}
+        }
+        outs = outs + [exec_nb_out]
+        return outs
+
+    @property
+    def dvc_clean_stage(self) -> dict:
+        """Create a DVC stage for notebook cleaning so the cleaned notebook
+        can be used as a DVC dependency.
+
+        TODO: Should we use Jupytext for this so diffs are nice?
+        """
+        clean_nb_path = self.cleaned_notebook_path
+        stage = {
+            "cmd": f'calkit nb clean "{self.notebook_path}"',
+            "deps": [self.notebook_path],
+            "outs": [
+                {clean_nb_path: {"cache": self.cleaned_ipynb_storage == "dvc"}}
+            ],
+        }
+        return stage
+
+    @property
+    def notebook_outputs(self) -> list[PathOutput]:
+        """Return a list of special notebook outputs so their storage can be
+        respected.
+        """
+        return [
+            PathOutput(
+                path=self.cleaned_notebook_path,
+                storage=self.cleaned_ipynb_storage,
+            ),
+            PathOutput(
+                path=self.executed_notebook_path,
+                storage=self.executed_ipynb_storage,
+            ),
+            PathOutput(path=self.html_path, storage=self.html_storage),
+        ]
+
+
+class WordToPdfStage(Stage):
+    kind: Literal["word-to-pdf"] = "word-to-pdf"
+    word_doc_path: str
+    environment: str = "_system"
+
+    @property
+    def dvc_deps(self) -> list[str]:
+        return [self.word_doc_path] + super().dvc_deps
+
+    @property
+    def out_path(self) -> str:
+        return PurePosixPath(
+            self.word_doc_path.removesuffix(".docx") + ".pdf"
+        ).as_posix()
+
+    @property
+    def dvc_outs(self) -> list[str | dict]:
+        outs = super().dvc_outs
+        out_path = self.out_path
+        if out_path not in outs:
+            outs.append(out_path)
+        return outs
+
+    @property
+    def dvc_cmd(self) -> str:
+        return (
+            f'calkit office word-to-pdf "{self.word_doc_path}" '
+            f'-o "{self.out_path}"'
+        )
 
 
 class Pipeline(BaseModel):
@@ -322,6 +429,8 @@ class Pipeline(BaseModel):
                 | ShellScriptStage
                 | DockerCommandStage
                 | RScriptStage
+                | WordToPdfStage
+                | JupyterNotebookStage
             ),
             Discriminator("kind"),
         ],

calkit_python-0.24.0/calkit/notebooks.py

@@ -0,0 +1,31 @@
+"""Functionality for working with notebooks."""
+
+import os
+from pathlib import PurePosixPath
+from typing import Literal
+
+
+def get_executed_notebook_path(
+    notebook_path: str, to: Literal["html", "notebook"], as_posix: bool = True
+) -> str:
+    """Return the path of an executed notebook."""
+    nb_dir = os.path.dirname(notebook_path)
+    nb_fname = os.path.basename(notebook_path)
+    if to == "html":
+        fname_out = nb_fname.removesuffix(".ipynb") + ".html"
+    else:
+        fname_out = nb_fname
+    # Different output types go to different subdirectories
+    subdirs = {"html": "html", "notebook": "executed"}
+    p = os.path.join(".calkit", "notebooks", subdirs[to], nb_dir, fname_out)
+    if as_posix:
+        p = PurePosixPath(p).as_posix()
+    return p
+
+
+def get_cleaned_notebook_path(path: str, as_posix: bool = True) -> str:
+    """Return the path of a cleaned notebook."""
+    p = os.path.join(".calkit", "notebooks", "cleaned", path)
+    if as_posix:
+        p = PurePosixPath(p).as_posix()
+    return p