calkit-python 0.16.1__tar.gz → 0.16.3__tar.gz

{calkit_python-0.16.1 → calkit_python-0.16.3}/.gitignore

@@ -4,3 +4,4 @@ dev.ipynb
 .vscode
 site
 .env
+.venv

{calkit_python-0.16.1 → calkit_python-0.16.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.16.1
+Version: 0.16.3
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -12,6 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.8
 Requires-Dist: arithmeval
+Requires-Dist: checksumdir
 Requires-Dist: docx2pdf
 Requires-Dist: dvc
 Requires-Dist: eval-type-backport; python_version < '3.10'

{calkit_python-0.16.1 → calkit_python-0.16.3}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.16.1"
+__version__ = "0.16.3"
 
 from .core import *
 from . import git

calkit_python-0.16.3/calkit/cli/check.py

@@ -0,0 +1,196 @@
+"""CLI for checking things."""
+
+from __future__ import annotations
+
+import functools
+import hashlib
+import json
+import os
+import subprocess
+from typing import Annotated
+
+import checksumdir
+import typer
+
+import calkit
+from calkit.check import check_reproducibility
+from calkit.cli import raise_error
+
+check_app = typer.Typer(no_args_is_help=True)
+
+
+@check_app.command(name="repro")
+def check_repro(
+    wdir: Annotated[
+        str, typer.Option("--wdir", help="Project working directory.")
+    ] = ".",
+):
+    """Check the reproducibility of a project."""
+    res = check_reproducibility(wdir=wdir, log_func=typer.echo)
+    typer.echo(res.to_pretty().encode("utf-8", errors="replace"))
+
+
+@check_app.command(name="call")
+def check_call(
+    cmd: Annotated[str, typer.Argument(help="Command to check.")],
+    if_error: Annotated[
+        str,
+        typer.Option(
+            "--if-error", help="Command to run if there is an error."
+        ),
+    ],
+):
+    """Check that a command succeeds and run an alternate if not."""
+    try:
+        subprocess.check_call(cmd, shell=True)
+        typer.echo("Command succeeded")
+    except subprocess.CalledProcessError:
+        typer.echo("Command failed")
+        try:
+            typer.echo("Attempting fallback call")
+            subprocess.check_call(if_error, shell=True)
+            typer.echo("Fallback call succeeded")
+        except subprocess.CalledProcessError:
+            raise_error("Fallback call failed")
+
+
+@check_app.command(
+    name="docker-env",
+    help="Check that Docker image is up-to-date.",
+)
+def check_docker_env(
+    tag: Annotated[str, typer.Argument(help="Image tag.")],
+    fpath: Annotated[
+        str, typer.Option("-i", "--input", help="Path to input Dockerfile.")
+    ] = "Dockerfile",
+    platform: Annotated[
+        str, typer.Option("--platform", help="Which platform(s) to build for.")
+    ] = None,
+    deps: Annotated[
+        list[str],
+        typer.Option(
+            "--dep",
+            "-d",
+            help="Declare an explicit dependency for this Docker image.",
+        ),
+    ] = [],
+    quiet: Annotated[
+        bool, typer.Option("--quiet", "-q", help="Be quiet.")
+    ] = False,
+):
+    def get_docker_inspect():
+        out = json.loads(
+            subprocess.check_output(["docker", "inspect", tag]).decode()
+        )
+        # Remove some keys that can change without the important aspects of
+        # the image changing
+        _ = out[0].pop("Id")
+        _ = out[0].pop("RepoDigests")
+        _ = out[0].pop("Metadata")
+        _ = out[0].pop("DockerVersion")
+        return out
+
+    def get_md5(path: str, exclude_files: list[str] | None = None) -> str:
+        if os.path.isdir(path):
+            return checksumdir.dirhash(dep, excluded_files=exclude_files)
+        else:
+            with open(path) as f:
+                content = f.read()
+            return hashlib.md5(content.encode()).hexdigest()
+
+    outfile = open(os.devnull, "w") if quiet else None
+    typer.echo(f"Checking for existing image with tag {tag}", file=outfile)
+    # First call Docker inspect
+    try:
+        inspect = get_docker_inspect()
+    except subprocess.CalledProcessError:
+        typer.echo(f"No image with tag {tag} found locally", file=outfile)
+        inspect = []
+    typer.echo(f"Reading Dockerfile from {fpath}", file=outfile)
+    dockerfile_md5 = get_md5(fpath)
+    lock_fpath = fpath + "-lock.json"
+    # Compute MD5s of any dependencies
+    deps_md5s = {}
+    for dep in deps:
+        deps_md5s[dep] = get_md5(dep, exclude_files=lock_fpath)
+    rebuild = True
+    if os.path.isfile(lock_fpath):
+        typer.echo(f"Reading lock file: {lock_fpath}", file=outfile)
+        with open(lock_fpath) as f:
+            lock = json.load(f)
+    else:
+        typer.echo(f"Lock file ({lock_fpath}) does not exist", file=outfile)
+        lock = None
+    if inspect and lock:
+        typer.echo(
+            "Checking image and Dockerfile against lock file", file=outfile
+        )
+        rebuild = inspect[0]["RootFS"]["Layers"] != lock[0]["RootFS"][
+            "Layers"
+        ] or dockerfile_md5 != lock[0].get("DockerfileMD5")
+        if not rebuild:
+            for dep, md5 in deps_md5s.items():
+                if md5 != lock[0].get("DepsMD5s", {}).get(dep):
+                    typer.echo(f"Found modified dependency: {dep}")
+                    rebuild = True
+                    break
+    if rebuild:
+        wdir, fname = os.path.split(fpath)
+        if not wdir:
+            wdir = None
+        cmd = ["docker", "build", "-t", tag, "-f", fname]
+        if platform is not None:
+            cmd += ["--platform", platform]
+        cmd.append(".")
+        subprocess.check_output(cmd, cwd=wdir)
+    # Write the lock file
+    inspect = get_docker_inspect()
+    inspect[0]["DockerfileMD5"] = dockerfile_md5
+    inspect[0]["DepsMD5s"] = deps_md5s
+    with open(lock_fpath, "w") as f:
+        json.dump(inspect, f, indent=4)
+
+
+@check_app.command(
+    name="conda-env",
+    help="Check a conda environment and rebuild if necessary.",
+)
+def check_conda_env(
+    env_fpath: Annotated[
+        str,
+        typer.Option(
+            "--file", "-f", help="Path to conda environment YAML file."
+        ),
+    ] = "environment.yml",
+    output_fpath: Annotated[
+        str,
+        typer.Option(
+            "--output",
+            "-o",
+            help=(
+                "Path to which existing environment should be exported. "
+                "If not specified, will have the same filename with '-lock' "
+                "appended to it, keeping the same extension."
+            ),
+        ),
+    ] = None,
+    relaxed: Annotated[
+        bool,
+        typer.Option(
+            "--relaxed", help="Treat conda and pip dependencies as equivalent."
+        ),
+    ] = False,
+    quiet: Annotated[
+        bool, typer.Option("--quiet", "-q", help="Be quiet.")
+    ] = False,
+):
+    if quiet:
+        log_func = functools.partial(typer.echo, file=open(os.devnull, "w"))
+    else:
+        log_func = typer.echo
+    calkit.conda.check_env(
+        env_fpath=env_fpath,
+        output_fpath=output_fpath,
+        log_func=log_func,
+        relaxed=relaxed,
+    )

{calkit_python-0.16.1 → calkit_python-0.16.3}/calkit/cli/main.py

@@ -3,9 +3,6 @@
 from __future__ import annotations
 
 import csv
-import functools
-import hashlib
-import json
 import os
 import platform as _platform
 import subprocess
@@ -22,7 +19,7 @@ from typing_extensions import Annotated, Optional
 
 import calkit
 from calkit.cli import print_sep, raise_error, run_cmd, warn
-from calkit.cli.check import check_app
+from calkit.cli.check import check_app, check_conda_env, check_docker_env
 from calkit.cli.config import config_app
 from calkit.cli.import_ import import_app
 from calkit.cli.list import list_app
@@ -60,7 +57,8 @@ def _to_shell_cmd(cmd: list[str]) -> str:
     """
     quoted_cmd = []
     for part in cmd:
-        if " " in part or '"' in part or "'" in part:
+        # Find quotes within quotes and escape them
+        if " " in part or '"' in part[1:-1] or "'" in part[1:-1]:
             part = part.replace('"', r"\"")
             quoted_cmd.append(f'"{part}"')
         else:
@@ -711,12 +709,11 @@ def run_in_env(
     if env_name not in envs:
         raise_error(f"Environment '{env_name}' does not exist")
     env = envs[env_name]
-    if wdir is not None:
-        cwd = os.path.abspath(wdir)
-    else:
-        cwd = os.getcwd()
     image_name = env.get("image", env_name)
     docker_wdir = env.get("wdir", "/work")
+    docker_wdir_mount = docker_wdir
+    if wdir is not None:
+        docker_wdir = os.path.join(docker_wdir, wdir)
     shell = env.get("shell", "sh")
     platform = env.get("platform")
     if env["kind"] == "docker":
@@ -727,6 +724,7 @@ def run_in_env(
                 tag=env["image"],
                 fpath=env["path"],
                 platform=env.get("platform"),
+                deps=env.get("deps", []),
                 quiet=True,
             )
         shell_cmd = _to_shell_cmd(cmd)
@@ -736,13 +734,14 @@ def run_in_env(
         ]
         if platform:
             docker_cmd += ["--platform", platform]
+        docker_cmd += env.get("args", [])
         docker_cmd += [
             "-it" if sys.stdin.isatty() else "-i",
             "--rm",
             "-w",
             docker_wdir,
             "-v",
-            f"{cwd}:{docker_wdir}",
+            f"{os.getcwd()}:{docker_wdir_mount}",
             image_name,
             shell,
             "-c",
@@ -846,78 +845,6 @@ def run_in_env(
         raise_error("Environment kind not supported")
 
 
-@app.command(
-    name="build-docker",
-    help="Build Docker image if missing or different from lock file.",
-)
-def check_docker_env(
-    tag: Annotated[str, typer.Argument(help="Image tag.")],
-    fpath: Annotated[
-        str, typer.Option("-i", "--input", help="Path to input Dockerfile.")
-    ] = "Dockerfile",
-    platform: Annotated[
-        str, typer.Option("--platform", help="Which platform(s) to build for.")
-    ] = None,
-    quiet: Annotated[
-        bool, typer.Option("--quiet", "-q", help="Be quiet.")
-    ] = False,
-):
-    def get_docker_inspect():
-        out = json.loads(
-            subprocess.check_output(["docker", "inspect", tag]).decode()
-        )
-        # Remove some keys that can change without the important aspects of
-        # the image changing
-        _ = out[0].pop("Id")
-        _ = out[0].pop("RepoDigests")
-        _ = out[0].pop("Metadata")
-        _ = out[0].pop("DockerVersion")
-        return out
-
-    outfile = open(os.devnull, "w") if quiet else None
-    typer.echo(f"Checking for existing image with tag {tag}", file=outfile)
-    # First call Docker inspect
-    try:
-        inspect = get_docker_inspect()
-    except subprocess.CalledProcessError:
-        typer.echo(f"No image with tag {tag} found locally", file=outfile)
-        inspect = []
-    typer.echo(f"Reading Dockerfile from {fpath}", file=outfile)
-    with open(fpath) as f:
-        dockerfile = f.read()
-    dockerfile_md5 = hashlib.md5(dockerfile.encode()).hexdigest()
-    lock_fpath = fpath + "-lock.json"
-    rebuild = True
-    if os.path.isfile(lock_fpath):
-        typer.echo(f"Reading lock file: {lock_fpath}", file=outfile)
-        with open(lock_fpath) as f:
-            lock = json.load(f)
-    else:
-        typer.echo(f"Lock file ({lock_fpath}) does not exist", file=outfile)
-        lock = None
-    if inspect and lock:
-        typer.echo(
-            "Checking image and Dockerfile against lock file", file=outfile
-        )
-        rebuild = inspect[0]["RootFS"]["Layers"] != lock[0]["RootFS"][
-            "Layers"
-        ] or dockerfile_md5 != lock[0].get("DockerfileMD5")
-    if rebuild:
-        wdir, fname = os.path.split(fpath)
-        if not wdir:
-            wdir = None
-        cmd = ["docker", "build", "-t", tag, "-f", fname]
-        if platform is not None:
-            cmd += ["--platform", platform]
-        cmd.append(".")
-        subprocess.check_call(cmd, cwd=wdir)
-    # Write the lock file
-    inspect = get_docker_inspect()
-    inspect[0]["DockerfileMD5"] = dockerfile_md5
-    with open(lock_fpath, "w") as f:
-        json.dump(inspect, f, indent=4)
-
-
 @app.command(name="runproc", help="Execute a procedure (alias for 'xproc').")
 @app.command(name="xproc", help="Execute a procedure.")
 def run_procedure(
@@ -1059,51 +986,6 @@ def run_procedure(
             wait(step.wait_after_s)
 
 
-@app.command(
-    name="check-conda-env",
-    help="Check a conda environment and rebuild if necessary.",
-)
-def check_conda_env(
-    env_fpath: Annotated[
-        str,
-        typer.Option(
-            "--file", "-f", help="Path to conda environment YAML file."
-        ),
-    ] = "environment.yml",
-    output_fpath: Annotated[
-        str,
-        typer.Option(
-            "--output",
-            "-o",
-            help=(
-                "Path to which existing environment should be exported. "
-                "If not specified, will have the same filename with '-lock' "
-                "appended to it, keeping the same extension."
-            ),
-        ),
-    ] = None,
-    relaxed: Annotated[
-        bool,
-        typer.Option(
-            "--relaxed", help="Treat conda and pip dependencies as equivalent."
-        ),
-    ] = False,
-    quiet: Annotated[
-        bool, typer.Option("--quiet", "-q", help="Be quiet.")
-    ] = False,
-):
-    if quiet:
-        log_func = functools.partial(typer.echo, file=open(os.devnull, "w"))
-    else:
-        log_func = typer.echo
-    calkit.conda.check_env(
-        env_fpath=env_fpath,
-        output_fpath=output_fpath,
-        log_func=log_func,
-        relaxed=relaxed,
-    )
-
-
 @app.command(name="calc")
 def run_calculation(
     name: Annotated[str, typer.Argument(help="Calculation name.")],

{calkit_python-0.16.1 → calkit_python-0.16.3}/calkit/cli/new.py

@@ -469,10 +469,6 @@ def new_docker_env(
             ),
         ),
     ] = None,
-    stage: Annotated[
-        str,
-        typer.Option("--stage", help="DVC pipeline stage name, deprecated."),
-    ] = None,
     layers: Annotated[
         list[str],
         typer.Option(
@@ -505,8 +501,6 @@ def new_docker_env(
         path = "Dockerfile"
     if base and os.path.isfile(path) and not overwrite:
         raise_error("Output path already exists (use -f to overwrite)")
-    if stage and not base:
-        raise_error("--from must be specified when creating a build stage")
     if image_name is None:
         typer.echo("No image name specified; using environment name")
         image_name = name
@@ -543,8 +537,6 @@ def new_docker_env(
     )
     if base is not None or path is not None:
         env["path"] = path
-    if stage is not None:
-        env["stage"] = stage
     if description is not None:
         env["description"] = description
     if layers:
@@ -555,35 +547,7 @@ def new_docker_env(
     ck_info["environments"] = envs
     with open("calkit.yaml", "w") as f:
         ryaml.dump(ck_info, f)
-    # If we're creating a stage, do so with DVC
-    if stage:
-        warn("--stage is deprecated since envs are checked at run time")
-        typer.echo(f"Creating DVC stage {stage}")
-        if not os.path.isfile(".dvc/config"):
-            typer.echo(f"Running dvc init")
-            subprocess.check_call(["dvc", "init"])
-        ck_cmd = f"calkit build-docker {image_name} -i {path}"
-        if platform:
-            ck_cmd += f" --platform {platform}"
-        subprocess.check_call(
-            [
-                "dvc",
-                "stage",
-                "add",
-                "-f",
-                "-n",
-                stage,
-                "--always-changed",
-                "-d",
-                path,
-                "--outs-persist-no-cache",
-                f"{path}-lock.json",
-                ck_cmd,
-            ]
-        )
     repo.git.add("calkit.yaml")
-    if stage:
-        repo.git.add("dvc.yaml")
     if not no_commit and repo.git.diff("--staged"):
         repo.git.commit(["-m", f"Add Docker environment {name}"])
 
@@ -1016,7 +980,7 @@ def new_conda_env(
     if stage:
         typer.echo(f"Creating DVC stage {stage}")
         if not os.path.isfile(".dvc/config"):
-            typer.echo(f"Running dvc init")
+            typer.echo("Running dvc init")
             subprocess.check_call(["dvc", "init"])
         ck_cmd = f"calkit check-conda-env -f {path}"
         fname, ext = os.path.splitext(path)

{calkit_python-0.16.1 → calkit_python-0.16.3}/calkit/conda.py

@@ -4,9 +4,10 @@ import json
 import os
 import re
 import subprocess
+import warnings
 
 from packaging.specifiers import SpecifierSet
-from packaging.version import Version
+from packaging.version import InvalidVersion, Version
 from pydantic import BaseModel
 
 import calkit
@@ -20,14 +21,22 @@ def _check_single(req: str, actual: str, conda: bool = False) -> bool:
     if conda and req_spec.startswith("="):
         req_spec = "=" + req_spec
         if not req_spec.endswith(".*"):
-            req_spec += ".*"
+            if len(req_spec.split(".")) < 3:
+                req_spec += ".*"
     actual_name, actual_vers = re.split("[=<>]+", actual, maxsplit=1)
     if actual_name != req_name:
         return False
     actual_spec = actual.removeprefix(actual_name)
     if conda and actual_spec.startswith("="):
         actual_spec = "=" + actual_spec
-    version = Version(actual_vers)
+    try:
+        version = Version(actual_vers)
+    except InvalidVersion:
+        warnings.warn(
+            f"Cannot properly check {actual_name} version {actual_vers}"
+        )
+        # TODO: Check exact version only
+        return True
     spec = SpecifierSet(req_spec)
     return spec.contains(version)
 

{calkit_python-0.16.1 → calkit_python-0.16.3}/calkit/tests/cli/test_main.py

@@ -11,7 +11,6 @@ from git.exc import InvalidGitRepositoryError
 
 import calkit
 from calkit.cli.main import _to_shell_cmd
-from calkit.core import ryaml
 
 
 def test_run_in_env(tmp_dir):
@@ -21,13 +20,11 @@ def test_run_in_env(tmp_dir):
     subprocess.check_call(
         "calkit new docker-env "
         "--name my-image "
-        "--stage build-image "
         "--from ubuntu "
         "--add-layer miniforge "
         "--description 'This is a test image'",
         shell=True,
     )
-    subprocess.check_call("calkit run", shell=True)
     out = (
         subprocess.check_output("calkit xenv echo sup", shell=True)
         .decode()
@@ -40,7 +37,6 @@ def test_run_in_env(tmp_dir):
         "calkit new docker-env "
         "-n env2 "
         "--image my-image-2 "
-        "--stage build-image-2 "
         "--path Dockerfile.2 "
         "--from ubuntu "
         "--add-layer miniforge "
@@ -48,12 +44,6 @@ def test_run_in_env(tmp_dir):
         "--description 'This is a test image 2'",
         shell=True,
     )
-    with open("dvc.yaml") as f:
-        pipeline = ryaml.load(f)
-    stg = pipeline["stages"]["build-image-2"]
-    cmd = stg["cmd"]
-    assert "-i Dockerfile.2" in cmd
-    subprocess.check_call("calkit run", shell=True)
     with pytest.raises(subprocess.CalledProcessError):
         out = (
             subprocess.check_output("calkit xenv echo sup", shell=True)
@@ -95,6 +85,26 @@ def test_run_in_env(tmp_dir):
     ck_info = calkit.load_calkit_info()
     env = ck_info["environments"]["py3.10"]
     assert env.get("path") is None
+    # Test that we can run a command that changes directory first
+    os.makedirs("my-new-dir/another", exist_ok=True)
+    out = (
+        subprocess.check_output(
+            "calkit xenv -n py3.10 --wdir my-new-dir -- ls",
+            shell=True,
+        )
+        .decode()
+        .strip()
+    )
+    assert out == "another"
+    out = (
+        subprocess.check_output(
+            "calkit xenv -n py3.10 --wdir my-new-dir -- ls ..",
+            shell=True,
+        )
+        .decode()
+        .strip()
+    )
+    assert "my-new-dir" in out.split("\n")
 
 
 def test_run_in_venv(tmp_dir):
@@ -178,7 +188,7 @@ def test_run_in_venv(tmp_dir):
     )
     ck_info = calkit.load_calkit_info(as_pydantic=True)
     envs = ck_info.environments
-    env = envs["my-pixi"]
+    assert "my-pixi" in envs
     out = (
         subprocess.check_output(
             [
@@ -216,6 +226,13 @@ def test_to_shell_cmd():
     shell_cmd = _to_shell_cmd(cmd)
     assert shell_cmd == 'python -c "print(\\"hello world\\")"'
     subprocess.check_call(shell_cmd, shell=True)
+    cmd = [
+        "sh",
+        "-c",
+        "cd dir1 && ls",
+    ]
+    good_shell_cmd = 'sh -c "cd dir1 && ls"'
+    assert _to_shell_cmd(cmd) == good_shell_cmd
 
 
 def test_add(tmp_dir):

{calkit_python-0.16.1 → calkit_python-0.16.3}/docs/environments.md

@@ -154,6 +154,21 @@ and another call to `calkit xenv -n foam2` will kick off a rebuild
 automatically,
 since the lock file will no longer match the Dockerfile.
 
+If you're copying local files into the Docker image,
+you can declare these
+dependencies in the environment definition so the content of those will be
+tracked as well:
+
+```yaml
+# In calkit.yaml
+environments:
+  foam2:
+    kind: docker
+    image: foam2
+    deps:
+      - src/mySolver.C
+```
+
 This highlights Calkit's declarative design philosophy.
 Simply declare the environment and use it in a pipeline stage
 and Calkit will ensure it is built and up to date.