calkit-python 0.16.2__tar.gz → 0.17.0__tar.gz

{calkit_python-0.16.2 → calkit_python-0.17.0}/.gitignore

@@ -4,3 +4,4 @@ dev.ipynb
 .vscode
 site
 .env
+.venv

{calkit_python-0.16.2 → calkit_python-0.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.16.2
+Version: 0.17.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -12,6 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.8
 Requires-Dist: arithmeval
+Requires-Dist: checksumdir
 Requires-Dist: docx2pdf
 Requires-Dist: dvc
 Requires-Dist: eval-type-backport; python_version < '3.10'

{calkit_python-0.16.2 → calkit_python-0.17.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.16.2"
+__version__ = "0.17.0"
 
 from .core import *
 from . import git

{calkit_python-0.16.2 → calkit_python-0.17.0}/calkit/cli/check.py

@@ -9,6 +9,7 @@ import os
 import subprocess
 from typing import Annotated
 
+import checksumdir
 import typer
 
 import calkit
@@ -65,6 +66,14 @@ def check_docker_env(
     platform: Annotated[
         str, typer.Option("--platform", help="Which platform(s) to build for.")
     ] = None,
+    deps: Annotated[
+        list[str],
+        typer.Option(
+            "--dep",
+            "-d",
+            help="Declare an explicit dependency for this Docker image.",
+        ),
+    ] = [],
     quiet: Annotated[
         bool, typer.Option("--quiet", "-q", help="Be quiet.")
     ] = False,
@@ -81,6 +90,14 @@ def check_docker_env(
         _ = out[0].pop("DockerVersion")
         return out
 
+    def get_md5(path: str, exclude_files: list[str] | None = None) -> str:
+        if os.path.isdir(path):
+            return checksumdir.dirhash(dep, excluded_files=exclude_files)
+        else:
+            with open(path) as f:
+                content = f.read()
+            return hashlib.md5(content.encode()).hexdigest()
+
     outfile = open(os.devnull, "w") if quiet else None
     typer.echo(f"Checking for existing image with tag {tag}", file=outfile)
     # First call Docker inspect
@@ -90,10 +107,12 @@ def check_docker_env(
         typer.echo(f"No image with tag {tag} found locally", file=outfile)
         inspect = []
     typer.echo(f"Reading Dockerfile from {fpath}", file=outfile)
-    with open(fpath) as f:
-        dockerfile = f.read()
-    dockerfile_md5 = hashlib.md5(dockerfile.encode()).hexdigest()
+    dockerfile_md5 = get_md5(fpath)
     lock_fpath = fpath + "-lock.json"
+    # Compute MD5s of any dependencies
+    deps_md5s = {}
+    for dep in deps:
+        deps_md5s[dep] = get_md5(dep, exclude_files=lock_fpath)
     rebuild = True
     if os.path.isfile(lock_fpath):
         typer.echo(f"Reading lock file: {lock_fpath}", file=outfile)
@@ -109,6 +128,12 @@ def check_docker_env(
         rebuild = inspect[0]["RootFS"]["Layers"] != lock[0]["RootFS"][
             "Layers"
         ] or dockerfile_md5 != lock[0].get("DockerfileMD5")
+        if not rebuild:
+            for dep, md5 in deps_md5s.items():
+                if md5 != lock[0].get("DepsMD5s", {}).get(dep):
+                    typer.echo(f"Found modified dependency: {dep}")
+                    rebuild = True
+                    break
     if rebuild:
         wdir, fname = os.path.split(fpath)
         if not wdir:
@@ -117,10 +142,11 @@ def check_docker_env(
         if platform is not None:
             cmd += ["--platform", platform]
         cmd.append(".")
-        subprocess.check_call(cmd, cwd=wdir)
+        subprocess.check_output(cmd, cwd=wdir)
     # Write the lock file
     inspect = get_docker_inspect()
     inspect[0]["DockerfileMD5"] = dockerfile_md5
+    inspect[0]["DepsMD5s"] = deps_md5s
     with open(lock_fpath, "w") as f:
         json.dump(inspect, f, indent=4)
 

{calkit_python-0.16.2 → calkit_python-0.17.0}/calkit/cli/main.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import csv
+import glob
 import os
 import platform as _platform
 import subprocess
@@ -57,7 +58,8 @@ def _to_shell_cmd(cmd: list[str]) -> str:
     """
     quoted_cmd = []
     for part in cmd:
-        if " " in part or '"' in part or "'" in part:
+        # Find quotes within quotes and escape them
+        if " " in part or '"' in part[1:-1] or "'" in part[1:-1]:
             part = part.replace('"', r"\"")
             quoted_cmd.append(f'"{part}"')
         else:
@@ -708,12 +710,11 @@ def run_in_env(
     if env_name not in envs:
         raise_error(f"Environment '{env_name}' does not exist")
     env = envs[env_name]
-    if wdir is not None:
-        cwd = os.path.abspath(wdir)
-    else:
-        cwd = os.getcwd()
     image_name = env.get("image", env_name)
     docker_wdir = env.get("wdir", "/work")
+    docker_wdir_mount = docker_wdir
+    if wdir is not None:
+        docker_wdir = os.path.join(docker_wdir, wdir)
     shell = env.get("shell", "sh")
     platform = env.get("platform")
     if env["kind"] == "docker":
@@ -724,6 +725,7 @@ def run_in_env(
                 tag=env["image"],
                 fpath=env["path"],
                 platform=env.get("platform"),
+                deps=env.get("deps", []),
                 quiet=True,
             )
         shell_cmd = _to_shell_cmd(cmd)
@@ -733,13 +735,14 @@ def run_in_env(
         ]
         if platform:
             docker_cmd += ["--platform", platform]
+        docker_cmd += env.get("args", [])
         docker_cmd += [
             "-it" if sys.stdin.isatty() else "-i",
             "--rm",
             "-w",
             docker_wdir,
             "-v",
-            f"{cwd}:{docker_wdir}",
+            f"{os.getcwd()}:{docker_wdir_mount}",
             image_name,
             shell,
             "-c",
@@ -839,6 +842,117 @@ def run_in_env(
             subprocess.check_call(cmd, shell=True, cwd=wdir)
         except subprocess.CalledProcessError:
             raise_error(f"Failed to run in {kind}")
+    elif env["kind"] == "ssh":
+        try:
+            host = os.path.expandvars(env["host"])
+            user = os.path.expandvars(env["user"])
+            remote_wdir = env["wdir"]
+        except KeyError:
+            raise_error(
+                "Host, user, and wdir must be defined for ssh environments"
+            )
+        send_paths = env.get("send_paths")
+        get_paths = env.get("get_paths")
+        key = env.get("key")
+        if key is not None:
+            key = os.path.expanduser(os.path.expandvars(key))
+        remote_shell_cmd = _to_shell_cmd(cmd)
+        # Run with nohup so we can disconnect
+        # TODO: Should we collect output instead of send to /dev/null?
+        remote_cmd = (
+            f"cd '{remote_wdir}' ; nohup {remote_shell_cmd} "
+            "> /dev/null 2>&1 & echo $! "
+        )
+        key_cmd = ["-i", key] if key is not None else []
+        # Check to see if we've already submitted a job with this command
+        jobs_fpath = ".calkit/jobs.yaml"
+        job_key = f"{env_name}::{remote_shell_cmd}"
+        remote_pid = None
+        if os.path.isfile(jobs_fpath):
+            with open(jobs_fpath) as f:
+                jobs = calkit.ryaml.load(f)
+            if jobs is None:
+                jobs = {}
+        else:
+            jobs = {}
+        job = jobs.get(job_key, {})
+        remote_pid = job.get("remote_pid")
+        if remote_pid is None:
+            # First make sure the remote working dir exists
+            typer.echo("Ensuring remote working directory exists")
+            subprocess.check_call(
+                ["ssh"]
+                + key_cmd
+                + [f"{user}@{host}", f"mkdir -p {remote_wdir}"]
+            )
+            # Now send any necessary files
+            if send_paths:
+                typer.echo("Sending to remote directory")
+                # Accept glob patterns
+                paths = []
+                for p in send_paths:
+                    paths += glob.glob(p)
+                scp_cmd = (
+                    ["scp", "-r"]
+                    + key_cmd
+                    + paths
+                    + [f"{user}@{host}:{remote_wdir}/"]
+                )
+                if verbose:
+                    typer.echo(f"scp cmd: {scp_cmd}")
+                subprocess.check_call(scp_cmd)
+            # Now run the command
+            typer.echo(f"Running remote command: {remote_shell_cmd}")
+            if verbose:
+                typer.echo(f"Full command: {remote_cmd}")
+            remote_pid = (
+                subprocess.check_output(
+                    ["ssh"] + key_cmd + [f"{user}@{host}", remote_cmd]
+                )
+                .decode()
+                .strip()
+            )
+            typer.echo(f"Running with remote PID: {remote_pid}")
+            # Save PID to jobs database so we can resume waiting
+            typer.echo("Updating jobs database")
+            os.makedirs(".calkit", exist_ok=True)
+            job["remote_pid"] = remote_pid
+            job["submitted"] = time.time()
+            job["finished"] = None
+            jobs[job_key] = job
+            with open(jobs_fpath, "w") as f:
+                calkit.ryaml.dump(jobs, f)
+        # Now wait for the job to complete
+        typer.echo(f"Waiting for remote PID {remote_pid} to finish")
+        ps_cmd = ["ssh"] + key_cmd + [f"{user}@{host}", "ps", "-p", remote_pid]
+        finished = False
+        while not finished:
+            try:
+                subprocess.check_output(ps_cmd)
+                finished = False
+                time.sleep(2)
+            except subprocess.CalledProcessError:
+                finished = True
+                typer.echo("Remote process finished")
+        # Now sync the files back
+        # TODO: Figure out how to do this in one command
+        # Getting the syntax right is troublesome since it appears to work
+        # differently on different platforms
+        if get_paths:
+            typer.echo("Copying files back from remote directory")
+            for src_path in get_paths:
+                src_path = remote_wdir + "/" + src_path
+                src = f"{user}@{host}:{src_path}"
+                scp_cmd = ["scp", "-r"] + key_cmd + [src, "."]
+                subprocess.check_call(scp_cmd)
+        # Now delete the remote PID from the jobs file
+        typer.echo("Updating jobs database")
+        os.makedirs(".calkit", exist_ok=True)
+        job["remote_pid"] = None
+        job["finished"] = time.time()
+        jobs[job_key] = job
+        with open(jobs_fpath, "w") as f:
+            calkit.ryaml.dump(jobs, f)
     else:
         raise_error("Environment kind not supported")
 
@@ -1041,9 +1155,17 @@ def set_env_var(
 @app.command(name="upgrade")
 def upgrade():
     """Upgrade Calkit."""
-    # See if uv is installed first
-    if calkit.check_dep_exists("uv"):
-        cmd = ["uv", "pip", "install", "--system"]
+    if calkit.check_dep_exists("pipx"):
+        cmd = ["pipx", "upgrade", "calkit-python"]
+    elif calkit.check_dep_exists("uv"):
+        cmd = [
+            "uv",
+            "pip",
+            "install",
+            "--system",
+            "--upgrade",
+            "calkit-python",
+        ]
     else:
-        cmd = ["pip", "install"]
-    subprocess.run(cmd + ["--upgrade", "calkit-python"])
+        cmd = ["pip", "install", "--upgrade", "calkit-python"]
+    subprocess.run(cmd)

{calkit_python-0.16.2 → calkit_python-0.17.0}/calkit/conda.py

@@ -4,9 +4,10 @@ import json
 import os
 import re
 import subprocess
+import warnings
 
 from packaging.specifiers import SpecifierSet
-from packaging.version import Version
+from packaging.version import InvalidVersion, Version
 from pydantic import BaseModel
 
 import calkit
@@ -20,14 +21,22 @@ def _check_single(req: str, actual: str, conda: bool = False) -> bool:
     if conda and req_spec.startswith("="):
         req_spec = "=" + req_spec
         if not req_spec.endswith(".*"):
-            req_spec += ".*"
+            if len(req_spec.split(".")) < 3:
+                req_spec += ".*"
     actual_name, actual_vers = re.split("[=<>]+", actual, maxsplit=1)
     if actual_name != req_name:
         return False
     actual_spec = actual.removeprefix(actual_name)
     if conda and actual_spec.startswith("="):
         actual_spec = "=" + actual_spec
-    version = Version(actual_vers)
+    try:
+        version = Version(actual_vers)
+    except InvalidVersion:
+        warnings.warn(
+            f"Cannot properly check {actual_name} version {actual_vers}"
+        )
+        # TODO: Check exact version only
+        return True
     spec = SpecifierSet(req_spec)
     return spec.contains(version)
 

{calkit_python-0.16.2 → calkit_python-0.17.0}/calkit/models.py

@@ -109,6 +109,16 @@ class REnvironment(Environment):
     prefix: str
 
 
+class SSHEnvironment(BaseModel):
+    kind: Literal["ssh"]
+    host: str
+    user: str
+    wdir: str
+    key: str | None = None
+    send_paths: list[str] = ["./*"]
+    get_paths: list[str] = ["*"]
+
+
 class Software(BaseModel):
     title: str
     path: str
@@ -248,7 +258,11 @@ class ProjectInfo(BaseModel):
     references: list[ReferenceCollection] = []
     environments: dict[
         str,
-        Environment | DockerEnvironment | VenvEnvironment | UvVenvEnvironment,
+        Environment
+        | DockerEnvironment
+        | VenvEnvironment
+        | UvVenvEnvironment
+        | SSHEnvironment,
     ] = {}
     software: list[Software] = []
     notebooks: list[Notebook] = []

{calkit_python-0.16.2 → calkit_python-0.17.0}/calkit/tests/cli/test_main.py

@@ -85,6 +85,26 @@ def test_run_in_env(tmp_dir):
     ck_info = calkit.load_calkit_info()
     env = ck_info["environments"]["py3.10"]
     assert env.get("path") is None
+    # Test that we can run a command that changes directory first
+    os.makedirs("my-new-dir/another", exist_ok=True)
+    out = (
+        subprocess.check_output(
+            "calkit xenv -n py3.10 --wdir my-new-dir -- ls",
+            shell=True,
+        )
+        .decode()
+        .strip()
+    )
+    assert out == "another"
+    out = (
+        subprocess.check_output(
+            "calkit xenv -n py3.10 --wdir my-new-dir -- ls ..",
+            shell=True,
+        )
+        .decode()
+        .strip()
+    )
+    assert "my-new-dir" in out.split("\n")
 
 
 def test_run_in_venv(tmp_dir):
@@ -206,6 +226,13 @@ def test_to_shell_cmd():
     shell_cmd = _to_shell_cmd(cmd)
     assert shell_cmd == 'python -c "print(\\"hello world\\")"'
     subprocess.check_call(shell_cmd, shell=True)
+    cmd = [
+        "sh",
+        "-c",
+        "cd dir1 && ls",
+    ]
+    good_shell_cmd = 'sh -c "cd dir1 && ls"'
+    assert _to_shell_cmd(cmd) == good_shell_cmd
 
 
 def test_add(tmp_dir):

{calkit_python-0.16.2 → calkit_python-0.17.0}/docs/environments.md

@@ -23,6 +23,7 @@ Calkit supports defining and running code in these environment types:
 - [`uv`](https://docs.astral.sh/uv/) (both `venv` and project-based)
 - [Pixi](https://github.com/prefix-dev/pixi)
 - [`renv`](https://rstudio.github.io/renv/index.html)
+- `ssh`
 
 Environment definitions live in the project's `calkit.yaml` file
 in the `environments` section.
@@ -154,6 +155,21 @@ and another call to `calkit xenv -n foam2` will kick off a rebuild
 automatically,
 since the lock file will no longer match the Dockerfile.
 
+If you're copying local files into the Docker image,
+you can declare these
+dependencies in the environment definition so the content of those will be
+tracked as well:
+
+```yaml
+# In calkit.yaml
+environments:
+  foam2:
+    kind: docker
+    image: foam2
+    deps:
+      - src/mySolver.C
+```
+
 This highlights Calkit's declarative design philosophy.
 Simply declare the environment and use it in a pipeline stage
 and Calkit will ensure it is built and up to date.
@@ -238,3 +254,60 @@ and an updated `environment-lock.yml` file will be created.
 Again this highlights Calkit's declarative design philosophy.
 Declare the environment and what command should be executed inside,
 and Calkit will handle the rest.
+
+### SSH
+
+It's possible to define a remote environment that uses `ssh` to connect
+and run commands,
+and `scp` to copy files back and forth.
+This could be useful, e.g.,
+for running one or more pipeline stages on a high performance computing (HPC)
+cluster,
+or simply offloading some work to a virtual machine in the cloud
+with specialized hardware like a more powerful GPU.
+
+It is assumed that dependencies on the remote machine are managed separately.
+
+An SSH environment defined in `calkit.yaml` looks like:
+
+```yaml
+environments:
+  cluster:
+    kind: ssh
+    host: "10.225.22.25"
+    user: my-user-name
+    wdir: /home/my-user-name/calkit/example-ssh
+    key: ~/.ssh/id_ed25519
+    send_paths:
+      - script.sh
+    get_paths:
+      - results
+```
+
+In the example above, we define an environment called `cluster`,
+where we specify the host IP address, our username on that machine,
+the working directory, the path to an SSH key on our local machine
+(so we can connect without a password),
+which paths we want to send before executing commands,
+and which we want to copy back after they finish.
+Wildcards in paths are supported, so the entire directory could be copied
+if desired by specifying `*`.
+
+To register an SSH key with the host, use `ssh-copy-id`. For example:
+
+```sh
+ssh-copy-id -i ~/.ssh/id_ed25519 my-user-name@10.225.22.25
+```
+
+To execute a command in this environment, we can add a stage like this
+to our DVC pipeline in `dvc.yaml`:
+
+```yaml
+stages:
+  run-simulation:
+    cmd: calkit xenv -n cluster bash script.sh
+    deps:
+      - script.sh
+    outs:
+      - results
+```

{calkit_python-0.16.2 → calkit_python-0.17.0}/docs/examples.md

@@ -52,6 +52,7 @@ Features:
 - Environmental variable dependencies
 - A pipeline designed to be run periodically to accumulate new data
 - A project showcase with interactive Plotly figures
+- A uv project-based environment and dedicated Python package
 
 ## OpenFOAM RANS boundary later validation
 
@@ -64,3 +65,13 @@ Features:
 - A LaTeX document built with a Docker container
 - A direct numerical simulation dataset for validation imported from a
   different project, derived from the Johns Hopkins Turbulence Database
+
+## SSH
+
+[Project page](https://calkit.io/calkit/example-ssh) |
+[GitHub repo](https://github.com/calkit/example-ssh)
+
+Features:
+
+- An SSH environment for running a remote command over SSH and copying back
+  results to the local machine

{calkit_python-0.16.2 → calkit_python-0.17.0}/pyproject.toml

@@ -13,6 +13,7 @@ classifiers = [
 ]
 dependencies = [
   "arithmeval",
+  "checksumdir",
   "docx2pdf",
   "dvc",
   "eval-type-backport; python_version < '3.10'",