calkit-python 0.8.4__tar.gz → 0.9.0__tar.gz

{calkit_python-0.8.4 → calkit_python-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: calkit-python
-Version: 0.8.4
+Version: 0.9.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://github.com/calkit/calkit
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -46,7 +46,8 @@ Our goal is to make reproducibility easier so it becomes more common.
 To do this, we try to make it easy for users to follow two simple rules:
 
 1. **Keep everything in version control.** This includes large files like
-   datasets, enabled by DVC. The [Calkit cloud](https://calkit.io)
+   datasets, enabled by DVC.
+   The [Calkit cloud](https://github.com/calkit/calkit-cloud)
    serves as a simple default DVC remote storage location for those who do not
    want to manage their own infrastructure.
 2. **Generate all important artifacts with a single pipeline.** There should be

{calkit_python-0.8.4 → calkit_python-0.9.0}/README.md

@@ -15,7 +15,8 @@ Our goal is to make reproducibility easier so it becomes more common.
 To do this, we try to make it easy for users to follow two simple rules:
 
 1. **Keep everything in version control.** This includes large files like
-   datasets, enabled by DVC. The [Calkit cloud](https://calkit.io)
+   datasets, enabled by DVC.
+   The [Calkit cloud](https://github.com/calkit/calkit-cloud)
    serves as a simple default DVC remote storage location for those who do not
    want to manage their own infrastructure.
 2. **Generate all important artifacts with a single pipeline.** There should be

{calkit_python-0.8.4 → calkit_python-0.9.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.8.4"
+__version__ = "0.9.0"
 
 from .core import *
 from . import git

{calkit_python-0.8.4 → calkit_python-0.9.0}/calkit/cli/main.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import csv
+import functools
 import hashlib
 import json
 import os
@@ -387,7 +388,10 @@ def run_dvc_repro(
         args += ["--pipeline", pipeline]
     if downstream is not None:
         args += downstream
-    subprocess.check_call(["dvc", "repro"] + args)
+    try:
+        subprocess.check_call(["dvc", "repro"] + args)
+    except subprocess.CalledProcessError:
+        raise_error("DVC pipeline failed")
     # Now parse stage metadata for calkit objects
     if not os.path.isfile("dvc.yaml"):
         raise_error("No dvc.yaml file found")
@@ -504,6 +508,20 @@ def run_in_env(
             ),
         ),
     ] = None,
+    no_check: Annotated[
+        bool,
+        typer.Option(
+            "--no-check",
+            help="Don't check the environment is valid before running in it.",
+        ),
+    ] = False,
+    relaxed_check: Annotated[
+        bool,
+        typer.Option(
+            "--relaxed",
+            help="Check the environment in a relaxed way, if applicable.",
+        ),
+    ] = False,
     verbose: Annotated[
         bool, typer.Option("--verbose", "-v", help="Print verbose output.")
     ] = False,
@@ -529,6 +547,8 @@ def run_in_env(
         env_name = default_env_name
     if env_name is None:
         raise_error("Environment must be specified if there are multiple")
+    if env_name not in envs:
+        raise_error(f"Environment '{env_name}' does not exist")
     env = envs[env_name]
     if wdir is not None:
         cwd = os.path.abspath(wdir)
@@ -539,6 +559,15 @@ def run_in_env(
     shell = env.get("shell", "sh")
     platform = env.get("platform")
     if env["kind"] == "docker":
+        if "image" not in env:
+            raise_error("Image must be defined for Docker environments")
+        if "path" in env and not no_check:
+            check_docker_env(
+                tag=env["image"],
+                fpath=env["path"],
+                platform=env.get("platform"),
+                quiet=True,
+            )
         shell_cmd = " ".join(cmd)
         docker_cmd = [
             "docker",
@@ -560,14 +589,75 @@ def run_in_env(
         ]
         if verbose:
             typer.echo(f"Running command: {docker_cmd}")
-        subprocess.check_call(docker_cmd, cwd=wdir)
+        try:
+            subprocess.check_call(docker_cmd, cwd=wdir)
+        except subprocess.CalledProcessError:
+            raise_error("Failed to run in Docker environment")
     elif env["kind"] == "conda":
         with open(env["path"]) as f:
             conda_env = calkit.ryaml.load(f)
+        if not no_check:
+            check_conda_env(
+                env_fpath=env["path"], relaxed=relaxed_check, quiet=True
+            )
         cmd = ["conda", "run", "-n", conda_env["name"]] + cmd
         if verbose:
             typer.echo(f"Running command: {cmd}")
-        subprocess.check_call(cmd, cwd=wdir)
+        try:
+            subprocess.check_call(cmd, cwd=wdir)
+        except subprocess.CalledProcessError:
+            raise_error("Failed to run in Conda environment")
+    elif env["kind"] in ["pixi", "uv"]:
+        env_cmd = []
+        if "name" in env:
+            env_cmd = ["--environment", env["name"]]
+        cmd = [env["kind"], "run"] + env_cmd + cmd
+        if verbose:
+            typer.echo(f"Running command: {cmd}")
+        try:
+            subprocess.check_call(cmd, cwd=wdir)
+        except subprocess.CalledProcessError:
+            raise_error(f"Failed to run in {env['kind']} environment")
+    elif env["kind"] == "uv-venv":
+        # TODO: This doesn't work on Windows
+        if "prefix" not in env:
+            raise_error("uv-venv environments require a prefix")
+        if "path" not in env:
+            raise_error("uv-venv environments require a path")
+        prefix = env["prefix"]
+        path = env["path"]
+        shell_cmd = " ".join(cmd)
+        # Check environment
+        if not no_check:
+            if not os.path.isdir(prefix):
+                if verbose:
+                    typer.echo(f"Creating uv-venv at {prefix}")
+                try:
+                    subprocess.check_call(["uv", "venv", prefix], cwd=wdir)
+                except subprocess.CalledProcessError:
+                    raise_error(f"Failed to create uv-venv at {prefix}")
+            fname, ext = os.path.splitext(path)
+            lock_fpath = fname + "-lock" + ext
+            check_cmd = (
+                f". {prefix}/bin/activate "
+                f"&& uv pip install -q -r {path} "
+                f"&& uv pip freeze > {lock_fpath} "
+                "&& deactivate"
+            )
+            try:
+                if verbose:
+                    typer.echo(f"Running command: {check_cmd}")
+                subprocess.check_output(check_cmd, shell=True, cwd=wdir)
+            except subprocess.CalledProcessError:
+                raise_error("Failed to check uv-venv")
+        # Now run the command
+        cmd = f". {prefix}/bin/activate && {shell_cmd} && deactivate"
+        if verbose:
+            typer.echo(f"Running command: {cmd}")
+        try:
+            subprocess.check_call(cmd, shell=True, cwd=wdir)
+        except subprocess.CalledProcessError:
+            raise_error("Failed to run in uv-venv")
     else:
         raise_error("Environment kind not supported")
 
@@ -605,7 +695,7 @@ def check_call(
     name="build-docker",
     help="Build Docker image if missing or different from lock file.",
 )
-def build_docker(
+def check_docker_env(
     tag: Annotated[str, typer.Argument(help="Image tag.")],
     fpath: Annotated[
         str, typer.Option("-i", "--input", help="Path to input Dockerfile.")
@@ -613,6 +703,9 @@ def build_docker(
     platform: Annotated[
         str, typer.Option("--platform", help="Which platform(s) to build for.")
     ] = None,
+    quiet: Annotated[
+        bool, typer.Option("--quiet", "-q", help="Be quiet.")
+    ] = False,
 ):
     def get_docker_inspect():
         out = json.loads(
@@ -626,28 +719,31 @@ def build_docker(
         _ = out[0].pop("DockerVersion")
         return out
 
-    typer.echo(f"Checking for existing image with tag {tag}")
+    outfile = open(os.devnull, "w") if quiet else None
+    typer.echo(f"Checking for existing image with tag {tag}", file=outfile)
     # First call Docker inspect
     try:
         inspect = get_docker_inspect()
     except subprocess.CalledProcessError:
-        typer.echo(f"No image with tag {tag} found locally")
+        typer.echo(f"No image with tag {tag} found locally", file=outfile)
         inspect = []
-    typer.echo(f"Reading Dockerfile from {fpath}")
+    typer.echo(f"Reading Dockerfile from {fpath}", file=outfile)
     with open(fpath) as f:
         dockerfile = f.read()
     dockerfile_md5 = hashlib.md5(dockerfile.encode()).hexdigest()
     lock_fpath = fpath + "-lock.json"
     rebuild = True
     if os.path.isfile(lock_fpath):
-        typer.echo(f"Reading lock file: {lock_fpath}")
+        typer.echo(f"Reading lock file: {lock_fpath}", file=outfile)
         with open(lock_fpath) as f:
             lock = json.load(f)
     else:
-        typer.echo(f"Lock file ({lock_fpath}) does not exist")
+        typer.echo(f"Lock file ({lock_fpath}) does not exist", file=outfile)
         lock = None
     if inspect and lock:
-        typer.echo("Checking image and Dockerfile against lock file")
+        typer.echo(
+            "Checking image and Dockerfile against lock file", file=outfile
+        )
         rebuild = inspect[0]["RootFS"]["Layers"] != lock[0]["RootFS"][
             "Layers"
         ] or dockerfile_md5 != lock[0].get("DockerfileMD5")
@@ -827,7 +923,23 @@ def check_conda_env(
             ),
         ),
     ] = None,
+    relaxed: Annotated[
+        bool,
+        typer.Option(
+            "--relaxed", help="Treat conda and pip dependencies as equivalent."
+        ),
+    ] = False,
+    quiet: Annotated[
+        bool, typer.Option("--quiet", "-q", help="Be quiet.")
+    ] = False,
 ):
+    if quiet:
+        log_func = functools.partial(typer.echo, file=open(os.devnull, "w"))
+    else:
+        log_func = typer.echo
     calkit.conda.check_env(
-        env_fpath=env_fpath, output_fpath=output_fpath, log_func=typer.echo
+        env_fpath=env_fpath,
+        output_fpath=output_fpath,
+        log_func=log_func,
+        relaxed=relaxed,
     )

{calkit_python-0.8.4 → calkit_python-0.9.0}/calkit/conda.py

@@ -4,27 +4,40 @@ import json
 import os
 import subprocess
 
+from pydantic import BaseModel
+
 import calkit
 from calkit import ryaml
 
 
+class EnvCheckResult(BaseModel):
+    env_exists: bool | None = None
+    env_needs_export: bool | None = None
+    env_needs_rebuild: bool | None = None
+
+
 def check_env(
     env_fpath: str = "environment.yml",
     use_mamba=True,
     log_func=None,
     output_fpath: str = None,
-):
+    relaxed: bool = False,
+) -> EnvCheckResult:
     """Check that a conda environment matches its spec.
 
     If it doesn't match, recreate it.
 
     Note that this only works with exact or no version specification.
     Using greater than and less than operators is not supported.
+
+    If ``relaxed`` is enabled, dependencies can exist in either the conda or
+    pip category.
     """
     conda = "mamba" if use_mamba else "conda"
     if log_func is None:
         log_func = calkit.logger.info
     log_func(f"Checking conda env defined in {env_fpath}")
+    res = EnvCheckResult()
     envs = json.loads(
         subprocess.check_output([conda, "env", "list", "--json"]).decode()
     )["envs"]
@@ -44,11 +57,13 @@ def check_env(
     # Check if env even exists
     if env_name not in existing_env_names:
         log_func(f"Environment {env_name} doesn't exist; creating")
+        res.env_exists = False
         # Environment doesn't exist, so create it
         subprocess.check_call([conda, "env", "create", "-y", "-f", env_fpath])
         env_needs_rebuild = False
         env_needs_export = True
     else:
+        res.env_exists = True
         env_needs_export = False
         # Environment does exist, so check it
         if os.path.isfile(env_check_fpath):
@@ -67,6 +82,7 @@ def check_env(
         else:
             env_needs_export = True
         if env_needs_export:
+            res.env_needs_export = True
             log_func(f"Exporting existing env to {env_check_fpath}")
             env_check = json.loads(
                 subprocess.check_output(
@@ -100,6 +116,12 @@ def check_env(
         else:
             required_conda_deps = env_spec["dependencies"]
             required_pip_deps = []
+        if relaxed:
+            log_func("Running in relaxed mode; combining pip and conda deps")
+            for dep in existing_pip_deps:
+                existing_conda_deps.append(dep.replace("==", "="))
+            for dep in required_pip_deps:
+                required_conda_deps.append(dep.replace("==", "="))
         log_func("Checking conda dependencies")
         for dep in required_conda_deps:
             dep_split = dep.split("=")
@@ -121,7 +143,7 @@ def check_env(
                     log_func(f"Found missing dependency: {dep}")
                     env_needs_rebuild = True
                     break
-        if not env_needs_rebuild:
+        if not env_needs_rebuild and not relaxed:
             log_func("Checking pip dependencies")
             for dep in required_pip_deps:
                 dep_split = dep.split("==")
@@ -142,11 +164,13 @@ def check_env(
                         env_needs_rebuild = True
                         break
     if env_needs_rebuild:
+        res.env_needs_rebuild = True
         log_func(f"Rebuilding {env_name} since it does not match spec")
         subprocess.check_call([conda, "env", "create", "-y", "-f", env_fpath])
         env_needs_export = True
     else:
         log_func(f"Environment {env_name} matches spec")
+        res.env_needs_rebuild = False
     # If the env was rebuilt, export the env check
     if env_needs_export:
         log_func(f"Exporting existing env to {env_check_fpath}")
@@ -171,8 +195,14 @@ def check_env(
     if output_fpath is None:
         fname, ext = os.path.splitext(env_fpath)
         output_fpath = fname + "-lock" + ext
-    log_func(f"Exporting lock file to {output_fpath}")
-    with open(output_fpath, "w") as f:
-        _ = env_check.pop("mtime")
-        _ = env_check.pop("prefix")
-        ryaml.dump(env_check, f)
+    if (
+        not res.env_exists
+        or res.env_needs_rebuild
+        or not os.path.isfile(output_fpath)
+    ):
+        log_func(f"Exporting lock file to {output_fpath}")
+        with open(output_fpath, "w") as f:
+            _ = env_check.pop("mtime")
+            _ = env_check.pop("prefix")
+            ryaml.dump(env_check, f)
+    return res

calkit_python-0.9.0/calkit/tests/test_conda.py

@@ -0,0 +1,126 @@
+"""Tests for the ``conda`` module."""
+
+import subprocess
+import uuid
+
+import pytest
+
+from calkit.conda import check_env
+
+
+def delete_env(name: str):
+    subprocess.check_call(["mamba", "env", "remove", "-y", "-n", name])
+
+
+@pytest.fixture
+def env_name():
+    # Setup code
+    name = "tmp_" + str(uuid.uuid4())[:12]
+    yield name
+    # Teardown code
+    delete_env(name)
+
+
+def test_check_env(tmp_dir, env_name):
+    subprocess.check_call(["git", "init"])
+    subprocess.check_call(["dvc", "init"])
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "conda-env",
+            "-n",
+            env_name,
+            "python",
+            "pip",
+            "--pip",
+            "pxl",
+        ]
+    )
+    res = check_env()
+    assert not res.env_exists
+    res = check_env()
+    assert res.env_exists
+    assert not res.env_needs_export
+    assert not res.env_needs_rebuild
+    # Now let's update the env spec so it needs a rebuild
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "conda-env",
+            "--overwrite",
+            "-n",
+            env_name,
+            "python=3.11.0",
+            "pip",
+            "--pip",
+            "pxl",
+        ]
+    )
+    res = check_env()
+    assert res.env_exists
+    assert not res.env_needs_export
+    assert res.env_needs_rebuild
+    res = check_env()
+    assert not res.env_needs_rebuild
+    # Check relaxed mode, where we allow dependencies to be in either the pip
+    # or conda section
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "conda-env",
+            "--overwrite",
+            "-n",
+            env_name,
+            "python=3.11.0",
+            "pip",
+            "sqlalchemy",
+        ]
+    )
+    subprocess.check_call(
+        [
+            "conda",
+            "run",
+            "-n",
+            env_name,
+            "pip",
+            "install",
+            "--upgrade",
+            "sqlalchemy",
+        ]
+    )
+    res = check_env()
+    assert res.env_needs_rebuild
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "conda-env",
+            "--overwrite",
+            "-n",
+            env_name,
+            "python=3.11.0",
+            "pip",
+            "sqlalchemy",
+        ]
+    )
+    res = check_env(relaxed=True)
+    assert not res.env_needs_rebuild
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "conda-env",
+            "--overwrite",
+            "-n",
+            env_name,
+            "python=3.11.0",
+            "pip",
+            "--pip",
+            "sqlalchemy",
+        ]
+    )
+    res = check_env(relaxed=True)
+    assert not res.env_needs_rebuild