calkit-python 0.1.1__tar.gz → 0.2.1__tar.gz

{calkit_python-0.1.1 → calkit_python-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: calkit-python
-Version: 0.1.1
+Version: 0.2.1
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://github.com/calkit/calkit
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.1.1 → calkit_python-0.2.1}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.1"
+__version__ = "0.2.1"
 
 from .core import *
 from . import git

{calkit_python-0.1.1 → calkit_python-0.2.1}/calkit/cli/core.py

@@ -21,3 +21,8 @@ def run_cmd(cmd: list[str]):
         import pty
 
         pty.spawn(cmd, lambda fd: os.read(fd, 1024))
+
+
+def raise_error(txt):
+    typer.echo(txt, err=txt)
+    raise typer.Exit(1)

{calkit_python-0.1.1 → calkit_python-0.2.1}/calkit/cli/main.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import subprocess
@@ -12,7 +13,7 @@ import typer
 from typing_extensions import Annotated, Optional
 
 import calkit
-from calkit.cli import print_sep, run_cmd
+from calkit.cli import print_sep, raise_error, run_cmd
 from calkit.cli.config import config_app
 from calkit.cli.import_ import import_app
 from calkit.cli.list import list_app
@@ -26,8 +27,11 @@ app = typer.Typer(
     pretty_exceptions_show_locals=False,
 )
 app.add_typer(config_app, name="config", help="Configure Calkit.")
+app.add_typer(new_app, name="new", help="Create a new Calkit object.")
 app.add_typer(
-    new_app, name="new", help="Add new Calkit object (to calkit.yaml)."
+    new_app,
+    name="create",
+    help="Create a new Calkit object (alias for 'new').",
 )
 app.add_typer(notebooks_app, name="nb", help="Work with Jupyter notebooks.")
 app.add_typer(list_app, name="list", help="List Calkit objects.")
@@ -46,6 +50,53 @@ def main(
         raise typer.Exit()
 
 
+@app.command(name="clone")
+def clone(
+    url: Annotated[str, typer.Argument(help="Repo URL.")],
+    location: Annotated[
+        str,
+        typer.Argument(
+            help="Location to clone to (default will be ./{repo_name})"
+        ),
+    ] = None,
+    no_config_remote: Annotated[
+        bool,
+        typer.Option(
+            "--no-config-remote",
+            help="Do not automatically configure Calkit DVC remote.",
+        ),
+    ] = False,
+    no_dvc_pull: Annotated[
+        bool, typer.Option("--no-dvc-pull", help="Do not pull DVC objects.")
+    ] = False,
+):
+    """Clone a Git repo and by default configure and pull from the DVC
+    remote.
+    """
+    # Git clone
+    cmd = ["git", "clone", url]
+    if location is not None:
+        cmd.append(location)
+    try:
+        subprocess.call(cmd)
+    except Exception as e:
+        raise_error(str(e))
+    if location is None:
+        location = url.split("/")[-1].removesuffix(".git")
+    typer.echo(f"Moving into repo dir: {location}")
+    os.chdir(location)
+    # Setup auth for any Calkit remotes
+    if not no_config_remote:
+        remotes = calkit.dvc.get_remotes()
+        for name, url in remotes.items():
+            if name == "calkit" or name.startswith("calkit:"):
+                typer.echo(f"Setting up authentication for DVC remote: {name}")
+                calkit.dvc.set_remote_auth(remote_name=name)
+    # DVC pull
+    if not no_dvc_pull:
+        subprocess.call(["dvc", "pull"])
+
+
 @app.command(name="status")
 def get_status():
     """Get a unified Git and DVC status."""
@@ -97,8 +148,7 @@ def add(
     adding any .dvc files to Git when adding to DVC.
     """
     if to is not None and to not in ["git", "dvc"]:
-        typer.echo(f"Invalid option for 'to': {to}")
-        raise typer.Exit(1)
+        raise_error(f"Invalid option for 'to': {to}")
     # Ensure autostage is enabled for DVC
     subprocess.call(["dvc", "config", "core.autostage", "true"])
     subprocess.call(["git", "add", ".dvc/config"])
@@ -117,13 +167,11 @@ def add(
         ]
         dvc_size_thresh_bytes = 1_000_000
         if "." in paths and to is None:
-            typer.echo("Cannot add '.' with calkit; use git or dvc")
-            raise typer.Exit(1)
+            raise_error("Cannot add '.' with calkit; use git or dvc")
         if to is None:
             for path in paths:
                 if os.path.isdir(path):
-                    typer.echo("Cannot auto-add directories; use git or dvc")
-                    raise typer.Exit(1)
+                    raise_error("Cannot auto-add directories; use git or dvc")
         repo = git.Repo()
         for path in paths:
             # Detect if this file should be tracked with Git or DVC
@@ -323,8 +371,7 @@ def run_dvc_repro(
     subprocess.call(["dvc", "repro"] + args)
     # Now parse stage metadata for calkit objects
     if not os.path.isfile("dvc.yaml"):
-        typer.echo("No dvc.yaml file found")
-        raise typer.Exit(1)
+        raise_error("No dvc.yaml file found")
     objects = []
     with open("dvc.yaml") as f:
         pipeline = calkit.ryaml.load(f)
@@ -332,21 +379,18 @@ def run_dvc_repro(
             ckmeta = stage_info.get("meta", {}).get("calkit")
             if ckmeta is not None:
                 if not isinstance(ckmeta, dict):
-                    typer.echo(
+                    raise_error(
                         f"Calkit metadata for {stage_name} is not a dictionary"
                     )
-                    typer.Exit(1)
                 # Stage must have a single output
                 outs = stage_info.get("outs", [])
                 if len(outs) != 1:
-                    typer.echo(
+                    raise_error(
                         f"Stage {stage_name} does not have exactly one output"
                     )
-                    raise typer.Exit(1)
                 cktype = ckmeta.get("type")
                 if cktype not in ["figure", "dataset", "publication"]:
-                    typer.echo(f"Invalid Calkit output type '{cktype}'")
-                    raise typer.Exit(1)
+                    raise_error(f"Invalid Calkit output type '{cktype}'")
                 objects.append(
                     dict(path=outs[0]) | ckmeta | dict(stage=stage_name)
                 )
@@ -433,21 +477,24 @@ def run_in_env(
     ck_info = calkit.load_calkit_info()
     envs = ck_info.get("environments", {})
     if not envs:
-        typer.echo("No environments defined in calkit.yaml", err=True)
-        raise typer.Exit(1)
+        raise_error("No environments defined in calkit.yaml")
     if isinstance(envs, list):
-        typer.echo(
-            "Error: Environments should be a dict, not a list", err=True
-        )
-        raise typer.Exit(1)
-    if len(envs) > 1 and env_name is None:
-        typer.echo(
-            "Environment must be specified if there are multiple",
-            err=True,
-        )
-        raise typer.Exit(1)
+        raise_error("Error: Environments should be a dict, not a list")
+    if env_name is None:
+        # See if there's a default env, or only one env defined
+        default_env_name = None
+        for n, e in envs.items():
+            if e.get("default"):
+                if default_env_name is not None:
+                    raise_error(
+                        "Only one default environment can be specified"
+                    )
+                default_env_name = n
+        if default_env_name is None and len(envs) == 1:
+            default_env_name = list(envs.keys())[0]
+        env_name = default_env_name
     if env_name is None:
-        env_name = list(envs.keys())[0]
+        raise_error("Environment must be specified if there are multiple")
     env = envs[env_name]
     cwd = os.getcwd()
     image_name = env.get("image", env_name)
@@ -477,8 +524,7 @@ def run_in_env(
             typer.echo(f"Running command: {cmd}")
         subprocess.call(cmd)
     else:
-        typer.echo("Environment kind not supported", err=True)
-        raise typer.Exit(1)
+        raise_error("Environment kind not supported")
 
 
 @app.command(
@@ -507,8 +553,7 @@ def check_call(
             subprocess.check_call(if_error, shell=True)
             typer.echo("Fallback call succeeded")
         except subprocess.CalledProcessError:
-            typer.echo("Fallback call failed", err=True)
-            raise typer.Exit(1)
+            raise_error("Fallback call failed")
 
 
 @app.command(
@@ -539,6 +584,10 @@ def build_docker(
     except subprocess.CalledProcessError:
         typer.echo(f"No image with tag {tag} found locally")
         inspect = []
+    typer.echo(f"Reading Dockerfile from {fpath}")
+    with open(fpath) as f:
+        dockerfile = f.read()
+    dockerfile_md5 = hashlib.md5(dockerfile.encode()).hexdigest()
     lock_fpath = fpath + "-lock.json"
     rebuild = True
     if os.path.isfile(lock_fpath):
@@ -549,11 +598,14 @@ def build_docker(
         typer.echo(f"Lock file ({lock_fpath}) does not exist")
         lock = None
     if inspect and lock:
-        typer.echo("Checking image against lock file")
-        rebuild = inspect[0]["RootFS"]["Layers"] != lock[0]["RootFS"]["Layers"]
+        typer.echo("Checking image and Dockerfile against lock file")
+        rebuild = inspect[0]["RootFS"]["Layers"] != lock[0]["RootFS"][
+            "Layers"
+        ] or dockerfile_md5 != lock[0].get("DockerfileMD5")
     if rebuild:
         subprocess.check_call(["docker", "build", "-t", tag, "-f", fpath, "."])
     # Write the lock file
     inspect = get_docker_inspect()
+    inspect[0]["DockerfileMD5"] = dockerfile_md5
     with open(lock_fpath, "w") as f:
         json.dump(inspect, f, indent=4)

{calkit_python-0.1.1 → calkit_python-0.2.1}/calkit/cli/new.py

@@ -10,17 +10,13 @@ import typer
 from typing_extensions import Annotated
 
 import calkit
+from calkit.cli import raise_error
 from calkit.core import ryaml
 from calkit.docker import LAYERS
 
 new_app = typer.Typer(no_args_is_help=True)
 
 
-def _error(txt):
-    typer.echo(txt, err=txt)
-    raise typer.Exit(1)
-
-
 @new_app.command(name="figure")
 def new_figure(
     path: str,
@@ -69,18 +65,18 @@ def new_figure(
         ),
     ] = False,
 ):
-    """Add a new figure."""
+    """Create a new figure."""
     ck_info = calkit.load_calkit_info()
     figures = ck_info.get("figures", [])
     paths = [f.get("path") for f in figures]
     if not overwrite and path in paths:
-        _error(f"Figure at path {path} already exists")
+        raise_error(f"Figure at path {path} already exists")
     if cmd is not None and stage_name is None:
-        _error("Stage name must be provided if command is specified")
+        raise_error("Stage name must be provided if command is specified")
     if (deps or outs or outs_from_stage) and not cmd:
-        _error("Command must be provided")
+        raise_error("Command must be provided")
     if (deps or outs or outs_from_stage) and not stage_name:
-        _error("Stage name must be provided")
+        raise_error("Stage name must be provided")
     obj = dict(path=path, title=title)
     if description is not None:
         obj["description"] = description
@@ -91,7 +87,7 @@ def new_figure(
             pipeline = calkit.dvc.read_pipeline()
             stages = pipeline.get("stages", {})
             if outs_from_stage not in stages:
-                _error(f"Stage {outs_from_stage} does not exist")
+                raise_error(f"Stage {outs_from_stage} does not exist")
             deps += stages[outs_from_stage].get("outs", [])
         if path not in outs:
             outs.append(path)
@@ -232,15 +228,9 @@ def new_docker_env(
 ):
     """Create a new Docker environment."""
     if base and os.path.isfile(path) and not overwrite:
-        typer.echo(
-            "Output path already exists (use -f to overwrite)", err=True
-        )
-        raise typer.Exit(1)
+        raise_error("Output path already exists (use -f to overwrite)")
     if stage and not base:
-        typer.echo(
-            "--from must be specified when creating a build stage", err=True
-        )
-        raise typer.Exit(1)
+        raise_error("--from must be specified when creating a build stage")
     if image_name is None:
         typer.echo("No image name specified; using environment name")
         image_name = name
@@ -249,8 +239,7 @@ def new_docker_env(
         txt = "FROM " + base + "\n\n"
         for layer in layers:
             if layer not in LAYERS:
-                typer.echo(f"Unknown layer type '{layer}'")
-                raise typer.Exit(1)
+                raise_error(f"Unknown layer type '{layer}'")
             txt += LAYERS[layer] + "\n\n"
         txt += f"RUN mkdir {wdir}\n"
         txt += f"WORKDIR {wdir}\n"
@@ -264,11 +253,10 @@ def new_docker_env(
         typer.echo("Converting environments from list to dict")
         envs = {env.pop("name"): env for env in envs}
     if name in envs and not overwrite:
-        typer.echo(
+        raise_error(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
-        raise typer.Exit(1)
     if base:
         repo.git.add(path)
     typer.echo("Adding environment to calkit.yaml")
@@ -309,7 +297,8 @@ def new_docker_env(
             ]
         )
     repo.git.add("calkit.yaml")
-    repo.git.add("dvc.yaml")
+    if stage:
+        repo.git.add("dvc.yaml")
     if not no_commit and repo.git.diff("--staged"):
         repo.git.commit(["-m", f"Add Docker environment {name}"])
 
@@ -348,8 +337,7 @@ def new_foreach_stage(
     """
     pipeline = calkit.dvc.read_pipeline()
     if name in pipeline and not overwrite:
-        typer.echo("Stage already exists; use -f to overwrite", err=True)
-        raise typer.Exit(1)
+        raise_error("Stage already exists; use -f to overwrite")
     if "stages" not in pipeline:
         pipeline["stages"] = {}
     pipeline["stages"][name] = dict(

{calkit_python-0.1.1 → calkit_python-0.2.1}/calkit/docker.py

@@ -1,6 +1,6 @@
 """Functionality for working with Docker."""
 
-MAMBAFORGE_LAYER_TXT = """
+MAMBAFORGE_LAYER_TXT = r"""
 # Install Miniforge
 ARG MINIFORGE_NAME=Mambaforge
 ARG MINIFORGE_VERSION=24.3.0-0
@@ -40,8 +40,10 @@ RUN apt-get update > /dev/null && \
     echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc
 """.strip()
 
-FOAMPY_LAYER_TEXT = """
-RUN pip install numpy pandas matplotlib && pip install scipy && pip install foampy
+FOAMPY_LAYER_TEXT = r"""
+RUN pip install --no-cache-dir numpy pandas matplotlib h5py \
+    && pip install --no-cache-dir scipy \
+    && pip install --no-cache-dir foampy
 """.strip()
 
 LAYERS = {

{calkit_python-0.1.1 → calkit_python-0.2.1}/calkit/dvc.py

@@ -67,9 +67,7 @@ def add_external_remote(owner_name: str, project_name: str):
     base_url = calkit.cloud.get_base_url()
     remote_url = f"{base_url}/projects/{owner_name}/{project_name}/dvc"
     remote_name = f"{get_app_name()}:{owner_name}/{project_name}"
-    subprocess.call(
-        ["dvc", "remote", "add", "-f", remote_name, remote_url]
-    )
+    subprocess.call(["dvc", "remote", "add", "-f", remote_name, remote_url])
     subprocess.call(["dvc", "remote", "modify", remote_name, "auth", "custom"])
     set_remote_auth(remote_name)
 
@@ -79,3 +77,17 @@ def read_pipeline() -> dict:
         return {}
     with open("dvc.yaml") as f:
         return calkit.ryaml.load(f)
+
+
+def get_remotes() -> dict[str, str]:
+    """Get a dictionary of DVC remotes, keyed by name, with URL as the
+    value.
+    """
+    out = subprocess.check_output(["dvc", "remote", "list"]).decode().strip()
+    if not out:
+        return {}
+    resp = {}
+    for line in out.split("\n"):
+        name, url = line.split("\t")
+        resp[name] = url
+    return resp

{calkit_python-0.1.1 → calkit_python-0.2.1}/calkit/models.py

@@ -66,6 +66,7 @@ class Environment(BaseModel):
     path: str | None = None
     description: str | None = None
     stage: str | None = None
+    default: bool | None = None
 
 
 class DockerEnvironment(Environment):
@@ -89,8 +90,24 @@ class Notebook(_CalkitObject):
 class ProjectInfo(BaseModel):
     """All of the project's information or metadata, written to the
     ``calkit.yaml`` file.
+
+    Attributes
+    ----------
+    parent : str
+        The project's parent project, if applicable. This should be set if
+        the project was created as a copy of another. This is similar to the
+        concept of forking, but unlike a fork, a child project's changes
+        are not meant to be merged back into the parent.
+        The format of this field should be something like
+        {owner_name}/{project_name}, e.g., 'someuser/some-project-name'.
+        Note that individual objects can be imported from other projects, but
+        that doesn't necessarily make them parent projects.
+        This is probably not that important of a distinction.
+        The real use case is being able to trace where things came from and
+        distinguish what has been newly created here.
     """
 
+    parent: str | None = None
     questions: list[str] = []
     datasets: list[Dataset] = []
     figures: list[Figure] = []

calkit_python-0.2.1/calkit/tests/test_dvc.py

@@ -0,0 +1,15 @@
+"""Tests for the ``dvc`` module."""
+
+import subprocess
+
+import calkit
+
+
+def test_get_remotes(tmp_dir):
+    subprocess.call(["git", "init"])
+    assert not calkit.dvc.get_remotes()
+    subprocess.call(["dvc", "init"])
+    assert not calkit.dvc.get_remotes()
+    subprocess.call(["dvc", "remote", "add", "something", "https://sup.com"])
+    resp = calkit.dvc.get_remotes()
+    assert resp == {"something": "https://sup.com"}