calkit-python 0.8.5__tar.gz → 0.9.1__tar.gz

{calkit_python-0.8.5 → calkit_python-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: calkit-python
-Version: 0.8.5
+Version: 0.9.1
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://github.com/calkit/calkit
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -46,7 +46,8 @@ Our goal is to make reproducibility easier so it becomes more common.
 To do this, we try to make it easy for users to follow two simple rules:
 
 1. **Keep everything in version control.** This includes large files like
-   datasets, enabled by DVC. The [Calkit cloud](https://calkit.io)
+   datasets, enabled by DVC.
+   The [Calkit cloud](https://github.com/calkit/calkit-cloud)
    serves as a simple default DVC remote storage location for those who do not
    want to manage their own infrastructure.
 2. **Generate all important artifacts with a single pipeline.** There should be
@@ -105,8 +106,8 @@ If you want to use [Docker](https://docker.com) containers,
 which is typically a good idea,
 that should also be installed.
 For Python, we recommend
-[Mambaforge](https://conda-forge.org/miniforge/).
-If you're a Windows user and decide to install Mambaforge or any other
+[Miniforge](https://conda-forge.org/miniforge/).
+If you're a Windows user and decide to install Miniforge or any other
 Conda-based distribution,
 e.g., Anaconda, you'll probably want to ensure that environment is
 [activated by default in Git Bash](https://discuss.codecademy.com/t/setting-up-conda-in-git-bash/534473).

{calkit_python-0.8.5 → calkit_python-0.9.1}/README.md

@@ -15,7 +15,8 @@ Our goal is to make reproducibility easier so it becomes more common.
 To do this, we try to make it easy for users to follow two simple rules:
 
 1. **Keep everything in version control.** This includes large files like
-   datasets, enabled by DVC. The [Calkit cloud](https://calkit.io)
+   datasets, enabled by DVC.
+   The [Calkit cloud](https://github.com/calkit/calkit-cloud)
    serves as a simple default DVC remote storage location for those who do not
    want to manage their own infrastructure.
 2. **Generate all important artifacts with a single pipeline.** There should be
@@ -74,8 +75,8 @@ If you want to use [Docker](https://docker.com) containers,
 which is typically a good idea,
 that should also be installed.
 For Python, we recommend
-[Mambaforge](https://conda-forge.org/miniforge/).
-If you're a Windows user and decide to install Mambaforge or any other
+[Miniforge](https://conda-forge.org/miniforge/).
+If you're a Windows user and decide to install Miniforge or any other
 Conda-based distribution,
 e.g., Anaconda, you'll probably want to ensure that environment is
 [activated by default in Git Bash](https://discuss.codecademy.com/t/setting-up-conda-in-git-bash/534473).

{calkit_python-0.8.5 → calkit_python-0.9.1}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.8.5"
+__version__ = "0.9.1"
 
 from .core import *
 from . import git

{calkit_python-0.8.5 → calkit_python-0.9.1}/calkit/cli/config.py

@@ -57,3 +57,11 @@ def setup_remote_auth():
         if name == "calkit" or name.startswith("calkit:"):
             typer.echo(f"Setting up authentication for DVC remote: {name}")
             set_remote_auth(remote_name=name)
+
+
+@config_app.command(name="list")
+def list_config_keys():
+    """List keys in the config."""
+    cfg = config.read()
+    for key in cfg.model_dump():
+        typer.echo(key)

{calkit_python-0.8.5 → calkit_python-0.9.1}/calkit/cli/main.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import csv
+import functools
 import hashlib
 import json
 import os
@@ -22,6 +23,7 @@ from calkit.cli.list import list_app
 from calkit.cli.new import new_app
 from calkit.cli.notebooks import notebooks_app
 from calkit.cli.office import office_app
+from calkit.cli.update import update_app
 from calkit.models import Procedure
 
 app = typer.Typer(
@@ -41,6 +43,7 @@ app.add_typer(notebooks_app, name="nb", help="Work with Jupyter notebooks.")
 app.add_typer(list_app, name="list", help="List Calkit objects.")
 app.add_typer(import_app, name="import", help="Import objects.")
 app.add_typer(office_app, name="office", help="Work with Microsoft Office.")
+app.add_typer(update_app, name="update", help="Update objects.")
 
 
 @app.callback()
@@ -290,26 +293,57 @@ def save(
         push()
 
 
-@app.command(name="pull", help="Pull with both Git and DVC.")
-def pull():
+@app.command(name="pull")
+def pull(
+    no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False
+):
+    """Pull with both Git and DVC."""
     typer.echo("Git pulling")
-    subprocess.call(["git", "pull"])
+    try:
+        subprocess.check_call(["git", "pull"])
+    except subprocess.CalledProcessError:
+        raise_error("Git pull failed")
     typer.echo("DVC pulling")
-    subprocess.call(["dvc", "pull"])
+    if not no_check_auth:
+        # Check that our dvc remotes all have our DVC token set for them
+        remotes = calkit.dvc.get_remotes()
+        for name, url in remotes.items():
+            if name == "calkit" or name.startswith("calkit:"):
+                typer.echo(f"Checking authentication for DVC remote: {name}")
+                calkit.dvc.set_remote_auth(remote_name=name)
+    try:
+        subprocess.check_call(["dvc", "pull"])
+    except subprocess.CalledProcessError:
+        raise_error("DVC pull failed")
 
 
-@app.command(name="push", help="Push with both Git and DVC.")
-def push():
+@app.command(name="push")
+def push(
+    no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False
+):
+    """Push with both Git and DVC."""
     typer.echo("Pushing to Git remote")
-    subprocess.call(["git", "push"])
+    try:
+        subprocess.check_call(["git", "push"])
+    except subprocess.CalledProcessError:
+        raise_error("Git push failed")
     typer.echo("Pushing to DVC remote")
-    subprocess.call(["dvc", "push"])
+    if not no_check_auth:
+        # Check that our dvc remotes all have our DVC token set for them
+        remotes = calkit.dvc.get_remotes()
+        for name, url in remotes.items():
+            if name == "calkit" or name.startswith("calkit:"):
+                typer.echo(f"Checking authentication for DVC remote: {name}")
+                calkit.dvc.set_remote_auth(remote_name=name)
+    try:
+        subprocess.check_call(["dvc", "push"])
+    except subprocess.CalledProcessError:
+        raise_error("DVC push failed")
 
 
-@app.command(
-    name="local-server", help="Run the local server to interact over HTTP."
-)
+@app.command(name="local-server")
 def run_local_server():
+    """Run the local server to interact over HTTP."""
     import uvicorn
 
     uvicorn.run(
@@ -387,7 +421,10 @@ def run_dvc_repro(
         args += ["--pipeline", pipeline]
     if downstream is not None:
         args += downstream
-    subprocess.check_call(["dvc", "repro"] + args)
+    try:
+        subprocess.check_call(["dvc", "repro"] + args)
+    except subprocess.CalledProcessError:
+        raise_error("DVC pipeline failed")
     # Now parse stage metadata for calkit objects
     if not os.path.isfile("dvc.yaml"):
         raise_error("No dvc.yaml file found")
@@ -504,6 +541,20 @@ def run_in_env(
             ),
         ),
     ] = None,
+    no_check: Annotated[
+        bool,
+        typer.Option(
+            "--no-check",
+            help="Don't check the environment is valid before running in it.",
+        ),
+    ] = False,
+    relaxed_check: Annotated[
+        bool,
+        typer.Option(
+            "--relaxed",
+            help="Check the environment in a relaxed way, if applicable.",
+        ),
+    ] = False,
     verbose: Annotated[
         bool, typer.Option("--verbose", "-v", help="Print verbose output.")
     ] = False,
@@ -529,6 +580,8 @@ def run_in_env(
         env_name = default_env_name
     if env_name is None:
         raise_error("Environment must be specified if there are multiple")
+    if env_name not in envs:
+        raise_error(f"Environment '{env_name}' does not exist")
     env = envs[env_name]
     if wdir is not None:
         cwd = os.path.abspath(wdir)
@@ -539,6 +592,15 @@ def run_in_env(
     shell = env.get("shell", "sh")
     platform = env.get("platform")
     if env["kind"] == "docker":
+        if "image" not in env:
+            raise_error("Image must be defined for Docker environments")
+        if "path" in env and not no_check:
+            check_docker_env(
+                tag=env["image"],
+                fpath=env["path"],
+                platform=env.get("platform"),
+                quiet=True,
+            )
         shell_cmd = " ".join(cmd)
         docker_cmd = [
             "docker",
@@ -560,14 +622,75 @@ def run_in_env(
         ]
         if verbose:
             typer.echo(f"Running command: {docker_cmd}")
-        subprocess.check_call(docker_cmd, cwd=wdir)
+        try:
+            subprocess.check_call(docker_cmd, cwd=wdir)
+        except subprocess.CalledProcessError:
+            raise_error("Failed to run in Docker environment")
     elif env["kind"] == "conda":
         with open(env["path"]) as f:
             conda_env = calkit.ryaml.load(f)
+        if not no_check:
+            check_conda_env(
+                env_fpath=env["path"], relaxed=relaxed_check, quiet=True
+            )
         cmd = ["conda", "run", "-n", conda_env["name"]] + cmd
         if verbose:
             typer.echo(f"Running command: {cmd}")
-        subprocess.check_call(cmd, cwd=wdir)
+        try:
+            subprocess.check_call(cmd, cwd=wdir)
+        except subprocess.CalledProcessError:
+            raise_error("Failed to run in Conda environment")
+    elif env["kind"] in ["pixi", "uv"]:
+        env_cmd = []
+        if "name" in env:
+            env_cmd = ["--environment", env["name"]]
+        cmd = [env["kind"], "run"] + env_cmd + cmd
+        if verbose:
+            typer.echo(f"Running command: {cmd}")
+        try:
+            subprocess.check_call(cmd, cwd=wdir)
+        except subprocess.CalledProcessError:
+            raise_error(f"Failed to run in {env['kind']} environment")
+    elif env["kind"] == "uv-venv":
+        # TODO: This doesn't work on Windows
+        if "prefix" not in env:
+            raise_error("uv-venv environments require a prefix")
+        if "path" not in env:
+            raise_error("uv-venv environments require a path")
+        prefix = env["prefix"]
+        path = env["path"]
+        shell_cmd = " ".join(cmd)
+        # Check environment
+        if not no_check:
+            if not os.path.isdir(prefix):
+                if verbose:
+                    typer.echo(f"Creating uv-venv at {prefix}")
+                try:
+                    subprocess.check_call(["uv", "venv", prefix], cwd=wdir)
+                except subprocess.CalledProcessError:
+                    raise_error(f"Failed to create uv-venv at {prefix}")
+            fname, ext = os.path.splitext(path)
+            lock_fpath = fname + "-lock" + ext
+            check_cmd = (
+                f". {prefix}/bin/activate "
+                f"&& uv pip install -q -r {path} "
+                f"&& uv pip freeze > {lock_fpath} "
+                "&& deactivate"
+            )
+            try:
+                if verbose:
+                    typer.echo(f"Running command: {check_cmd}")
+                subprocess.check_output(check_cmd, shell=True, cwd=wdir)
+            except subprocess.CalledProcessError:
+                raise_error("Failed to check uv-venv")
+        # Now run the command
+        cmd = f". {prefix}/bin/activate && {shell_cmd} && deactivate"
+        if verbose:
+            typer.echo(f"Running command: {cmd}")
+        try:
+            subprocess.check_call(cmd, shell=True, cwd=wdir)
+        except subprocess.CalledProcessError:
+            raise_error("Failed to run in uv-venv")
     else:
         raise_error("Environment kind not supported")
 
@@ -605,7 +728,7 @@ def check_call(
     name="build-docker",
     help="Build Docker image if missing or different from lock file.",
 )
-def build_docker(
+def check_docker_env(
     tag: Annotated[str, typer.Argument(help="Image tag.")],
     fpath: Annotated[
         str, typer.Option("-i", "--input", help="Path to input Dockerfile.")
@@ -613,6 +736,9 @@ def build_docker(
     platform: Annotated[
         str, typer.Option("--platform", help="Which platform(s) to build for.")
     ] = None,
+    quiet: Annotated[
+        bool, typer.Option("--quiet", "-q", help="Be quiet.")
+    ] = False,
 ):
     def get_docker_inspect():
         out = json.loads(
@@ -626,28 +752,31 @@ def build_docker(
         _ = out[0].pop("DockerVersion")
         return out
 
-    typer.echo(f"Checking for existing image with tag {tag}")
+    outfile = open(os.devnull, "w") if quiet else None
+    typer.echo(f"Checking for existing image with tag {tag}", file=outfile)
     # First call Docker inspect
     try:
         inspect = get_docker_inspect()
     except subprocess.CalledProcessError:
-        typer.echo(f"No image with tag {tag} found locally")
+        typer.echo(f"No image with tag {tag} found locally", file=outfile)
         inspect = []
-    typer.echo(f"Reading Dockerfile from {fpath}")
+    typer.echo(f"Reading Dockerfile from {fpath}", file=outfile)
     with open(fpath) as f:
         dockerfile = f.read()
     dockerfile_md5 = hashlib.md5(dockerfile.encode()).hexdigest()
     lock_fpath = fpath + "-lock.json"
     rebuild = True
     if os.path.isfile(lock_fpath):
-        typer.echo(f"Reading lock file: {lock_fpath}")
+        typer.echo(f"Reading lock file: {lock_fpath}", file=outfile)
         with open(lock_fpath) as f:
             lock = json.load(f)
     else:
-        typer.echo(f"Lock file ({lock_fpath}) does not exist")
+        typer.echo(f"Lock file ({lock_fpath}) does not exist", file=outfile)
         lock = None
     if inspect and lock:
-        typer.echo("Checking image and Dockerfile against lock file")
+        typer.echo(
+            "Checking image and Dockerfile against lock file", file=outfile
+        )
         rebuild = inspect[0]["RootFS"]["Layers"] != lock[0]["RootFS"][
             "Layers"
         ] or dockerfile_md5 != lock[0].get("DockerfileMD5")
@@ -833,10 +962,17 @@ def check_conda_env(
             "--relaxed", help="Treat conda and pip dependencies as equivalent."
         ),
     ] = False,
+    quiet: Annotated[
+        bool, typer.Option("--quiet", "-q", help="Be quiet.")
+    ] = False,
 ):
+    if quiet:
+        log_func = functools.partial(typer.echo, file=open(os.devnull, "w"))
+    else:
+        log_func = typer.echo
     calkit.conda.check_env(
         env_fpath=env_fpath,
         output_fpath=output_fpath,
-        log_func=typer.echo,
+        log_func=log_func,
         relaxed=relaxed,
     )

{calkit_python-0.8.5 → calkit_python-0.9.1}/calkit/cli/new.py

@@ -216,7 +216,7 @@ def new_docker_env(
     layers: Annotated[
         list[str],
         typer.Option(
-            "--add-layer", help="Add a layer (options: mambaforge, foampy)."
+            "--add-layer", help="Add a layer (options: miniforge, foampy)."
         ),
     ] = [],
     wdir: Annotated[
@@ -623,9 +623,8 @@ def new_publication(
         envs[env_name] = env
         env_remote = dict(
             kind="docker",
-            image="kjarosh/latex:2024.4",
-            description="TeXlive full from kjarosh.",
-            platform="linux/amd64",
+            image="texlive/texlive:latest-full",
+            description="TeXlive full.",
         )
         with open(env_path, "w") as f:
             calkit.ryaml.dump(env_remote, f)
@@ -644,7 +643,10 @@ def new_publication(
         repo.git.add(path)
     # Create stage if applicable
     if stage_name is not None and template_type == "latex":
-        cmd = f"cd {path} && latexmk -pdf {template_obj.target}"
+        cmd = (
+            f"cd {path} && latexmk -interaction=nonstopmode "
+            f"-pdf {template_obj.target}"
+        )
         if env_name is not None:
             cmd = f'calkit runenv -n {env_name} "{cmd}"'
         target_dep = os.path.join(path, template_obj.target)

calkit_python-0.9.1/calkit/cli/update.py

@@ -0,0 +1,39 @@
+"""CLI for updating objects."""
+
+from __future__ import annotations
+
+import os
+
+import requests
+import typer
+from typing_extensions import Annotated
+
+update_app = typer.Typer(no_args_is_help=True)
+
+
+@update_app.command(name="devcontainer")
+def update_devcontainer(
+    wdir: Annotated[
+        str,
+        typer.Option(
+            "--wdir",
+            help=(
+                "Working directory. "
+                "By default will run current working directory."
+            ),
+        ),
+    ] = None,
+):
+    """Update a project's devcontainer to match the latest Calkit spec."""
+    url = (
+        "https://raw.githubusercontent.com/calkit/devcontainer/"
+        "refs/heads/main/devcontainer.json"
+    )
+    typer.echo(f"Downloading {url}")
+    resp = requests.get(url)
+    out_dir = os.path.join(wdir or ".", ".devcontainer")
+    os.makedirs(out_dir, exist_ok=True)
+    out_fpath = os.path.join(out_dir, "devcontainer.json")
+    typer.echo(f"Writing to {out_fpath}")
+    with open(out_fpath, "w") as f:
+        f.write(resp.text)

{calkit_python-0.8.5 → calkit_python-0.9.1}/calkit/config.py

@@ -7,8 +7,14 @@ from typing import Literal
 
 import keyring
 import yaml
+from keyring.errors import NoKeyringError
 from pydantic import computed_field
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+    YamlConfigSettingsSource,
+)
 
 
 def get_env() -> Literal["local", "staging", "production"]:
@@ -21,9 +27,9 @@ def set_env(name: Literal["local", "staging", "production"]) -> None:
     os.environ[f"{__package__.upper()}_ENV"] = name
 
 
-def get_env_suffix() -> str:
+def get_env_suffix(sep: str = "-") -> str:
     if get_env() != "production":
-        return "-" + get_env()
+        return sep + get_env()
     return ""
 
 
@@ -39,16 +45,35 @@ class Settings(BaseSettings):
             f"config{get_env_suffix()}.yaml",
         ),
         extra="ignore",
+        env_prefix="CALKIT" + get_env_suffix(sep="_") + "_",
     )
     username: str | None = None
     token: str | None = None
     dvc_token: str | None = None
     dataframe_engine: Literal["pandas", "polars"] = "pandas"
 
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource]:
+        return (
+            init_settings,
+            env_settings,
+            YamlConfigSettingsSource(settings_cls),
+        )
+
     @computed_field
     @property
-    def password(self) -> str:
-        return keyring.get_password(get_app_name(), self.username)
+    def password(self) -> str | None:
+        try:
+            return keyring.get_password(get_app_name(), self.username)
+        except NoKeyringError:
+            return None
 
     @password.setter
     def password(self, value: str) -> None:
@@ -66,8 +91,4 @@ class Settings(BaseSettings):
 
 def read() -> Settings:
     """Read the config."""
-    fpath = Settings.model_config["yaml_file"]
-    if not os.path.isfile(fpath):
-        return Settings()
-    with open(fpath) as f:
-        return Settings.model_validate(yaml.safe_load(f))
+    return Settings()

{calkit_python-0.8.5 → calkit_python-0.9.1}/calkit/docker.py

@@ -1,9 +1,9 @@
 """Functionality for working with Docker."""
 
-MAMBAFORGE_LAYER_TXT = r"""
+MINIFORGE_LAYER_TXT = r"""
 # Install Miniforge
-ARG MINIFORGE_NAME=Mambaforge
-ARG MINIFORGE_VERSION=24.3.0-0
+ARG MINIFORGE_NAME=Miniforge3
+ARG MINIFORGE_VERSION=24.9.2-0
 ARG TARGETPLATFORM
 
 ENV CONDA_DIR=/opt/conda
@@ -47,6 +47,7 @@ RUN pip install --no-cache-dir numpy pandas matplotlib h5py \
 """.strip()
 
 LAYERS = {
-    "mambaforge": MAMBAFORGE_LAYER_TXT,
+    "mambaforge": MINIFORGE_LAYER_TXT,
+    "miniforge": MINIFORGE_LAYER_TXT,
     "foampy": FOAMPY_LAYER_TEXT,
 }

{calkit_python-0.8.5 → calkit_python-0.9.1}/calkit/templates/latex/core.py

@@ -7,5 +7,6 @@ GITIGNORE = """
 *.log
 *.pdf
 *.out
+*.gz
 *.DS_Store
 """

{calkit_python-0.8.5 → calkit_python-0.9.1}/calkit/tests/cli/test_main.py

@@ -17,7 +17,7 @@ def test_run_in_env(tmp_dir):
         "--name my-image "
         "--stage build-image "
         "--from ubuntu "
-        "--add-layer mambaforge "
+        "--add-layer miniforge "
         "--description 'This is a test image'",
         shell=True,
     )
@@ -37,7 +37,7 @@ def test_run_in_env(tmp_dir):
         "--stage build-image-2 "
         "--path Dockerfile.2 "
         "--from ubuntu "
-        "--add-layer mambaforge "
+        "--add-layer miniforge "
         "--add-layer foampy "
         "--description 'This is a test image 2'",
         shell=True,

{calkit_python-0.8.5 → calkit_python-0.9.1}/calkit/tests/cli/test_new.py

@@ -167,5 +167,5 @@ def test_new_publication(tmp_dir):
     stage = dvc_pipeline["stages"]["build-latex-article"]
     assert stage["cmd"] == (
         "calkit runenv -n my-latex-env "
-        '"cd my-paper && latexmk -pdf paper.tex"'
+        '"cd my-paper && latexmk -interaction=nonstopmode -pdf paper.tex"'
     )

{calkit_python-0.8.5 → calkit_python-0.9.1}/docs/tutorials/conda-envs.md

@@ -69,16 +69,17 @@ If you run something like:
 
 ```sh
 calkit new conda-env \
-    -n my-project-py11 \
+    -n my-project-py311 \
     python=3.11 \
     pip \
     matplotlib \
     pandas \
     jupyter \
-    --pip tensorflow \
-    --stage check-conda-env
+    --pip tensorflow
 ```
 
-Calkit will create an environment definition in `calkit.yaml` for use with
-`calkit runenv`, and since `--stage` was specified, Calkit will also add
-an environment check stage to the pipeline automatically.
+Calkit will create an environment definition in `calkit.yaml`,
+which enables running a command in this environment with
+`calkit runenv -n my-project-py311 my-command-here`.
+That call will automatically create or update the Conda environment on the fly
+as needed and export a lock file describing the actual environment.