calkit-python 0.49.2__py3-none-any.whl → 0.49.4__py3-none-any.whl

calkit/agent_skills/conventions/SKILL.md

@@ -37,9 +37,8 @@ A minimal example:
 ```yaml
 environments:
   main:
-    kind: uv-venv
-    path: requirements.txt
-    python: "3.13"
+    kind: uv
+    path: pyproject.toml
 
 pipeline:
   stages:
@@ -61,7 +60,8 @@ kinds:
 
 | Kind      | Spec file                               |
 | --------- | --------------------------------------- |
-| `uv-venv` | `requirements.txt` or `pyproject.toml`  |
+| `uv`      | `pyproject.toml` (with `uv.lock`)       |
+| `uv-venv` | `requirements.txt`                      |
 | `venv`    | `requirements.txt`                      |
 | `conda`   | `environment.yml`                       |
 | `pixi`    | `pixi.toml`                             |
@@ -72,6 +72,12 @@ kinds:
 | `ssh`     | (remote host config)                    |
 | `slurm`   | (HPC cluster config)                    |
 
+For Python, default to a `uv` project environment, created with
+`calkit new uv-env -n <name> <packages>`, which writes `pyproject.toml` and
+`uv.lock`; set the Python version with `--python`, which becomes
+`requires-python`. Use `uv-venv` only for a project that already has a
+`requirements.txt` it should keep.
+
 Example environment definitions:
 
 ```yaml
@@ -79,12 +85,6 @@ environments:
   py:
     kind: uv
     path: pyproject.toml
-    python: "3.13"
-
-  py2:
-    kind: uv-venv
-    path: requirements.txt
-    python: "3.13"
 
   texlive:
     kind: docker

calkit/agent_skills/create-pipeline/SKILL.md

@@ -45,8 +45,9 @@ no `pipeline` section, skip this—you will add one.
 
 Every stage must reference a named environment. Identify what's needed:
 
-- **Python**: `requirements.txt` or `pyproject.toml` → `uv-venv`;
-  `environment.yml` → `conda`
+- **Python**: `pyproject.toml`, or no spec yet → `uv`, created with
+  `calkit new uv-env`; an existing `requirements.txt` the project should
+  keep → `uv-venv`; `environment.yml` → `conda`
 - **R**: `renv.lock` → `renv`
 - **Julia**: `Project.toml` → `julia`
 - **LaTeX**: `docker` with `texlive/texlive:latest-full`
@@ -57,9 +58,8 @@ Add environments to `calkit.yaml`:
 ```yaml
 environments:
   main:
-    kind: uv-venv
-    path: requirements.txt
-    python: "3.13"
+    kind: uv
+    path: pyproject.toml
 ```
 
 Name a single Python environment `main`. With multiple environments, use

calkit/cli/__init__.py

@@ -19,6 +19,16 @@ def run() -> None:
     # otherwise print the *current* CLI's help instead of re-execing
     # under the requested version.
     _maybe_exec_with_version()
+    if sys.platform == "win32":
+        import io
+
+        from calkit.core import is_mintty
+
+        # mintty shows UTF-8, so emoji needn't be replaced, and output is
+        # line buffered as it would be to a terminal
+        for stream in (sys.stdout, sys.stderr):
+            if isinstance(stream, io.TextIOWrapper) and is_mintty(stream):
+                stream.reconfigure(encoding="utf-8", line_buffering=True)
     from .main import app
 
     # Completion lists commands, so needs them all

calkit/cli/check.py

@@ -919,35 +919,49 @@ def check_environments(
         bool, typer.Option("--verbose", help="Print verbose output.")
     ] = False,
 ) -> None:
+    """Check that the project's environments are up to date.
+
+    Subprojects' environments are checked too, as 'calkit run' does.
+    """
     ck_info = calkit.load_calkit_info()
-    envs = ck_info.get("environments", {})
-    if not envs:
+    projects = list(calkit.iter_projects(ck_info=ck_info))
+    if not any(info.get("environments") for _, info in projects):
         typer.echo("No environments defined in calkit.yaml")
         return
     # Set any project-level environmental variables before checking
     # environments
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
-    calkit.set_env_vars(ck_info=ck_info, cli=True)
     failures = []
-    for env_name, env in envs.items():
-        if env.get("kind") in calkit.environments.KINDS_NO_CHECK:
-            if verbose:
-                typer.echo(
-                    f"Skipping check for {env['kind']} env '{env_name}'"
-                )
+    for wdir, info in projects:
+        # A parent's variables apply to its subprojects even if it has no
+        # environments of its own, as in 'calkit run'
+        calkit.set_env_vars(ck_info=info, cli=True)
+        if not info.get("environments"):
             continue
-        typer.echo(f"Checking environment: '{env_name}'")
-        try:
-            check_environment(env_name=env_name, verbose=verbose)
-        except typer.Exit:
-            # The check reported its own reason through raise_error, which
-            # has already printed it. Repeating it here would replace a
-            # real message ("Unknown system property to lock: ...") with
-            # the exit code, since that is all this exception carries.
-            failures.append(env_name)
-        except Exception as e:
-            warn(f"Error checking environment '{env_name}': {e}")
-            failures.append(env_name)
+        # check_environment reads the project it's in, so go there
+        with calkit.in_project(wdir):
+            for env_name, env in info["environments"].items():
+                if env.get("kind") in calkit.environments.KINDS_NO_CHECK:
+                    if verbose:
+                        typer.echo(
+                            f"Skipping check for {env['kind']} env "
+                            f"'{env_name}'"
+                        )
+                    continue
+                qualified = calkit.qualify_with_project(wdir, env_name)
+                typer.echo(f"Checking environment: '{qualified}'")
+                try:
+                    check_environment(env_name=env_name, verbose=verbose)
+                except typer.Exit:
+                    # The check reported its own reason through raise_error,
+                    # which has already printed it. Repeating it here would
+                    # replace a real message ("Unknown system property to
+                    # lock: ...") with the exit code, since that is all this
+                    # exception carries.
+                    failures.append(qualified)
+                except Exception as e:
+                    warn(f"Error checking environment '{qualified}': {e}")
+                    failures.append(qualified)
     if failures:
         raise_error(
             f"Failed to check the following environments: {', '.join(failures)}"
@@ -2195,15 +2209,21 @@ def check_project_requirements(
         ),
     ] = False,
 ) -> None:
-    """Check that a project's system-level requirements are met."""
+    """Check that a project's system-level requirements are met.
+
+    Subprojects' requirements are checked too, since the host has to meet
+    them for the project's pipeline to run.
+    """
     typer.echo("Checking project requirements")
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
     if no_cache:
         calkit.requirements.cache_clear()
-    try:
-        calkit.check_requirements(use_cache=not no_cache)
-    except Exception as e:
-        raise_error(str(e))
+    for wdir, info in calkit.iter_projects():
+        try:
+            with calkit.in_project(wdir):
+                calkit.check_requirements(ck_info=info, use_cache=not no_cache)
+        except Exception as e:
+            raise_error(f"{calkit.describe_project(wdir)}{e}")
     message = "✅ All set!"
     calkit.echo(message)
 
@@ -2219,34 +2239,71 @@ def check_stage_runs() -> None:
     mistakes and shortcuts, not forgery, since a matching entry can be
     written by anyone.
     """
+    # Every pipeline 'calkit run' runs with this one is checked too, e.g., a
+    # subproject's, against the record kept beside its dvc.lock: any other
+    # dvc.yaml in this DVC repo. One inside a DVC repo of its own is run by
+    # a wrapper stage, which doesn't record its stages.
     try:
-        with open(calkit.STAGE_RUNS_FPATH) as f:
-            stage_runs = json.load(f)
-    except OSError:
-        stage_runs = {}
-    except json.JSONDecodeError as e:
-        raise_error(f"Can't read {calkit.STAGE_RUNS_FPATH}: {e}")
-    # By directory, for a subproject's stages
-    lock_stages = {None: calkit.dvc.read_lock_stages()}
-    problems = []
-    for name in sorted(set(lock_stages[None]) | set(stage_runs)):
-        lock_dir, lock_name = calkit.dvc.split_lock_stage_name(name)
-        if lock_dir not in lock_stages:
-            lock_stages[lock_dir] = calkit.dvc.read_lock_stages(lock_dir)
-        locked = lock_stages[lock_dir].get(lock_name)
-        # One no longer in dvc.lock has nothing to vouch for
-        if not isinstance(locked, dict):
-            continue
-        recorded = stage_runs.get(name)
-        if not isinstance(recorded, dict):
-            problems.append(f"Stage '{name}' has no recorded run")
-        elif recorded.get("dvc_lock_hash") != calkit.dvc.stage_lock_hash(
-            locked
+        listed = subprocess.check_output(
+            [
+                "git",
+                "ls-files",
+                "--cached",
+                "--others",
+                "--exclude-standard",
+                "--",
+                "*/dvc.yaml",
+            ],
+            text=True,
+        ).splitlines()
+    except (OSError, subprocess.CalledProcessError):
+        listed = []
+    lock_dirs: list[str | None] = [None]
+    for path in sorted(listed):
+        parts = path.split("/")[:-1]
+        if not os.path.isfile(path) or any(
+            os.path.isdir(os.path.join(*parts[:i], ".dvc"))
+            for i in range(1, len(parts) + 1)
         ):
-            problems.append(
-                f"Stage '{name}' has changed in dvc.lock since its last "
-                "recorded run"
+            continue
+        lock_dirs.append("/".join(parts))
+    stage_runs: dict[str | None, dict] = {}
+    for lock_dir in lock_dirs:
+        try:
+            stage_runs[lock_dir] = calkit.read_stage_runs(lock_dir)
+        except json.JSONDecodeError as e:
+            fpath = os.path.join(lock_dir or "", calkit.STAGE_RUNS_FPATH)
+            raise_error(f"Can't read {fpath}: {e}")
+    # This project's records of other pipelines' stages, under the names
+    # they have here, from before they were kept beside their own dvc.lock
+    parent_runs = {
+        k.replace("\\", "/"): v for k, v in stage_runs[None].items()
+    }
+    problems = []
+    for lock_dir in lock_dirs:
+        lock_stages = calkit.dvc.read_lock_stages(lock_dir)
+        for lock_name, locked in sorted(lock_stages.items()):
+            if not isinstance(locked, dict):
+                continue
+            name = (
+                lock_name
+                if lock_dir is None
+                else f"{lock_dir}/dvc.yaml:{lock_name}"
             )
+            # Either record matching will do, since a stage may have been
+            # run from both directories before they were kept together
+            candidates = [stage_runs[lock_dir].get(lock_name)]
+            if lock_dir is not None:
+                candidates.append(parent_runs.get(name))
+            records = [r for r in candidates if isinstance(r, dict)]
+            lock_hash = calkit.dvc.stage_lock_hash(locked)
+            if not records:
+                problems.append(f"Stage '{name}' has no recorded run")
+            elif all(r.get("dvc_lock_hash") != lock_hash for r in records):
+                problems.append(
+                    f"Stage '{name}' has changed in dvc.lock since its last "
+                    "recorded run"
+                )
     for problem in problems:
         typer.echo(problem)
     if problems:
@@ -2264,23 +2321,31 @@ def check_env_vars(
         bool, typer.Option("--verbose", "-v", help="Print verbose output")
     ] = False,
 ) -> None:
-    """Check that the project's required environmental variables exist."""
+    """Check that the project's required environmental variables exist.
+
+    Subprojects' are included, since they are read from the same
+    environment.
+    """
     from calkit.core import _normalize_requirement
 
     typer.echo("Checking project environmental variables")
     dotenv.load_dotenv(dotenv_path=".env")
-    ck_info = calkit.load_calkit_info()
-    for raw in calkit.get_requirements(ck_info):
-        req = _normalize_requirement(raw)
-        if req["kind"] != "env-var":
-            continue
-        name = req["name"]
-        if verbose:
-            typer.echo(f"Checking for environmental variable '{name}'")
-        if name not in os.environ:
-            value = calkit.requirements.prompt_for_env_var_req(req)
-            if value is None:
-                raise_error(f"No value provided for '{name}'")
+    for wdir, info in calkit.iter_projects():
+        with calkit.in_project(wdir):
+            for raw in calkit.get_requirements(info):
+                req = _normalize_requirement(raw)
+                if req["kind"] != "env-var":
+                    continue
+                name = req["name"]
+                if verbose:
+                    typer.echo(f"Checking for environmental variable '{name}'")
+                if name not in os.environ:
+                    value = calkit.requirements.prompt_for_env_var_req(req)
+                    if value is None:
+                        raise_error(
+                            f"{calkit.describe_project(wdir)}no value "
+                            f"provided for '{name}'"
+                        )
     message = "✅ All set!"
     calkit.echo(message)
 
@@ -2296,21 +2361,34 @@ def check_pipeline(
         ),
     ] = False,
 ) -> None:
-    """Check that the project pipeline is defined correctly."""
+    """Check that the project pipeline is defined correctly.
+
+    Subprojects' pipelines are checked too, since those are stages the
+    project runs.
+    """
     from calkit.models.pipeline import Pipeline
 
     ck_info = calkit.load_calkit_info()
-    if "pipeline" not in ck_info:
+    projects = [
+        (wdir, info)
+        for wdir, info in calkit.iter_projects(ck_info=ck_info)
+        if "pipeline" in info
+    ]
+    if not projects:
         raise_error("No pipeline is defined in calkit.yaml")
-    try:
-        pipeline = Pipeline.model_validate(ck_info["pipeline"], strict=True)
-    except Exception as e:
-        raise_error(f"Pipeline is not defined correctly: {e}")
-    # Check that we have no leading underscores in stage names, since those
-    # are reserved for auto-generated stages
-    for stage_name in pipeline.stages.keys():
-        if stage_name.startswith("_"):
-            raise_error("Stage names cannot start with an underscore")
+    for wdir, info in projects:
+        where = calkit.describe_project(wdir)
+        try:
+            pipeline = Pipeline.model_validate(info["pipeline"], strict=True)
+        except Exception as e:
+            raise_error(f"{where}pipeline is not defined correctly: {e}")
+        # Check that we have no leading underscores in stage names, since
+        # those are reserved for auto-generated stages
+        for stage_name in pipeline.stages.keys():
+            if stage_name.startswith("_"):
+                raise_error(
+                    f"{where}stage names cannot start with an underscore"
+                )
     message = "✅ This project's pipeline is defined correctly!"
     calkit.echo(message)
     if compile_to_dvc:

calkit/cli/core.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import os
 import re
 import subprocess
+import sys
 from enum import Enum
 from typing import Any, NoReturn
 
@@ -152,8 +153,24 @@ def echo_json(obj: Any) -> None:
     typer.echo(json.dumps(obj, default=str))
 
 
+def _echo_above_spinner(text: str, err: bool) -> bool:
+    """Print through a spinner's stand-in for the stream, if there is one.
+
+    Click writes to a Windows console's handle directly, which would put the
+    text on the spinner's line rather than above it.
+    """
+    file_proxy = sys.modules.get("rich.file_proxy")
+    stream = sys.stderr if err else sys.stdout
+    if file_proxy is None or not isinstance(stream, file_proxy.FileProxy):
+        return False
+    print(text, file=stream, flush=True)
+    return True
+
+
 def raise_error(txt: str) -> NoReturn:
-    typer.echo(typer.style("Error: " + str(txt), fg="red"), err=True)
+    text = typer.style("Error: " + str(txt), fg="red")
+    if not _echo_above_spinner(text, err=True):
+        typer.echo(text, err=True)
     raise typer.Exit(1)
 
 
@@ -164,7 +181,6 @@ def warn(txt: str, prefix: str = "Warning: ", err: bool = False):
     # defined.
     from calkit.core import encode_safe
 
-    typer.echo(
-        typer.style(prefix + encode_safe(str(txt), err=err), fg="yellow"),
-        err=err,
-    )
+    text = typer.style(prefix + encode_safe(str(txt), err=err), fg="yellow")
+    if not _echo_above_spinner(text, err=err):
+        typer.echo(text, err=err)

calkit/cli/describe.py

@@ -109,6 +109,44 @@ def describe_envs(
         _echo_description(desc, indent="    ")
 
 
+@describe_app.command(name="subprojects")
+def describe_subprojects(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+) -> None:
+    """Describe subprojects, including nested ones.
+
+    Stages and environments are named as in the subproject itself; from
+    here, they're addressed as '<path>:<name>'.
+    """
+    subprojects = calkit.get_subprojects()
+    result = {}
+    for path, sp in subprojects.items():
+        info = sp["ck_info"]
+        result[path] = {
+            "parent": sp["parent"],
+            "description": sp["entry"].get("description"),
+            "name": info.get("name"),
+            "title": info.get("title"),
+            # With its own DVC repo, a subproject runs as one parent stage
+            "isolated": os.path.isdir(os.path.join(path, ".dvc")),
+            "environments": list(info.get("environments") or {}),
+            "stages": list((info.get("pipeline") or {}).get("stages") or {}),
+            "subprojects": [
+                p
+                for p, child in subprojects.items()
+                if child["parent"] == path
+            ],
+        }
+    if json_output:
+        echo_json(result)
+        return
+    for path, desc in result.items():
+        typer.echo(path + ":")
+        _echo_description(desc, indent="    ")
+
+
 @describe_app.command(name="schema")
 def describe_schema(
     output: Annotated[

calkit/cli/latex.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import glob
 import hashlib
 import json
 import os
@@ -646,6 +647,8 @@ def build(
     system environment if available. If not available, a TeX Live Docker
     container will be used.
     """
+    import time
+
     # latexmk records a failed run in its file database and then refuses
     # to try again, reporting "Nothing to do" and exiting non-zero with no
     # PDF. Running it again is the first thing anyone does after a
@@ -681,6 +684,7 @@ def build(
         rel = Path(os.path.relpath(aux_dir, tex_dir)).as_posix()
         latexmk_cmd.append(f"-auxdir={rel}")
     latexmk_cmd.append("-interaction=nonstopmode")
+    latexmk_cmd.append(f"-usepretex={calkit.latex.PDF_PRETEX}")
     # User pass-through args come last so they can override Calkit's defaults.
     latexmk_cmd += latexmk_args
     latexmk_cmd.append(tex_file)
@@ -694,6 +698,8 @@ def build(
         env_vars=tex_env_vars,
     )
     log_dir = aux_dir or output_dir or tex_dir
+    # A little early, for file systems that keep times coarsely
+    started = time.time() - 2
     if _run_latexmk(
         cmd,
         env=(os.environ | tex_env_vars) if tex_env_vars else None,
@@ -703,6 +709,39 @@ def build(
         verbose=verbose,
     ):
         raise_error("latexmk failed")
+    # A latexmkrc or a pass-through -outdir can put the PDF elsewhere, so
+    # whichever this build wrote gets its ID
+    pdf_fpaths = [os.path.join(pdf_dir, stem + ".pdf")] + glob.glob(
+        os.path.join(tex_dir, "**", stem + ".pdf"), recursive=True
+    )
+    for pdf_fpath in dict.fromkeys(map(os.path.normpath, pdf_fpaths)):
+        if (
+            os.path.isfile(pdf_fpath)
+            and os.path.getmtime(pdf_fpath) >= started
+        ):
+            calkit.latex.set_content_id(pdf_fpath)
+    if not no_synctex:
+        # A container build records its own paths, which reverse search in
+        # a viewer can't open
+        synctex_paths = [
+            os.path.join(pdf_dir, stem + ext)
+            for ext in (".synctex.gz", ".synctex")
+        ]
+        if latexmk_rc_path is not None and output_dir is None:
+            # The rc can set $out_dir, which Calkit doesn't read
+            synctex_paths += sorted(
+                glob.glob(
+                    os.path.join(tex_dir, "**", stem + ".synctex*"),
+                    recursive=True,
+                ),
+                key=os.path.getmtime,
+                reverse=True,
+            )
+        synctex_path = next(
+            (p for p in synctex_paths if os.path.isfile(p)), None
+        )
+        if synctex_path is not None:
+            calkit.latex.localize_synctex(synctex_path, tex_dir)
 
 
 DIFF_TMP_DIR = calkit.latex.DIFF_TMP_DIR
@@ -1760,6 +1799,7 @@ def _build_diff(
             "-interaction=nonstopmode",
             f"-auxdir={rel_aux}",
             f"-outdir={rel_aux}",
+            f"-usepretex={calkit.latex.PDF_PRETEX}",
         ]
         if not verbose:
             latexmk_cmd.append("-silent")
@@ -1828,6 +1868,7 @@ def _build_diff(
             )
         if not os.path.isfile(built):
             raise_error("latexmk did not produce a PDF")
+        calkit.latex.set_content_id(built)
         os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
         shutil.move(built, output)
         # A checkout is removed afterwards, but the working tree's document