calkit-python 0.49.2__py3-none-any.whl → 0.49.3__py3-none-any.whl

calkit/agent_skills/conventions/SKILL.md

@@ -37,9 +37,8 @@ A minimal example:
 ```yaml
 environments:
   main:
-    kind: uv-venv
-    path: requirements.txt
-    python: "3.13"
+    kind: uv
+    path: pyproject.toml
 
 pipeline:
   stages:
@@ -61,7 +60,8 @@ kinds:
 
 | Kind      | Spec file                               |
 | --------- | --------------------------------------- |
-| `uv-venv` | `requirements.txt` or `pyproject.toml`  |
+| `uv`      | `pyproject.toml` (with `uv.lock`)       |
+| `uv-venv` | `requirements.txt`                      |
 | `venv`    | `requirements.txt`                      |
 | `conda`   | `environment.yml`                       |
 | `pixi`    | `pixi.toml`                             |
@@ -72,6 +72,12 @@ kinds:
 | `ssh`     | (remote host config)                    |
 | `slurm`   | (HPC cluster config)                    |
 
+For Python, default to a `uv` project environment, created with
+`calkit new uv-env -n <name> <packages>`, which writes `pyproject.toml` and
+`uv.lock`; set the Python version with `--python`, which becomes
+`requires-python`. Use `uv-venv` only for a project that already has a
+`requirements.txt` it should keep.
+
 Example environment definitions:
 
 ```yaml
@@ -79,12 +85,6 @@ environments:
   py:
     kind: uv
     path: pyproject.toml
-    python: "3.13"
-
-  py2:
-    kind: uv-venv
-    path: requirements.txt
-    python: "3.13"
 
   texlive:
     kind: docker

calkit/agent_skills/create-pipeline/SKILL.md

@@ -45,8 +45,9 @@ no `pipeline` section, skip this—you will add one.
 
 Every stage must reference a named environment. Identify what's needed:
 
-- **Python**: `requirements.txt` or `pyproject.toml` → `uv-venv`;
-  `environment.yml` → `conda`
+- **Python**: `pyproject.toml`, or no spec yet → `uv`, created with
+  `calkit new uv-env`; an existing `requirements.txt` the project should
+  keep → `uv-venv`; `environment.yml` → `conda`
 - **R**: `renv.lock` → `renv`
 - **Julia**: `Project.toml` → `julia`
 - **LaTeX**: `docker` with `texlive/texlive:latest-full`
@@ -57,9 +58,8 @@ Add environments to `calkit.yaml`:
 ```yaml
 environments:
   main:
-    kind: uv-venv
-    path: requirements.txt
-    python: "3.13"
+    kind: uv
+    path: pyproject.toml
 ```
 
 Name a single Python environment `main`. With multiple environments, use

calkit/cli/check.py

@@ -919,35 +919,49 @@ def check_environments(
         bool, typer.Option("--verbose", help="Print verbose output.")
     ] = False,
 ) -> None:
+    """Check that the project's environments are up to date.
+
+    Subprojects' environments are checked too, as 'calkit run' does.
+    """
     ck_info = calkit.load_calkit_info()
-    envs = ck_info.get("environments", {})
-    if not envs:
+    projects = list(calkit.iter_projects(ck_info=ck_info))
+    if not any(info.get("environments") for _, info in projects):
         typer.echo("No environments defined in calkit.yaml")
         return
     # Set any project-level environmental variables before checking
     # environments
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
-    calkit.set_env_vars(ck_info=ck_info, cli=True)
     failures = []
-    for env_name, env in envs.items():
-        if env.get("kind") in calkit.environments.KINDS_NO_CHECK:
-            if verbose:
-                typer.echo(
-                    f"Skipping check for {env['kind']} env '{env_name}'"
-                )
+    for wdir, info in projects:
+        # A parent's variables apply to its subprojects even if it has no
+        # environments of its own, as in 'calkit run'
+        calkit.set_env_vars(ck_info=info, cli=True)
+        if not info.get("environments"):
             continue
-        typer.echo(f"Checking environment: '{env_name}'")
-        try:
-            check_environment(env_name=env_name, verbose=verbose)
-        except typer.Exit:
-            # The check reported its own reason through raise_error, which
-            # has already printed it. Repeating it here would replace a
-            # real message ("Unknown system property to lock: ...") with
-            # the exit code, since that is all this exception carries.
-            failures.append(env_name)
-        except Exception as e:
-            warn(f"Error checking environment '{env_name}': {e}")
-            failures.append(env_name)
+        # check_environment reads the project it's in, so go there
+        with calkit.in_project(wdir):
+            for env_name, env in info["environments"].items():
+                if env.get("kind") in calkit.environments.KINDS_NO_CHECK:
+                    if verbose:
+                        typer.echo(
+                            f"Skipping check for {env['kind']} env "
+                            f"'{env_name}'"
+                        )
+                    continue
+                qualified = calkit.qualify_with_project(wdir, env_name)
+                typer.echo(f"Checking environment: '{qualified}'")
+                try:
+                    check_environment(env_name=env_name, verbose=verbose)
+                except typer.Exit:
+                    # The check reported its own reason through raise_error,
+                    # which has already printed it. Repeating it here would
+                    # replace a real message ("Unknown system property to
+                    # lock: ...") with the exit code, since that is all this
+                    # exception carries.
+                    failures.append(qualified)
+                except Exception as e:
+                    warn(f"Error checking environment '{qualified}': {e}")
+                    failures.append(qualified)
     if failures:
         raise_error(
             f"Failed to check the following environments: {', '.join(failures)}"
@@ -2195,15 +2209,21 @@ def check_project_requirements(
         ),
     ] = False,
 ) -> None:
-    """Check that a project's system-level requirements are met."""
+    """Check that a project's system-level requirements are met.
+
+    Subprojects' requirements are checked too, since the host has to meet
+    them for the project's pipeline to run.
+    """
     typer.echo("Checking project requirements")
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
     if no_cache:
         calkit.requirements.cache_clear()
-    try:
-        calkit.check_requirements(use_cache=not no_cache)
-    except Exception as e:
-        raise_error(str(e))
+    for wdir, info in calkit.iter_projects():
+        try:
+            with calkit.in_project(wdir):
+                calkit.check_requirements(ck_info=info, use_cache=not no_cache)
+        except Exception as e:
+            raise_error(f"{calkit.describe_project(wdir)}{e}")
     message = "✅ All set!"
     calkit.echo(message)
 
@@ -2219,34 +2239,71 @@ def check_stage_runs() -> None:
     mistakes and shortcuts, not forgery, since a matching entry can be
     written by anyone.
     """
+    # Every pipeline 'calkit run' runs with this one is checked too, e.g., a
+    # subproject's, against the record kept beside its dvc.lock: any other
+    # dvc.yaml in this DVC repo. One inside a DVC repo of its own is run by
+    # a wrapper stage, which doesn't record its stages.
     try:
-        with open(calkit.STAGE_RUNS_FPATH) as f:
-            stage_runs = json.load(f)
-    except OSError:
-        stage_runs = {}
-    except json.JSONDecodeError as e:
-        raise_error(f"Can't read {calkit.STAGE_RUNS_FPATH}: {e}")
-    # By directory, for a subproject's stages
-    lock_stages = {None: calkit.dvc.read_lock_stages()}
-    problems = []
-    for name in sorted(set(lock_stages[None]) | set(stage_runs)):
-        lock_dir, lock_name = calkit.dvc.split_lock_stage_name(name)
-        if lock_dir not in lock_stages:
-            lock_stages[lock_dir] = calkit.dvc.read_lock_stages(lock_dir)
-        locked = lock_stages[lock_dir].get(lock_name)
-        # One no longer in dvc.lock has nothing to vouch for
-        if not isinstance(locked, dict):
-            continue
-        recorded = stage_runs.get(name)
-        if not isinstance(recorded, dict):
-            problems.append(f"Stage '{name}' has no recorded run")
-        elif recorded.get("dvc_lock_hash") != calkit.dvc.stage_lock_hash(
-            locked
+        listed = subprocess.check_output(
+            [
+                "git",
+                "ls-files",
+                "--cached",
+                "--others",
+                "--exclude-standard",
+                "--",
+                "*/dvc.yaml",
+            ],
+            text=True,
+        ).splitlines()
+    except (OSError, subprocess.CalledProcessError):
+        listed = []
+    lock_dirs: list[str | None] = [None]
+    for path in sorted(listed):
+        parts = path.split("/")[:-1]
+        if not os.path.isfile(path) or any(
+            os.path.isdir(os.path.join(*parts[:i], ".dvc"))
+            for i in range(1, len(parts) + 1)
         ):
-            problems.append(
-                f"Stage '{name}' has changed in dvc.lock since its last "
-                "recorded run"
+            continue
+        lock_dirs.append("/".join(parts))
+    stage_runs: dict[str | None, dict] = {}
+    for lock_dir in lock_dirs:
+        try:
+            stage_runs[lock_dir] = calkit.read_stage_runs(lock_dir)
+        except json.JSONDecodeError as e:
+            fpath = os.path.join(lock_dir or "", calkit.STAGE_RUNS_FPATH)
+            raise_error(f"Can't read {fpath}: {e}")
+    # This project's records of other pipelines' stages, under the names
+    # they have here, from before they were kept beside their own dvc.lock
+    parent_runs = {
+        k.replace("\\", "/"): v for k, v in stage_runs[None].items()
+    }
+    problems = []
+    for lock_dir in lock_dirs:
+        lock_stages = calkit.dvc.read_lock_stages(lock_dir)
+        for lock_name, locked in sorted(lock_stages.items()):
+            if not isinstance(locked, dict):
+                continue
+            name = (
+                lock_name
+                if lock_dir is None
+                else f"{lock_dir}/dvc.yaml:{lock_name}"
             )
+            # Either record matching will do, since a stage may have been
+            # run from both directories before they were kept together
+            candidates = [stage_runs[lock_dir].get(lock_name)]
+            if lock_dir is not None:
+                candidates.append(parent_runs.get(name))
+            records = [r for r in candidates if isinstance(r, dict)]
+            lock_hash = calkit.dvc.stage_lock_hash(locked)
+            if not records:
+                problems.append(f"Stage '{name}' has no recorded run")
+            elif all(r.get("dvc_lock_hash") != lock_hash for r in records):
+                problems.append(
+                    f"Stage '{name}' has changed in dvc.lock since its last "
+                    "recorded run"
+                )
     for problem in problems:
         typer.echo(problem)
     if problems:
@@ -2264,23 +2321,31 @@ def check_env_vars(
         bool, typer.Option("--verbose", "-v", help="Print verbose output")
     ] = False,
 ) -> None:
-    """Check that the project's required environmental variables exist."""
+    """Check that the project's required environmental variables exist.
+
+    Subprojects' are included, since they are read from the same
+    environment.
+    """
     from calkit.core import _normalize_requirement
 
     typer.echo("Checking project environmental variables")
     dotenv.load_dotenv(dotenv_path=".env")
-    ck_info = calkit.load_calkit_info()
-    for raw in calkit.get_requirements(ck_info):
-        req = _normalize_requirement(raw)
-        if req["kind"] != "env-var":
-            continue
-        name = req["name"]
-        if verbose:
-            typer.echo(f"Checking for environmental variable '{name}'")
-        if name not in os.environ:
-            value = calkit.requirements.prompt_for_env_var_req(req)
-            if value is None:
-                raise_error(f"No value provided for '{name}'")
+    for wdir, info in calkit.iter_projects():
+        with calkit.in_project(wdir):
+            for raw in calkit.get_requirements(info):
+                req = _normalize_requirement(raw)
+                if req["kind"] != "env-var":
+                    continue
+                name = req["name"]
+                if verbose:
+                    typer.echo(f"Checking for environmental variable '{name}'")
+                if name not in os.environ:
+                    value = calkit.requirements.prompt_for_env_var_req(req)
+                    if value is None:
+                        raise_error(
+                            f"{calkit.describe_project(wdir)}no value "
+                            f"provided for '{name}'"
+                        )
     message = "✅ All set!"
     calkit.echo(message)
 
@@ -2296,21 +2361,34 @@ def check_pipeline(
         ),
     ] = False,
 ) -> None:
-    """Check that the project pipeline is defined correctly."""
+    """Check that the project pipeline is defined correctly.
+
+    Subprojects' pipelines are checked too, since those are stages the
+    project runs.
+    """
     from calkit.models.pipeline import Pipeline
 
     ck_info = calkit.load_calkit_info()
-    if "pipeline" not in ck_info:
+    projects = [
+        (wdir, info)
+        for wdir, info in calkit.iter_projects(ck_info=ck_info)
+        if "pipeline" in info
+    ]
+    if not projects:
         raise_error("No pipeline is defined in calkit.yaml")
-    try:
-        pipeline = Pipeline.model_validate(ck_info["pipeline"], strict=True)
-    except Exception as e:
-        raise_error(f"Pipeline is not defined correctly: {e}")
-    # Check that we have no leading underscores in stage names, since those
-    # are reserved for auto-generated stages
-    for stage_name in pipeline.stages.keys():
-        if stage_name.startswith("_"):
-            raise_error("Stage names cannot start with an underscore")
+    for wdir, info in projects:
+        where = calkit.describe_project(wdir)
+        try:
+            pipeline = Pipeline.model_validate(info["pipeline"], strict=True)
+        except Exception as e:
+            raise_error(f"{where}pipeline is not defined correctly: {e}")
+        # Check that we have no leading underscores in stage names, since
+        # those are reserved for auto-generated stages
+        for stage_name in pipeline.stages.keys():
+            if stage_name.startswith("_"):
+                raise_error(
+                    f"{where}stage names cannot start with an underscore"
+                )
     message = "✅ This project's pipeline is defined correctly!"
     calkit.echo(message)
     if compile_to_dvc:

calkit/cli/describe.py

@@ -109,6 +109,44 @@ def describe_envs(
         _echo_description(desc, indent="    ")
 
 
+@describe_app.command(name="subprojects")
+def describe_subprojects(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+) -> None:
+    """Describe subprojects, including nested ones.
+
+    Stages and environments are named as in the subproject itself; from
+    here, they're addressed as '<path>:<name>'.
+    """
+    subprojects = calkit.get_subprojects()
+    result = {}
+    for path, sp in subprojects.items():
+        info = sp["ck_info"]
+        result[path] = {
+            "parent": sp["parent"],
+            "description": sp["entry"].get("description"),
+            "name": info.get("name"),
+            "title": info.get("title"),
+            # With its own DVC repo, a subproject runs as one parent stage
+            "isolated": os.path.isdir(os.path.join(path, ".dvc")),
+            "environments": list(info.get("environments") or {}),
+            "stages": list((info.get("pipeline") or {}).get("stages") or {}),
+            "subprojects": [
+                p
+                for p, child in subprojects.items()
+                if child["parent"] == path
+            ],
+        }
+    if json_output:
+        echo_json(result)
+        return
+    for path, desc in result.items():
+        typer.echo(path + ":")
+        _echo_description(desc, indent="    ")
+
+
 @describe_app.command(name="schema")
 def describe_schema(
     output: Annotated[

calkit/cli/latex.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import glob
 import hashlib
 import json
 import os
@@ -646,6 +647,8 @@ def build(
     system environment if available. If not available, a TeX Live Docker
     container will be used.
     """
+    import time
+
     # latexmk records a failed run in its file database and then refuses
     # to try again, reporting "Nothing to do" and exiting non-zero with no
     # PDF. Running it again is the first thing anyone does after a
@@ -681,6 +684,7 @@ def build(
         rel = Path(os.path.relpath(aux_dir, tex_dir)).as_posix()
         latexmk_cmd.append(f"-auxdir={rel}")
     latexmk_cmd.append("-interaction=nonstopmode")
+    latexmk_cmd.append(f"-usepretex={calkit.latex.PDF_PRETEX}")
     # User pass-through args come last so they can override Calkit's defaults.
     latexmk_cmd += latexmk_args
     latexmk_cmd.append(tex_file)
@@ -694,6 +698,8 @@ def build(
         env_vars=tex_env_vars,
     )
     log_dir = aux_dir or output_dir or tex_dir
+    # A little early, for file systems that keep times coarsely
+    started = time.time() - 2
     if _run_latexmk(
         cmd,
         env=(os.environ | tex_env_vars) if tex_env_vars else None,
@@ -703,6 +709,39 @@ def build(
         verbose=verbose,
     ):
         raise_error("latexmk failed")
+    # A latexmkrc or a pass-through -outdir can put the PDF elsewhere, so
+    # whichever this build wrote gets its ID
+    pdf_fpaths = [os.path.join(pdf_dir, stem + ".pdf")] + glob.glob(
+        os.path.join(tex_dir, "**", stem + ".pdf"), recursive=True
+    )
+    for pdf_fpath in dict.fromkeys(map(os.path.normpath, pdf_fpaths)):
+        if (
+            os.path.isfile(pdf_fpath)
+            and os.path.getmtime(pdf_fpath) >= started
+        ):
+            calkit.latex.set_content_id(pdf_fpath)
+    if not no_synctex:
+        # A container build records its own paths, which reverse search in
+        # a viewer can't open
+        synctex_paths = [
+            os.path.join(pdf_dir, stem + ext)
+            for ext in (".synctex.gz", ".synctex")
+        ]
+        if latexmk_rc_path is not None and output_dir is None:
+            # The rc can set $out_dir, which Calkit doesn't read
+            synctex_paths += sorted(
+                glob.glob(
+                    os.path.join(tex_dir, "**", stem + ".synctex*"),
+                    recursive=True,
+                ),
+                key=os.path.getmtime,
+                reverse=True,
+            )
+        synctex_path = next(
+            (p for p in synctex_paths if os.path.isfile(p)), None
+        )
+        if synctex_path is not None:
+            calkit.latex.localize_synctex(synctex_path, tex_dir)
 
 
 DIFF_TMP_DIR = calkit.latex.DIFF_TMP_DIR
@@ -1760,6 +1799,7 @@ def _build_diff(
             "-interaction=nonstopmode",
             f"-auxdir={rel_aux}",
             f"-outdir={rel_aux}",
+            f"-usepretex={calkit.latex.PDF_PRETEX}",
         ]
         if not verbose:
             latexmk_cmd.append("-silent")
@@ -1828,6 +1868,7 @@ def _build_diff(
             )
         if not os.path.isfile(built):
             raise_error("latexmk did not produce a PDF")
+        calkit.latex.set_content_id(built)
         os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
         shutil.move(built, output)
         # A checkout is removed afterwards, but the working tree's document

calkit/cli/list.py

@@ -183,20 +183,23 @@ def list_presentations(
     _list_artifacts("presentations", json_output, declared_only)
 
 
-def _echo_question(n: int, question: str | dict) -> None:
+def _echo_question(n: int, question: str | dict, project: str = "") -> None:
     """Print a single question in the human-readable YAML-ish listing format.
 
     String questions are shown as a numbered line. Rich (dict) questions are
     shown with their text on the first line followed by hypothesis, answer,
-    evidence, and any other fields in YAML-like indented form.
+    evidence, and any other fields in YAML-like indented form. A
+    subproject's are numbered within it and prefixed with its path, as
+    ``calkit check questions`` names them.
     """
+    where = f"{project}: " if project else ""
     if isinstance(question, str):
-        typer.echo(f"{n}. {question}")
+        typer.echo(f"{where}{n}. {question}")
         return
     # Copy so popping 'question' doesn't mutate the caller's dict.
     question = dict(question)
     text = question.pop("question", "")
-    typer.echo(f"{n}. question: {text}")
+    typer.echo(f"{where}{n}. question: {text}")
     # Rendering fills every field, so leave out the ones that aren't set
     for k, v in question.items():
         if v is None:
@@ -244,8 +247,12 @@ def list_questions(
 
     Placeholders in the text, such as ``{improvement:.1f}``, are filled from
     the question's value evidence, so numbers shown are read from the
-    results files rather than retyped into ``calkit.yaml``.
+    results files rather than retyped into ``calkit.yaml``. Subprojects'
+    questions are listed too, numbered within each, and in the JSON output
+    each has a ``project`` key with the subproject's path.
     """
+    from pathlib import Path
+
     from calkit.questions import (
         TEMPLATED_FIELDS,
         placeholders,
@@ -260,10 +267,17 @@ def list_questions(
             if isinstance(ev, dict)
         ]
 
-    ck_info = calkit.load_calkit_info()
-    questions = ck_info.get("questions", []) or []
+    # Each with its 1-based index in its own project and that project's
+    # path, empty for this one
+    listed: list[tuple[int, str | dict, str]] = []
+    for wdir, info in calkit.iter_projects():
+        project = "" if wdir == "." else Path(wdir).as_posix()
+        for n, q in enumerate(info.get("questions", []) or [], start=1):
+            if not raw:
+                q = render_question(q, info, wdir=wdir)
+            listed.append((n, q, project))
+    questions = [q for _, q, _ in listed]
     if not raw:
-        rendered = [render_question(q, ck_info) for q in questions]
         # A placeholder left as written looks exactly like text somebody
         # meant literally, so say when one could not be filled rather than
         # let a fresh clone read as a project that types its braces. Any
@@ -273,7 +287,7 @@ def list_questions(
         # answer still in clauses is one whose conditions could not be read.
         unfilled = any(
             isinstance(text, dict) or placeholders(text)
-            for q in rendered
+            for q in questions
             if isinstance(q, dict)
             for text in _texts(q)
         )
@@ -285,12 +299,20 @@ def list_questions(
                 "the results files are not here yet.",
                 err=json_output,
             )
-        questions = rendered
     if json_output:
-        echo_json(questions)
+        echo_json(
+            [
+                q
+                if not project
+                else {"project": project, "question": q}
+                if isinstance(q, str)
+                else {"project": project, **q}
+                for _, q, project in listed
+            ]
+        )
         return
-    for n, question in enumerate(questions, start=1):
-        _echo_question(n, question)
+    for n, question, project in listed:
+        _echo_question(n, question, project)
 
 
 @list_app.command(name="comments")
@@ -639,9 +661,16 @@ def list_stages(
         bool, typer.Option("--json", help="Output result as JSON.")
     ] = False,
 ):
-    """List pipeline stages."""
+    """List pipeline stages, including subprojects'."""
     ck_info = calkit.load_calkit_info()
-    stages = ck_info.get("pipeline", {}).get("stages", {})
+    # Named as 'calkit status' names them and 'calkit run' accepts them
+    stages = {
+        calkit.qualify_with_project(wdir, name): stage
+        for wdir, info in calkit.iter_projects(ck_info=ck_info)
+        for name, stage in (
+            (info.get("pipeline") or {}).get("stages") or {}
+        ).items()
+    }
     # If we only want stale stages, we need to get the status first.
     # This compiles the pipeline, cleans notebooks, and checks environments,
     # all of which can affect whether a stage is stale, so we don't skip them.
@@ -666,6 +695,29 @@ def list_stages(
         typer.echo(name)
 
 
+@list_app.command(name="subprojects")
+def list_subprojects(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+) -> None:
+    """List subprojects, including nested ones, by path."""
+    subprojects = calkit.get_subprojects()
+    if json_output:
+        echo_json(
+            {
+                path: {
+                    "parent": sp["parent"],
+                    "description": sp["entry"].get("description"),
+                }
+                for path, sp in subprojects.items()
+            }
+        )
+        return
+    for path in subprojects:
+        typer.echo(path)
+
+
 @list_app.command(name="latex-diffs")
 def list_latex_diffs(
     json_output: Annotated[