calkit-python 0.41.13__tar.gz → 0.41.15__tar.gz

{calkit_python-0.41.13 → calkit_python-0.41.15}/AGENTS.md

@@ -43,3 +43,6 @@ The only exception here is if splitting up a function makes it easier to
 write meaningful unit tests.
 If a function is only used inside one other function, nest it inside the
 caller at the top.
+
+No spaces on either side of en or em dashes used for ranges or pauses in prose,
+respectively.

{calkit_python-0.41.13 → calkit_python-0.41.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.41.13
+Version: 0.41.15
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.41.13 → calkit_python-0.41.15}/calkit/cli/check.py

@@ -180,7 +180,8 @@ def _check_julia_env(
         try:
             subprocess.check_call(
                 cmd,
-                env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"},
+                env=os.environ.copy()
+                | {"JULIA_LOAD_PATH": calkit.julia.load_path()},
             )
         except subprocess.CalledProcessError:
             calkit.environments.save_cache(
@@ -206,7 +207,9 @@ def _check_julia_env(
     cmd = calkit.julia.ensure_startup_file_disabled_in_command(cmd)
     try:
         subprocess.check_call(
-            cmd, env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"}
+            cmd,
+            env=os.environ.copy()
+            | {"JULIA_LOAD_PATH": calkit.julia.load_path()},
         )
     except subprocess.CalledProcessError:
         calkit.environments.save_cache(

calkit_python-0.41.15/calkit/cli/delete.py

@@ -0,0 +1,33 @@
+"""CLI for deleting objects."""
+
+from __future__ import annotations
+
+import typer
+
+import calkit
+from calkit.cli import AliasGroup, raise_error
+from calkit.core import ryaml
+
+delete_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
+
+
+@delete_app.command(name="question")
+def delete_question(
+    index: int = typer.Argument(
+        ...,
+        help="1-based index of the question to remove (see `calkit list questions`).",
+    ),
+):
+    """Remove a question by its 1-based index."""
+    ck_info = calkit.load_calkit_info()
+    questions = ck_info.get("questions", []) or []
+    if index < 1 or index > len(questions):
+        raise_error(
+            f"Invalid question index {index}; "
+            f"there are {len(questions)} question(s)."
+        )
+    removed = questions.pop(index - 1)
+    ck_info["questions"] = questions
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    typer.echo(f"Removed question: {removed}")

{calkit_python-0.41.13 → calkit_python-0.41.15}/calkit/cli/list.py

@@ -55,7 +55,7 @@ def _list_objects(
 
 
 def _list_artifacts(
-    kind: Literal["figures", "datasets"],
+    kind: Literal["figures", "datasets", "results", "presentations"],
     json_output: bool,
     declared_only: bool,
 ):
@@ -134,6 +134,61 @@ def list_datasets(
     _list_artifacts("datasets", json_output, declared_only)
 
 
+@list_app.command(name="results")
+def list_results(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+    declared_only: Annotated[
+        bool,
+        typer.Option(
+            "--declared-only",
+            help=(
+                "Only list results declared in calkit.yaml; "
+                "skip auto-detection."
+            ),
+        ),
+    ] = False,
+):
+    """List results in the project."""
+    _list_artifacts("results", json_output, declared_only)
+
+
+@list_app.command(name="presentations|pres")
+def list_presentations(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+    declared_only: Annotated[
+        bool,
+        typer.Option(
+            "--declared-only",
+            help=(
+                "Only list presentations declared in calkit.yaml; "
+                "skip auto-detection."
+            ),
+        ),
+    ] = False,
+):
+    """List presentations in the project."""
+    _list_artifacts("presentations", json_output, declared_only)
+
+
+@list_app.command(name="questions")
+def list_questions(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+):
+    """List the project's questions (1-indexed)."""
+    questions = calkit.load_calkit_info().get("questions", []) or []
+    if json_output:
+        typer.echo(json.dumps(questions))
+        return
+    for n, question in enumerate(questions, start=1):
+        typer.echo(f"{n}. {question}")
+
+
 @list_app.command(name="publications|pubs")
 def list_publications():
     """List publications in the project."""

{calkit_python-0.41.13 → calkit_python-0.41.15}/calkit/cli/main/core.py

@@ -48,6 +48,7 @@ from calkit.cli.check import (
 )
 from calkit.cli.cloud import cloud_app
 from calkit.cli.config import config_app
+from calkit.cli.delete import delete_app
 from calkit.cli.describe import describe_app
 from calkit.cli.dev import dev_app
 from calkit.cli.import_ import import_app
@@ -69,6 +70,7 @@ app = typer.Typer(
 )
 app.add_typer(config_app, name="config", help="Configure Calkit.")
 app.add_typer(new_app, name="new|create", help="Create a new Calkit object.")
+app.add_typer(delete_app, name="delete|rm", help="Delete a Calkit object.")
 app.add_typer(
     notebooks_app, name="notebooks|nb", help="Work with Jupyter notebooks."
 )
@@ -579,6 +581,18 @@ def get_status(
         if pipeline_status and not pipeline_status.has_pipeline:
             typer.echo("This project has no pipeline.")
             return
+        # A failed environment check makes get_status bail out before computing
+        # staleness, leaving stale_stages empty. Don't let that masquerade as an
+        # up-to-date pipeline---say plainly that the status is unknown.
+        elif pipeline_status and pipeline_status.failed_environment_checks:
+            # The failing environments were already named in the warning emitted
+            # right after computing pipeline_status, so keep this explanatory
+            # only rather than repeating the list.
+            typer.echo(
+                "Pipeline status unavailable until the failed environment "
+                "checks above are resolved."
+            )
+            return
         elif pipeline_status and pipeline_status.is_stale:
             typer.echo("Stale stages:")
             for stage_name in pipeline_status.stale_stage_names:
@@ -1172,7 +1186,10 @@ def pull(
         and "-R" not in dvc_args
     ):
         dvc_args.append("--recursive")
-    result = calkit.dvc.run_dvc_command(["pull"] + dvc_args, lock_retries=5)
+    result = calkit.dvc.run_dvc_command(
+        ["pull"] + dvc_args,
+        lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
+    )
     if result != 0:
         raise_error("DVC pull failed")
     calkit.dvc.zip.sync_all(direction="to-workspace")
@@ -1187,7 +1204,9 @@ def pull(
                 continue
             typer.echo(f"DVC pulling subproject: {sp_path}")
             sp_result = calkit.dvc.run_dvc_command(
-                ["pull"] + dvc_args, cwd=sp_path, lock_retries=5
+                ["pull"] + dvc_args,
+                cwd=sp_path,
+                lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
             )
             if sp_result != 0:
                 raise_error(f"DVC pull failed for subproject: {sp_path}")
@@ -1225,7 +1244,8 @@ def push(
         if remotes:
             typer.echo("Pushing to DVC remote")
             result = calkit.dvc.run_dvc_command(
-                ["push"] + dvc_args, lock_retries=5
+                ["push"] + dvc_args,
+                lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
             )
             if result != 0:
                 raise_error("DVC push failed")
@@ -2035,7 +2055,10 @@ def run(
             original_dir = os.getcwd()
             try:
                 os.chdir(sp_path)
-                sp_res = dvc_cli_main(["repro"] + sp_args)
+                with calkit.dvc.dvc_lock_timeout(
+                    calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT
+                ):
+                    sp_res = dvc_cli_main(["repro"] + sp_args)
             finally:
                 os.chdir(original_dir)
             if sp_res != 0:
@@ -2104,7 +2127,15 @@ def run(
     if force:
         os.environ["CALKIT_FORCE"] = "1"
     try:
-        res = dvc_cli_main(["repro"] + args)
+        # Wait generously for the repo lock instead of failing after DVC's 3s
+        # default. Brief contention is common and benign: a background
+        # `calkit status` poller (e.g. the VS Code extension), a stage whose
+        # command is itself `calkit run`, or DVC's own per-stage re-lock (DVC
+        # releases the repo lock while a stage command runs and re-acquires it
+        # the instant it finishes) can all hold it momentarily. Without this,
+        # such a collision aborts the whole run with "Unable to acquire lock".
+        with calkit.dvc.dvc_lock_timeout(calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT):
+            res = dvc_cli_main(["repro"] + args)
     finally:
         os.environ.pop("CALKIT_FORCE", None)
     failed = failed or res != 0
@@ -2567,7 +2598,8 @@ def run_in_env(
             subprocess.check_call(
                 julia_cmd,
                 cwd=wdir,
-                env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"},
+                env=os.environ.copy()
+                | {"JULIA_LOAD_PATH": calkit.julia.load_path()},
             )
         except subprocess.CalledProcessError:
             raise_error("Failed to run in julia environment")

{calkit_python-0.41.13 → calkit_python-0.41.15}/calkit/cli/new.py

@@ -9,6 +9,7 @@ import shutil
 import subprocess
 import time
 from enum import Enum
+from typing import Literal
 
 import typer
 from typing_extensions import Annotated
@@ -591,6 +592,102 @@ def new_figure(
             repo.git.commit(["-m", f"Add figure {path}"])
 
 
+def _new_simple_artifact(
+    kind: Literal["results", "presentations"],
+    path: str,
+    title: str,
+    description: str | None,
+    stage_name: str | None,
+    no_commit: bool,
+    overwrite: bool,
+) -> None:
+    """Declare a simple artifact (path/title/description/stage) in calkit.yaml."""
+    singular = kind.rstrip("s")
+    ck_info = calkit.load_calkit_info()
+    objects = ck_info.get(kind, [])
+    paths = [o.get("path") for o in objects]
+    if not overwrite and path in paths:
+        raise_error(f"{singular.capitalize()} at path {path} already exists")
+    if overwrite and path in paths:
+        objects = [o for o in objects if o.get("path") != path]
+    obj = dict(path=path, title=title)
+    if description is not None:
+        obj["description"] = description
+    if stage_name is not None:
+        obj["stage"] = stage_name
+    objects.append(obj)
+    ck_info[kind] = objects
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    if not no_commit:
+        repo = calkit.git.get_repo()
+        repo.git.add("calkit.yaml")
+        if repo.git.diff("--staged"):
+            repo.git.commit(["-m", f"Add {singular} {path}"])
+
+
+@new_app.command(name="result")
+def new_result(
+    path: str,
+    title: Annotated[str, typer.Option("--title")],
+    description: Annotated[str | None, typer.Option("--description")] = None,
+    stage_name: Annotated[
+        str | None,
+        typer.Option(
+            "--stage",
+            help="Name of the pipeline stage that generates this result.",
+        ),
+    ] = None,
+    no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite existing result if one exists.",
+        ),
+    ] = False,
+):
+    """Declare a new result."""
+    _new_simple_artifact(
+        "results", path, title, description, stage_name, no_commit, overwrite
+    )
+
+
+@new_app.command(name="presentation|pres")
+def new_presentation(
+    path: str,
+    title: Annotated[str, typer.Option("--title")],
+    description: Annotated[str | None, typer.Option("--description")] = None,
+    stage_name: Annotated[
+        str | None,
+        typer.Option(
+            "--stage",
+            help="Name of the pipeline stage that generates this presentation.",
+        ),
+    ] = None,
+    no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite existing presentation if one exists.",
+        ),
+    ] = False,
+):
+    """Declare a new presentation."""
+    _new_simple_artifact(
+        "presentations",
+        path,
+        title,
+        description,
+        stage_name,
+        no_commit,
+        overwrite,
+    )
+
+
 @new_app.command("question")
 def new_question(
     question: str,
@@ -2011,7 +2108,7 @@ def new_julia_env(
         except subprocess.CalledProcessError:
             raise_error(f"Failed to install Julia version {julia_version}")
         cmd = ["julia", f"+{julia_version}", f"--project={env_dir}", "-e"]
-        install_cmd = f'using Pkg; Pkg.activate("{env_dir}");'
+        install_cmd = f'using Pkg; Pkg.activate("{calkit.julia.escape_string(env_dir)}");'
         for package in packages or []:
             install_cmd += f' Pkg.add("{package}");'
         cmd.append(install_cmd)

{calkit_python-0.41.13 → calkit_python-0.41.15}/calkit/cli/notebooks.py

@@ -310,14 +310,18 @@ def check_env_kernel(
                     "still failed."
                 )
         # Don't include version in display_name; IJulia appends it automatically
+        # Escape interpolated values for the Julia string literals---notably the
+        # Windows project path, whose backslashes are otherwise read as invalid
+        # unicode escapes.
+        esc = calkit.julia.escape_string
         julia_cmd = (
             "import IJulia;"
             "kp=IJulia.installkernel("
-            f'"{kernel_name}",'
-            f'"--project={env_dir_abs}",'
+            f'"{esc(kernel_name)}",'
+            f'"--project={esc(env_dir_abs)}",'
             '"--startup-file=no",'
-            f'displayname="{display_name}",'
-            'env=Dict("JULIA_LOAD_PATH" => "@:@stdlib")'
+            f'displayname="{esc(display_name)}",'
+            f'env=Dict("JULIA_LOAD_PATH" => "{esc(calkit.julia.load_path())}")'
             ");"
             "println(kp);"
         )
@@ -474,8 +478,7 @@ def execute_notebook(
         # Create this environment and write it to file
         envs[res.name] = res.env
         ck_info["environments"] = envs
-        with open("calkit.yaml", "w") as f:
-            calkit.ryaml.dump(ck_info, f)
+        calkit.save_calkit_info(ck_info)
     # Detect language from environment
     if language is None:
         detected_language = language_from_notebook(path)
@@ -501,16 +504,20 @@ def execute_notebook(
         raise ValueError(
             "Language must be one of 'python', 'matlab', 'julia', or 'r'"
         )
-    # Try to set kernelspec metadata so execution uses the expected kernel
+    # Try to set kernelspec metadata so execution uses the expected kernel.
+    # Always read/write as UTF-8: notebooks routinely contain non-ASCII (e.g.,
+    # Greek letters such as "ν" in code), and on Windows the default open()
+    # encoding is cp1252, which mangles UTF-8 content into mojibake ("ν" ->
+    # "ν") and breaks execution.
     try:
-        with open(path, "r") as f:
+        with open(path, "r", encoding="utf-8") as f:
             nb_json = json.load(f)
         metadata = nb_json.setdefault("metadata", {})
         kernelspec = metadata.setdefault("kernelspec", {})
         kernelspec["name"] = kernel_name
         kernelspec["display_name"] = display_name
-        kernelspec.setdefault("language", language.lower())
-        with open(path, "w") as f:
+        kernelspec["language"] = language.lower()
+        with open(path, "w", encoding="utf-8") as f:
             json.dump(nb_json, f, indent=1)
     except Exception as e:
         if verbose:

{calkit_python-0.41.13 → calkit_python-0.41.15}/calkit/cli/scheduler.py

@@ -77,6 +77,23 @@ def _record_job(name: str, info: dict) -> None:
         jobs[name] = info
 
 
+def _delete_job(name: str) -> None:
+    """Remove a job record from the database.
+
+    Used when a job is canceled (e.g. the user hits Ctrl+C while waiting for
+    it). Without this the record would linger, and the next ``calkit run``
+    would find the job gone from the queue with no exit status and wrongly
+    treat it as successful instead of resubmitting it.
+    """
+    if not os.path.isfile(JOBS_DB_PATH):
+        return
+    with SqliteDict(
+        JOBS_DB_PATH, autocommit=True, timeout=JOBS_DB_TIMEOUT
+    ) as jobs:
+        if name in jobs:
+            del jobs[name]
+
+
 def _mock_enabled() -> bool:
     """Whether scheduler commands should run jobs locally.
 
@@ -390,6 +407,9 @@ def _wait_until_done(kind: str, job_id: str, name: str) -> int | None:
         ok, stderr = _cancel(kind, job_id)
         if not ok:
             typer.echo(f"Failed to cancel job '{name}' ({job_id}): {stderr}")
+        # Drop the record so the next run resubmits this job instead of
+        # mistaking the canceled job (which has no exit status) for a success.
+        _delete_job(name)
         raise typer.Exit(130)
 
 

{calkit_python-0.41.13 → calkit_python-0.41.15}/calkit/cli/update.py

@@ -746,7 +746,7 @@ def update_julia_env(
     env_dir = os.path.dirname(spec_path) or "."
     julia_version = env.get("julia")
     julia_bin = ["julia", f"+{julia_version}"] if julia_version else ["julia"]
-    cmds = [f'Pkg.activate("{env_dir}")']
+    cmds = [f'Pkg.activate("{calkit.julia.escape_string(env_dir)}")']
     if add:
         pkg_list = "[" + ", ".join(f'"{p}"' for p in add) + "]"
         cmds.append(f"Pkg.add({pkg_list})")

{calkit_python-0.41.13 → calkit_python-0.41.15}/calkit/core.py

@@ -144,7 +144,9 @@ def load_calkit_info(
     if wdir is not None:
         fpath = os.path.join(wdir, fpath)
     if os.path.isfile(fpath):
-        with open(fpath) as f:
+        # Always read as UTF-8; on Windows the default open() encoding is
+        # cp1252, which mangles non-ASCII content (e.g., Greek letters).
+        with open(fpath, encoding="utf-8") as f:
             info = ryaml.load(f)
     if info is None:
         info = {}
@@ -166,7 +168,7 @@ def load_calkit_info(
                         if wdir is not None:
                             include_fpath = os.path.join(wdir, include_fpath)
                         if os.path.isfile(include_fpath):
-                            with open(include_fpath) as f:
+                            with open(include_fpath, encoding="utf-8") as f:
                                 include_data = ryaml.load(f)
                             info[kind][obj_name] |= include_data
     return info
@@ -180,7 +182,9 @@ def save_calkit_info(
     fpath = "calkit.yaml"
     if wdir is not None:
         fpath = os.path.join(wdir, fpath)
-    with open(fpath, "w") as f:
+    # Always write as UTF-8; on Windows the default open() encoding is cp1252,
+    # which mangles non-ASCII content (e.g., Greek letters) into mojibake.
+    with open(fpath, "w", encoding="utf-8") as f:
         ryaml.dump(info, f)
 
 

{calkit_python-0.41.13 → calkit_python-0.41.15}/calkit/detect.py

@@ -1837,6 +1837,31 @@ DATASET_EXTENSIONS = {
     ".ndjson",
 }
 DATA_DIRS = {"data", "datasets", "dataset"}
+RESULT_DIRS = {"results", "result"}
+RESULT_EXTENSIONS = {
+    ".json",
+    ".csv",
+    ".tsv",
+    ".yaml",
+    ".yml",
+    ".parquet",
+    ".h5",
+    ".hdf5",
+    ".txt",
+    ".html",
+}
+PRESENTATION_DIRS = {"presentations", "presentation", "slides", "talks"}
+PRESENTATION_EXTENSIONS = {".pdf", ".pptx", ".key"}
+# Files that look like a presentation regardless of which directory they live in.
+PRESENTATION_NAMES = {
+    "slides.pdf",
+    "slides.pptx",
+    "slides.key",
+    "presentation.pdf",
+    "presentation.pptx",
+    "presentation.key",
+    "talk.pdf",
+}
 
 
 def _ancestor_dir_names(rel_path: str) -> set[str]:
@@ -1871,6 +1896,37 @@ def is_dataset_path(rel_path: str) -> bool:
     )
 
 
+def is_result_path(rel_path: str) -> bool:
+    """Whether a repo-relative path looks like an auto-detectable result.
+
+    A result is a data-like file (JSON, CSV, etc.) under a ``results``-style
+    directory. Anything already detected as a figure or dataset is excluded.
+    """
+    return (
+        _path_ext(rel_path) in RESULT_EXTENSIONS
+        and bool(_ancestor_dir_names(rel_path) & RESULT_DIRS)
+        and not is_figure_path(rel_path)
+        and not is_dataset_path(rel_path)
+    )
+
+
+def is_presentation_path(rel_path: str) -> bool:
+    """Whether a repo-relative path looks like an auto-detectable presentation.
+
+    Either a slide-deck file (PDF/PPTX/KEY) under a ``slides``/``presentations``
+    directory, or a file with a presentation-like name (e.g. ``slides.pdf``,
+    ``presentation.pdf``) anywhere. Figures are excluded.
+    """
+    if is_figure_path(rel_path):
+        return False
+    name = rel_path.rsplit("/", 1)[-1].lower()
+    if name in PRESENTATION_NAMES:
+        return True
+    return _path_ext(rel_path) in PRESENTATION_EXTENSIONS and bool(
+        _ancestor_dir_names(rel_path) & PRESENTATION_DIRS
+    )
+
+
 def _is_hidden_path(rel_path: str) -> bool:
     return any(part.startswith(".") for part in rel_path.split("/"))
 
@@ -1949,6 +2005,44 @@ def detect_datasets(
     return _collapse_dataset_folders(sorted(files))
 
 
+def detect_results(
+    candidate_paths: list[str],
+    reserved_paths: list[str] | tuple[str, ...] = (),
+) -> list[str]:
+    """Auto-detected result paths among ``candidate_paths``.
+
+    Paths under a dot-directory or under one of ``reserved_paths`` (declared
+    artifacts) are skipped.
+    """
+    reserved = list(reserved_paths)
+    return sorted(
+        {
+            p
+            for p in candidate_paths
+            if not _is_hidden_path(p)
+            and not _is_under_any_dir(p, reserved)
+            and is_result_path(p)
+        }
+    )
+
+
+def detect_presentations(
+    candidate_paths: list[str],
+    reserved_paths: list[str] | tuple[str, ...] = (),
+) -> list[str]:
+    """Auto-detected presentation paths among ``candidate_paths``."""
+    reserved = list(reserved_paths)
+    return sorted(
+        {
+            p
+            for p in candidate_paths
+            if not _is_hidden_path(p)
+            and not _is_under_any_dir(p, reserved)
+            and is_presentation_path(p)
+        }
+    )
+
+
 def list_repo_files(wdir: str | None = None) -> list[str]:
     """Repo-relative paths of files Git does not ignore.
 
@@ -2014,7 +2108,7 @@ def _reserved_artifact_paths(
     if ck_info is None:
         ck_info = calkit.load_calkit_info(wdir=wdir)
     paths: list[str] = []
-    for kind in ("figures", "datasets"):
+    for kind in ("figures", "datasets", "results", "presentations"):
         for obj in ck_info.get(kind, []) or []:
             if isinstance(obj, dict) and isinstance(obj.get("path"), str):
                 paths.append(obj["path"])
@@ -2044,4 +2138,11 @@ def detect_project_artifacts(
     datasets = detect_datasets(
         candidates, reserved_paths=reserved, figure_paths=figures
     )
-    return {"figures": figures, "datasets": datasets}
+    results = detect_results(candidates, reserved_paths=reserved)
+    presentations = detect_presentations(candidates, reserved_paths=reserved)
+    return {
+        "figures": figures,
+        "datasets": datasets,
+        "results": results,
+        "presentations": presentations,
+    }