calkit-python 0.41.20__py3-none-any.whl → 0.41.22__py3-none-any.whl

calkit/cli/list.py

@@ -174,6 +174,40 @@ def list_presentations(
     _list_artifacts("presentations", json_output, declared_only)
 
 
+def _echo_question(n: int, question: str | dict) -> None:
+    """Print a single question in the human-readable YAML-ish listing format.
+
+    String questions are shown as a numbered line. Rich (dict) questions are
+    shown with their text on the first line followed by hypothesis, answer,
+    evidence, and any other fields in YAML-like indented form.
+    """
+    if isinstance(question, str):
+        typer.echo(f"{n}. {question}")
+        return
+    # Copy so popping 'question' doesn't mutate the caller's dict.
+    question = dict(question)
+    text = question.pop("question", "")
+    typer.echo(f"{n}. question: {text}")
+    for k, v in question.items():
+        if isinstance(v, dict):
+            typer.echo(f"    {k}:")
+            for k1, v1 in v.items():
+                typer.echo(f"      {k1}: {v1}")
+        elif isinstance(v, list):
+            typer.echo(f"    {k}:")
+            for item in v:
+                if isinstance(item, dict):
+                    for n1, (k1, v1) in enumerate(item.items()):
+                        if n1 == 0:
+                            typer.echo(f"      - {k1}: {v1}")
+                        else:
+                            typer.echo(f"        {k1}: {v1}")
+                else:
+                    typer.echo(f"        - {item}")
+        else:
+            typer.echo(f"    {k}: {v}")
+
+
 @list_app.command(name="questions")
 def list_questions(
     json_output: Annotated[
@@ -186,7 +220,7 @@ def list_questions(
         typer.echo(json.dumps(questions))
         return
     for n, question in enumerate(questions, start=1):
-        typer.echo(f"{n}. {question}")
+        _echo_question(n, question)
 
 
 @list_app.command(name="publications|pubs")

calkit/cli/main/core.py

@@ -786,6 +786,8 @@ def add(
         elif to == "git":
             subprocess.call(["git", "add"] + paths)
         elif to == "dvc":
+            for path in paths:
+                calkit.git.ensure_dvc_pointer_is_not_ignored(repo, path)
             calkit.dvc.run_dvc_command(["add"] + paths)
         elif to == "dvc-zip":
             for path in paths:
@@ -899,6 +901,7 @@ def add(
                         f"Adding {path} to DVC since it's already tracked "
                         "with DVC"
                     )
+                    calkit.git.ensure_dvc_pointer_is_not_ignored(repo, path)
                     calkit.dvc.run_dvc_command(["add", path])
             elif posix_path in pipeline_output_storage:
                 # Respect storage explicitly set in the pipeline definition
@@ -943,6 +946,7 @@ def add(
                     typer.echo(f"Would add {path} to DVC (per extension)")
                 else:
                     typer.echo(f"Adding {path} to DVC per its extension")
+                    calkit.git.ensure_dvc_pointer_is_not_ignored(repo, path)
                     calkit.dvc.run_dvc_command(["add", path])
             elif calkit.dvc.zip.is_zip_candidate(path):
                 if dry_run:
@@ -963,6 +967,7 @@ def add(
                     typer.echo(
                         f"Adding {path} to DVC since it's greater than 1 MB"
                     )
+                    calkit.git.ensure_dvc_pointer_is_not_ignored(repo, path)
                     calkit.dvc.run_dvc_command(["add", path])
             else:
                 if dry_run:
@@ -1414,6 +1419,39 @@ def _stage_run_info_from_log_content(log_content: str) -> dict:
     return res
 
 
+def _run_dvc_repro(argv: list[str]) -> int | None:
+    """Run ``dvc repro`` via the DVC CLI, tolerating teardown failures.
+
+    Returns DVC's exit code, or ``None`` if the command ran but DVC's
+    post-command teardown failed to import a module. After ``do_run`` finishes,
+    ``dvc.cli.main`` reports anonymous analytics and cleans up cached repos,
+    importing ``dvc.daemon`` and ``dvc.repo.open_repo`` only at that point. In
+    some broken or mixed installs those submodules can't be imported, so the
+    teardown raises a ``ModuleNotFoundError`` that escapes DVC entirely and
+    crashes the run with a confusing traceback once the pipeline has already
+    finished (see issue #1018). That doesn't affect the pipeline result, so
+    swallow it and signal ``None`` so the caller derives success/failure from
+    the run log instead of a lost exit code.
+
+    Only those two modules are tolerated, and deliberately so. ``dvc.cli.main``
+    runs the command inside a broad ``except Exception`` that turns any failure
+    into an exit code, so nothing from the command itself reaches here; but it
+    also imports ``dvc._debug``/``dvc.config``/``dvc.logger`` *before* that
+    block, and a broken install can fail there too. Swallowing those would leave
+    an empty run log and report a pipeline that never ran as successful, so
+    anything but a teardown module must propagate and fail loudly.
+    """
+    from dvc.cli import main as dvc_cli_main
+
+    try:
+        return int(dvc_cli_main(argv))
+    except ModuleNotFoundError as e:
+        if e.name not in ("dvc.daemon", "dvc.repo.open_repo"):
+            raise
+        warn(f"DVC post-run teardown failed and was ignored ({e})")
+        return None
+
+
 def _prune_run_logs(
     logs_dir: str, keep: int = 10, protect: str | None = None
 ) -> None:
@@ -1823,7 +1861,6 @@ def run(
     import dvc.repo
     import dvc.repo.reproduce
     import dvc.ui
-    from dvc.cli import main as dvc_cli_main
     from git.exc import InvalidGitRepositoryError
 
     import calkit.dvc.zip
@@ -2075,13 +2112,21 @@ def run(
                 with calkit.dvc.dvc_lock_timeout(
                     calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT
                 ):
-                    sp_res = dvc_cli_main(["repro"] + sp_args)
+                    sp_res = _run_dvc_repro(["repro"] + sp_args)
             finally:
                 os.chdir(original_dir)
-            if sp_res != 0:
+            # ``None`` means the stage ran but DVC's teardown failed; treat that
+            # as success since the exit code is unrecoverable here.
+            if sp_res is not None and sp_res != 0:
                 failed = True
         if not args or all(a.startswith("--") for a in args):
-            # Only isolated subproject stage targets were given; skip parent run
+            # Only isolated subproject stage targets were given; skip the parent
+            # run, but still report failure the way the parent path does below,
+            # else a failing subproject stage would exit zero.
+            os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
+            if failed:
+                raise_error("Pipeline failed")
+            calkit.echo("Pipeline completed successfully ✅")
             return {}
     if pipeline is not None:
         args += ["--pipeline", pipeline]
@@ -2152,14 +2197,21 @@ def run(
         # the instant it finishes) can all hold it momentarily. Without this,
         # such a collision aborts the whole run with "Unable to acquire lock".
         with calkit.dvc.dvc_lock_timeout(calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT):
-            res = dvc_cli_main(["repro"] + args)
+            res = _run_dvc_repro(["repro"] + args)
     finally:
         os.environ.pop("CALKIT_FORCE", None)
-    failed = failed or res != 0
     # Parse log to get timing and which stages ran
     with open(log_fpath, "r") as f:
         log_content = f.read()
         stage_run_info = _stage_run_info_from_log_content(log_content)
+    if res is None:
+        # DVC's exit code was lost to a teardown failure; fall back to the log,
+        # which records any stage that failed to reproduce.
+        failed = failed or any(
+            info.get("status") == "failed" for info in stage_run_info.values()
+        )
+    else:
+        failed = failed or res != 0
     # Zip dvc-zip outputs for stages that actually ran
     if stage_run_info:
         from calkit.models.io import PathOutput
@@ -3218,6 +3270,14 @@ def call_dvc(
     and DVC is not installed.
     """
     result = calkit.dvc.run_dvc_command(sys.argv[2:])
+    if result != 0 and len(sys.argv) > 2 and sys.argv[2] == "add":
+        typer.secho(
+            "Hint: If DVC failed because a .dvc pointer file is git-ignored, "
+            "use `calkit add --to=dvc <path>` instead so Calkit can "
+            "automatically manage the gitignore exception.",
+            fg=typer.colors.YELLOW,
+            err=True,
+        )
     sys.exit(result)
 
 

calkit/cli/new.py

@@ -3174,7 +3174,7 @@ def new_release(
         detected_kind, artifact = calkit.releases.find_artifact(path, ck_info)
         if detected_kind is None:
             detected_kind = calkit.detect.detect_artifact_kind(
-                pathlib.Path(path).as_posix()
+                pathlib.Path(path).as_posix(),
             )
         if release_kind is None:
             if detected_kind is None:
@@ -3258,16 +3258,12 @@ def new_release(
         # stored as-is (renamed); folders and whole-project releases are
         # zipped.
         project_name = calkit.detect_project_name(prepend_owner=False)
-        if path == ".":
+        if path == "." or os.path.isdir(path):
             stored_filename = f"{project_name}-{name}.zip"
             is_zip = True
-        elif os.path.isdir(path):
-            folder_name = os.path.basename(os.path.normpath(path))
-            stored_filename = f"{project_name}-{folder_name}-{name}.zip"
-            is_zip = True
         elif os.path.isfile(path):
-            stem, ext = os.path.splitext(os.path.basename(path))
-            stored_filename = f"{project_name}-{stem}-{name}{ext}"
+            _, ext = os.path.splitext(os.path.basename(path))
+            stored_filename = f"{project_name}-{name}{ext}"
             is_zip = False
         else:
             raise_error(f"Release path '{path}' does not exist")

calkit/cli/overleaf.py

@@ -437,7 +437,12 @@ def sync(
         raise_error("No Overleaf sync info found")
     overleaf_sync_dirs = list(overleaf_info.keys())
     if paths is not None:
-        paths = [os.path.dirname(p) if os.path.isfile(p) else p for p in paths]
+        paths = [
+            (os.path.dirname(p) if os.path.isfile(p) else p)
+            .strip()
+            .rstrip("/\\")
+            for p in paths
+        ]
         for path in paths:
             if path not in overleaf_sync_dirs:
                 raise_error(f"Path '{path}' is not synced with Overleaf")
@@ -611,7 +616,12 @@ def get_status(
         raise_error("No Overleaf sync info found")
     overleaf_sync_dirs = list(overleaf_info.keys())
     if paths is not None:
-        paths = [os.path.dirname(p) if os.path.isfile(p) else p for p in paths]
+        paths = [
+            (os.path.dirname(p) if os.path.isfile(p) else p)
+            .strip()
+            .rstrip("/\\")
+            for p in paths
+        ]
         for path in paths:
             if path not in overleaf_sync_dirs:
                 raise_error(f"Path '{path}' is not synced with Overleaf")

calkit/detect.py

@@ -13,6 +13,8 @@ import sys
 from pathlib import Path
 from typing import Literal
 
+import pathspec
+
 NotebookLanguage = Literal["python", "julia", "r"]
 
 
@@ -1549,6 +1551,20 @@ def _is_stdlib_module(module_name: str) -> bool:
     return base_module in common_stdlib
 
 
+_PYTHON_IMPORT_TO_DISTRIBUTION = {
+    "sklearn": "scikit-learn",
+    "skimage": "scikit-image",
+    "cv2": "opencv-python",
+    "PIL": "Pillow",
+    "bs4": "beautifulsoup4",
+    "yaml": "PyYAML",
+    "dateutil": "python-dateutil",
+    "dotenv": "python-dotenv",
+    "serial": "pyserial",
+    "git": "GitPython",
+}
+
+
 def detect_python_dependencies(
     script_path: str | None = None,
     code: str | None = None,
@@ -1614,13 +1630,21 @@ def detect_python_dependencies(
                 if not _is_stdlib_module(module_name):
                     # Get the top-level package name
                     top_level = module_name.split(".")[0]
-                    dependencies.add(top_level)
+                    dependencies.add(
+                        _PYTHON_IMPORT_TO_DISTRIBUTION.get(
+                            top_level, top_level
+                        )
+                    )
         elif isinstance(node, ast.ImportFrom):
             if node.module and node.level == 0:
                 module_name = node.module
                 if not _is_stdlib_module(module_name):
                     top_level = module_name.split(".")[0]
-                    dependencies.add(top_level)
+                    dependencies.add(
+                        _PYTHON_IMPORT_TO_DISTRIBUTION.get(
+                            top_level, top_level
+                        )
+                    )
     return sorted(list(dependencies))
 
 
@@ -1927,6 +1951,21 @@ PRESENTATION_NAMES = {
     "talk.pdf",
 }
 
+DETECTION_IGNORE_FPATH = os.path.join(".calkit", "ignore")
+
+
+def load_detection_ignore(wdir: str | None = None) -> pathspec.PathSpec | None:
+    """Load ``.calkit/ignore`` artifact detection exclusions."""
+    import pathspec
+
+    fpath = DETECTION_IGNORE_FPATH
+    if wdir is not None:
+        fpath = os.path.join(wdir, fpath)
+    if not os.path.isfile(fpath):
+        return None
+    with open(fpath, encoding="utf-8") as f:
+        return pathspec.PathSpec.from_lines("gitwildmatch", f)
+
 
 def _ancestor_dir_names(rel_path: str) -> set[str]:
     """Lower-cased names of the ancestor directories of a "/"-separated path."""
@@ -1938,8 +1977,12 @@ def _path_ext(rel_path: str) -> str:
     return ("." + name.rsplit(".", 1)[-1].lower()) if "." in name else ""
 
 
-def is_figure_path(rel_path: str) -> bool:
+def is_figure_path(
+    rel_path: str, *, ignore: pathspec.PathSpec | None = None
+) -> bool:
     """Whether a repo-relative path looks like an auto-detectable figure."""
+    if ignore is not None and ignore.match_file(rel_path):
+        return False
     ancestors = _ancestor_dir_names(rel_path)
     ext = _path_ext(rel_path)
     if ext in FIGURE_EXTENSIONS and ancestors & FIGURE_DIRS:
@@ -1951,37 +1994,49 @@ def is_figure_path(rel_path: str) -> bool:
     return False
 
 
-def is_dataset_path(rel_path: str) -> bool:
+def is_dataset_path(
+    rel_path: str, *, ignore: pathspec.PathSpec | None = None
+) -> bool:
     """Whether a repo-relative path looks like an auto-detectable dataset."""
+    if ignore is not None and ignore.match_file(rel_path):
+        return False
     return (
         _path_ext(rel_path) in DATASET_EXTENSIONS
         and bool(_ancestor_dir_names(rel_path) & DATA_DIRS)
-        and not is_figure_path(rel_path)
+        and not is_figure_path(rel_path, ignore=ignore)
     )
 
 
-def is_result_path(rel_path: str) -> bool:
+def is_result_path(
+    rel_path: str, *, ignore: pathspec.PathSpec | None = None
+) -> bool:
     """Whether a repo-relative path looks like an auto-detectable result.
 
     A result is a data-like file (JSON, CSV, etc.) under a ``results``-style
     directory. Anything already detected as a figure or dataset is excluded.
     """
+    if ignore is not None and ignore.match_file(rel_path):
+        return False
     return (
         _path_ext(rel_path) in RESULT_EXTENSIONS
         and bool(_ancestor_dir_names(rel_path) & RESULT_DIRS)
-        and not is_figure_path(rel_path)
-        and not is_dataset_path(rel_path)
+        and not is_figure_path(rel_path, ignore=ignore)
+        and not is_dataset_path(rel_path, ignore=ignore)
     )
 
 
-def is_presentation_path(rel_path: str) -> bool:
+def is_presentation_path(
+    rel_path: str, *, ignore: pathspec.PathSpec | None = None
+) -> bool:
     """Whether a repo-relative path looks like an auto-detectable presentation.
 
     Either a slide-deck file (PDF/PPTX/KEY) under a ``slides``/``presentations``
     directory, or a file with a presentation-like name (e.g. ``slides.pdf``,
     ``presentation.pdf``) anywhere. Figures are excluded.
     """
-    if is_figure_path(rel_path):
+    if ignore is not None and ignore.match_file(rel_path):
+        return False
+    if is_figure_path(rel_path, ignore=ignore):
         return False
     name = rel_path.rsplit("/", 1)[-1].lower()
     if name in PRESENTATION_NAMES:
@@ -2018,7 +2073,9 @@ PUBLICATION_NAMES = {
 }
 
 
-def is_publication_path(rel_path: str) -> bool:
+def is_publication_path(
+    rel_path: str, *, ignore: pathspec.PathSpec | None = None
+) -> bool:
     """Whether a repo-relative path looks like an auto-detectable publication.
 
     Either a document (PDF/TeX/DOCX) under a ``paper``/``publication``/
@@ -2026,7 +2083,11 @@ def is_publication_path(rel_path: str) -> bool:
     ``manuscript.pdf``, ``main.tex``) anywhere. Figures and presentations are
     excluded.
     """
-    if is_figure_path(rel_path) or is_presentation_path(rel_path):
+    if ignore is not None and ignore.match_file(rel_path):
+        return False
+    if is_figure_path(rel_path, ignore=ignore) or is_presentation_path(
+        rel_path, ignore=ignore
+    ):
         return False
     name = rel_path.rsplit("/", 1)[-1].lower()
     if name in PUBLICATION_NAMES:
@@ -2036,19 +2097,21 @@ def is_publication_path(rel_path: str) -> bool:
     )
 
 
-def detect_artifact_kind(rel_path: str) -> str | None:
+def detect_artifact_kind(
+    rel_path: str, *, ignore: pathspec.PathSpec | None = None
+) -> str | None:
     """Infer an artifact's release kind from its repo-relative path.
 
     Returns one of ``"figure"``, ``"presentation"``, ``"publication"``, or
     ``"dataset"``, or ``None`` if the path doesn't look like any of these.
     """
-    if is_figure_path(rel_path):
+    if is_figure_path(rel_path, ignore=ignore):
         return "figure"
-    if is_presentation_path(rel_path):
+    if is_presentation_path(rel_path, ignore=ignore):
         return "presentation"
-    if is_publication_path(rel_path):
+    if is_publication_path(rel_path, ignore=ignore):
         return "publication"
-    if is_dataset_path(rel_path):
+    if is_dataset_path(rel_path, ignore=ignore):
         return "dataset"
     return None
 
@@ -2090,6 +2153,7 @@ def _collapse_dataset_folders(paths: list[str]) -> list[str]:
 def detect_figures(
     candidate_paths: list[str],
     reserved_paths: list[str] | tuple[str, ...] = (),
+    ignore: pathspec.PathSpec | None = None,
 ) -> list[str]:
     """Auto-detected figure paths among ``candidate_paths``.
 
@@ -2103,7 +2167,7 @@ def detect_figures(
             for p in candidate_paths
             if not _is_hidden_path(p)
             and not _is_under_any_dir(p, reserved)
-            and is_figure_path(p)
+            and is_figure_path(p, ignore=ignore)
         }
     )
 
@@ -2112,6 +2176,7 @@ def detect_datasets(
     candidate_paths: list[str],
     reserved_paths: list[str] | tuple[str, ...] = (),
     figure_paths: list[str] | tuple[str, ...] = (),
+    ignore: pathspec.PathSpec | None = None,
 ) -> list[str]:
     """Auto-detected dataset paths among ``candidate_paths``.
 
@@ -2126,7 +2191,7 @@ def detect_datasets(
         if not _is_hidden_path(p)
         and not _is_under_any_dir(p, reserved)
         and p not in figset
-        and is_dataset_path(p)
+        and is_dataset_path(p, ignore=ignore)
     }
     return _collapse_dataset_folders(sorted(files))
 
@@ -2134,6 +2199,7 @@ def detect_datasets(
 def detect_results(
     candidate_paths: list[str],
     reserved_paths: list[str] | tuple[str, ...] = (),
+    ignore: pathspec.PathSpec | None = None,
 ) -> list[str]:
     """Auto-detected result paths among ``candidate_paths``.
 
@@ -2147,7 +2213,7 @@ def detect_results(
             for p in candidate_paths
             if not _is_hidden_path(p)
             and not _is_under_any_dir(p, reserved)
-            and is_result_path(p)
+            and is_result_path(p, ignore=ignore)
         }
     )
 
@@ -2155,6 +2221,7 @@ def detect_results(
 def detect_presentations(
     candidate_paths: list[str],
     reserved_paths: list[str] | tuple[str, ...] = (),
+    ignore: pathspec.PathSpec | None = None,
 ) -> list[str]:
     """Auto-detected presentation paths among ``candidate_paths``."""
     reserved = list(reserved_paths)
@@ -2164,7 +2231,7 @@ def detect_presentations(
             for p in candidate_paths
             if not _is_hidden_path(p)
             and not _is_under_any_dir(p, reserved)
-            and is_presentation_path(p)
+            and is_presentation_path(p, ignore=ignore)
         }
     )
 
@@ -2254,18 +2321,28 @@ def detect_project_artifacts(
 
     if ck_info is None:
         ck_info = calkit.load_calkit_info(wdir=wdir)
+    ignore = load_detection_ignore(wdir=wdir)
     reserved = _reserved_artifact_paths(wdir=wdir, ck_info=ck_info)
     candidates = list(
         dict.fromkeys(
             [*list_repo_files(wdir=wdir), *list_dvc_tracked_files(wdir=wdir)]
         )
     )
-    figures = detect_figures(candidates, reserved_paths=reserved)
+    figures = detect_figures(
+        candidates, reserved_paths=reserved, ignore=ignore
+    )
     datasets = detect_datasets(
-        candidates, reserved_paths=reserved, figure_paths=figures
+        candidates,
+        reserved_paths=reserved,
+        figure_paths=figures,
+        ignore=ignore,
+    )
+    results = detect_results(
+        candidates, reserved_paths=reserved, ignore=ignore
+    )
+    presentations = detect_presentations(
+        candidates, reserved_paths=reserved, ignore=ignore
     )
-    results = detect_results(candidates, reserved_paths=reserved)
-    presentations = detect_presentations(candidates, reserved_paths=reserved)
     return {
         "figures": figures,
         "datasets": datasets,

calkit/git.py

@@ -337,3 +337,32 @@ def ensure_path_is_not_ignored(
     if target_repo.ignored(target_path) and _depth < 10:
         ensure_path_is_not_ignored(target_repo, target_path, _depth + 1)
     return True
+
+
+def ensure_dvc_pointer_is_not_ignored(repo, path: str) -> None:
+    """Ensure the .dvc pointer for ``path`` will not be Git-ignored.
+
+    A broad pattern in a ``.gitignore`` (e.g. ``*.pdf*``) can also match the
+    ``<path>.dvc`` pointer DVC commits to Git, causing ``dvc add`` to fail with
+    "bad DVC file name ... is git-ignored". This appends a ``!*.dvc`` negation
+    to the ``.gitignore`` in the pointer's own directory (which wins under Git
+    precedence) so pointers stay tracked. Idempotent.
+    """
+    path = path.replace("\\", "/").rstrip("/")
+    pointer = path + ".dvc"
+    if pointer not in repo.ignored(pointer):
+        return
+    pointer_dir = os.path.dirname(pointer)
+    gitignore_path = os.path.join(repo.working_dir, pointer_dir, ".gitignore")
+    exception = "!*.dvc"
+    existing_lines = []
+    if os.path.isfile(gitignore_path):
+        with open(gitignore_path, "r", encoding="utf-8") as f:
+            existing_lines = f.read().splitlines()
+    if exception in existing_lines:
+        return  # Already present
+    os.makedirs(os.path.dirname(gitignore_path), exist_ok=True)
+    with open(gitignore_path, "a", encoding="utf-8") as f:
+        if existing_lines and existing_lines[-1] != "":
+            f.write("\n")
+        f.write(exception + "\n")

calkit/models/core.py

@@ -351,13 +351,23 @@ class ResultsEvidence(BaseModel):
     explanation: str | None = None
 
 
+class PublicationEvidence(BaseModel):
+    """Evidence in the form of a publication."""
+
+    kind: Literal["publication"] = "publication"
+    path: str
+    explanation: str | None = None
+
+
 class Question(BaseModel):
     """A question the project hopes to answer."""
 
     question: str
     hypothesis: str | None = None
     answer: str | None = None
-    evidence: list[FigureEvidence | ResultsEvidence] | None = None
+    evidence: (
+        list[FigureEvidence | ResultsEvidence | PublicationEvidence] | None
+    ) = None
 
 
 class Dependency(BaseModel):