calkit-python 0.45.0__py3-none-any.whl → 0.45.2__py3-none-any.whl

calkit/__init__.py

@@ -10,7 +10,6 @@ from .core import *  # noqa: F403, I001
 if TYPE_CHECKING:
     from calkit import (  # noqa: F401
         calc,
-        check,
         conda,
         config,
         datasets,
@@ -37,7 +36,10 @@ if TYPE_CHECKING:
         ops,
         overleaf,
         pipeline,
+        procedures,
+        provenance,
         releases,
+        reproducibility,
         resources,
         schema,
         server,
@@ -59,7 +61,9 @@ _SUBMODULES = {
     "templates",
     "conda",
     "calc",
-    "check",
+    "procedures",
+    "provenance",
+    "reproducibility",
     "github",
     "invenio",
     "releases",

calkit/cli/check.py

@@ -308,7 +308,7 @@ def check_repro(
     ] = ".",
 ) -> None:
     """Check the reproducibility of a project."""
-    from calkit.check import check_reproducibility
+    from calkit.reproducibility import check_reproducibility
 
     res = check_reproducibility(wdir=wdir, log_func=typer.echo)
     calkit.echo(res.to_pretty())

calkit/cli/import_.py

@@ -208,7 +208,9 @@ def import_dataset(
             filter_paths=filter_paths,
         ),
     )
-    datasets.append(new_ds.model_dump())
+    # Nulls left out, so the entry reads as what was recorded rather than as
+    # a form with most of its fields blank
+    datasets.append(new_ds.model_dump(exclude_none=True))
     ck_info["datasets"] = datasets
     with open("calkit.yaml", "w") as f:
         calkit.ryaml.dump(ck_info, f)
@@ -465,13 +467,24 @@ def import_from_zenodo(
     if kind is not None:
         ck_info = calkit.load_calkit_info()
         items = ck_info.get(kind + "s", [])
+        # A record without a DOI (not every one has been minted one) is still
+        # somewhere in particular, so fall back to its URL rather than
+        # writing ``doi: null``, which nothing would then accept
+        doi = calkit.invenio.extract_doi(record)
+        if doi is not None:
+            imported_from: dict = {"doi": doi}
+        else:
+            imported_from = {
+                "url": record.get("links", {}).get("self_html")
+                or f"https://zenodo.org/records/{record_id}"
+            }
         item_record = {
             "path": dest_dir,
-            "imported_from": {
-                "doi": record.get("doi"),
-            },
-            "title": record.get("metadata", {}).get("title"),
+            "imported_from": imported_from,
         }
+        title = record.get("metadata", {}).get("title")
+        if title:
+            item_record["title"] = title
         items.append(item_record)
         ck_info[kind + "s"] = items
         with open("calkit.yaml", "w") as f:

calkit/cli/list.py

@@ -45,6 +45,7 @@ def _list_objects(
         "figures",
         "references",
         "publications",
+        "misc",
     ],
     json_output: bool = False,
 ):
@@ -241,6 +242,18 @@ def list_publications(
     _list_objects("publications", json_output)
 
 
+@list_app.command(name="misc")
+def list_misc(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+):
+    """List misc artifacts in the project, i.e., attributed paths that
+    aren't one of the typed kinds.
+    """
+    _list_objects("misc", json_output)
+
+
 @list_app.command(name="references|refs")
 def list_references(
     json_output: Annotated[

calkit/cli/main/core.py

@@ -286,9 +286,18 @@ def init(
         needs_own_repo = True
     if needs_own_repo:
         subprocess.run(["git", "init"])
-    result = calkit.dvc.init(force=force)
-    if result != 0:
-        raise_error("Failed to initialize DVC")
+    # A DVC repo can already exist here without a calkit.yaml, e.g., on
+    # platforms that set one up for us, so leave it alone rather than
+    # failing on DVC's refusal to initialize over an existing .dvc. The
+    # marker is .dvc/config, as elsewhere: a .dvc directory without one is
+    # an interrupted initialization, which has nothing worth keeping and
+    # which DVC will only initialize over with --force.
+    if os.path.isfile(os.path.join(".dvc", "config")) and not force:
+        typer.echo("DVC is already initialized")
+    else:
+        result = calkit.dvc.init(force=force or os.path.isdir(".dvc"))
+        if result != 0:
+            raise_error("Failed to initialize DVC")
     # Ensure autostage is enabled for DVC
     result = calkit.dvc.run_dvc_command(["config", "core.autostage", "true"])
     if result != 0:
@@ -944,9 +953,11 @@ def add(
             if dvc_repo is not None:
                 dvc_status = dvc_repo.data_status()
                 uncommitted = dvc_status["uncommitted"]
+                dvc_not_in_cache = set(dvc_status.get("not_in_cache", []))
                 dvc_uncommitted_all = uncommitted.get(
                     "modified", []
                 ) + uncommitted.get("deleted", [])
+                missing_from_cache = []
                 for dvc_uncommitted in dvc_uncommitted_all:
                     if os.path.exists(dvc_uncommitted):
                         if dry_run:
@@ -955,11 +966,11 @@ def add(
                             )
                         else:
                             typer.echo(f"Adding {dvc_uncommitted} to DVC")
-                            dvc_repo.commit(
-                                dvc_uncommitted,
-                                force=True,
-                                allow_missing=True,
-                            )
+                            calkit.dvc.commit_path(dvc_repo, dvc_uncommitted)
+                    elif dvc_uncommitted in dvc_not_in_cache:
+                        # The data isn't on this machine at all, so this isn't
+                        # a real deletion and there's nothing to commit
+                        missing_from_cache.append(dvc_uncommitted)
                     else:
                         if dry_run:
                             typer.echo(
@@ -969,11 +980,20 @@ def add(
                             typer.echo(
                                 f"Committing deleted {dvc_uncommitted} to DVC"
                             )
-                            dvc_repo.commit(
-                                dvc_uncommitted,
-                                force=True,
-                                allow_missing=True,
-                            )
+                            calkit.dvc.commit_path(dvc_repo, dvc_uncommitted)
+                if missing_from_cache:
+                    n_shown = 5
+                    paths_txt = ", ".join(missing_from_cache[:n_shown])
+                    if len(missing_from_cache) > n_shown:
+                        paths_txt += (
+                            f", and {len(missing_from_cache) - n_shown} more"
+                        )
+                    warn(
+                        f"Not committing {len(missing_from_cache)} missing "
+                        f"DVC-tracked path(s) since their data is not in the "
+                        f"cache ({paths_txt}); "
+                        "run 'calkit pull' to fetch their data"
+                    )
             if not disable_auto_ignore:
                 for untracked_file in untracked_git_files:
                     if (
@@ -3506,15 +3526,14 @@ def run_procedure(
             return bool(value)
         return value
 
-    from calkit.models import Procedure
-
     ck_info = calkit.load_calkit_info()
     calkit.set_env_vars(ck_info=ck_info)
-    procs = ck_info.get("procedures", {})
-    if name not in procs:
+    if name not in ck_info.get("procedures", {}):
         raise_error(f"'{name}' is not defined as a procedure")
+    # Read from its own file if that's where it's kept, so the rest of this
+    # sees the same thing either way
     try:
-        proc = Procedure.model_validate(procs[name])
+        proc = calkit.procedures.load(name, ck_info=ck_info)
     except Exception as e:
         raise_error(f"Procedure '{name}' is invalid: {e}")
     git_repo = calkit.git.get_repo()

calkit/cli/new.py

@@ -32,6 +32,39 @@ def _check_path_dir(path: str):
         os.makedirs(dirname, exist_ok=True)
 
 
+def _person_from_options(
+    option: str, email: str | None, orcid: str | None, with_ai: list[str]
+) -> dict | None:
+    """Build a ``created_by`` entry from CLI options.
+
+    Returns ``None`` when none were given. Validated through the model so
+    a mistyped ORCID or a ``--with-ai`` with nobody to answer for it is
+    refused here rather than by the next ``calkit.yaml`` validation.
+    """
+    from pydantic import ValidationError
+
+    from calkit.models.core import _Person
+
+    if email is None and orcid is None and not with_ai:
+        return None
+    # ``with_ai`` is a list, one entry per --with-ai given. A single tool is
+    # written as a scalar, which is what the docs show and what reads best
+    # in calkit.yaml; several stay a list
+    with_ai_value: str | list[str] | None = None
+    if len(with_ai) == 1:
+        with_ai_value = with_ai[0]
+    elif with_ai:
+        with_ai_value = with_ai
+    try:
+        person = _Person(email=email, orcid=orcid, with_ai=with_ai_value)
+    except ValidationError as e:
+        raise_error(
+            f"Invalid --{option}: "
+            + "; ".join(str(err["msg"]) for err in e.errors())
+        )
+    return person.model_dump(exclude_none=True)
+
+
 def _split_template_subdir(
     template: str, git_url: str
 ) -> tuple[str, str | None]:
@@ -600,6 +633,32 @@ def new_figure(
             help="Stage name from which to add outputs as dependencies.",
         ),
     ] = None,
+    created_by_email: Annotated[
+        str | None,
+        typer.Option(
+            "--created-by-email",
+            help=(
+                "Email of whoever made this figure, for one drawn by hand "
+                "rather than produced by a stage."
+            ),
+        ),
+    ] = None,
+    created_by_orcid: Annotated[
+        str | None,
+        typer.Option(
+            "--created-by-orcid", help="ORCID of whoever made this figure."
+        ),
+    ] = None,
+    created_with_ai: Annotated[
+        list[str],
+        typer.Option(
+            "--created-with-ai",
+            help=(
+                "Generative AI tool they used, e.g. 'Claude Opus 5'. "
+                "Repeat for several."
+            ),
+        ),
+    ] = [],
     no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
     overwrite: Annotated[
         bool,
@@ -626,11 +685,16 @@ def new_figure(
         raise_error("Command must be provided")
     if (deps or outs or outs_from_stage) and not stage_name:
         raise_error("Stage name must be provided")
-    obj = dict(path=path, title=title)
+    created_by = _person_from_options(
+        "created-by", created_by_email, created_by_orcid, created_with_ai
+    )
+    obj: dict = dict(path=path, title=title)
     if description is not None:
         obj["description"] = description
     if stage_name is not None:
         obj["stage"] = stage_name
+    if created_by is not None:
+        obj["created_by"] = created_by
     if cmd:
         if outs_from_stage:
             pipeline = calkit.dvc.read_pipeline()
@@ -1197,6 +1261,33 @@ def new_dataset(
             help="Stage name from which to add outputs as dependencies.",
         ),
     ] = None,
+    created_by_email: Annotated[
+        str | None,
+        typer.Option(
+            "--created-by-email",
+            help=(
+                "Email of whoever collected this data for the project, "
+                "which marks it as primary rather than imported or computed."
+            ),
+        ),
+    ] = None,
+    created_by_orcid: Annotated[
+        str | None,
+        typer.Option(
+            "--created-by-orcid",
+            help="ORCID of whoever collected this data.",
+        ),
+    ] = None,
+    created_with_ai: Annotated[
+        list[str],
+        typer.Option(
+            "--created-with-ai",
+            help=(
+                "Generative AI tool they used, e.g. 'Claude Opus 5'. "
+                "Repeat for several."
+            ),
+        ),
+    ] = [],
     no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
     overwrite: Annotated[
         bool,
@@ -1223,11 +1314,16 @@ def new_dataset(
         raise_error("Command must be provided")
     if (deps or outs or outs_from_stage) and not stage_name:
         raise_error("Stage name must be provided")
-    obj = dict(path=path, title=title)
+    created_by = _person_from_options(
+        "created-by", created_by_email, created_by_orcid, created_with_ai
+    )
+    obj: dict = dict(path=path, title=title)
     if description is not None:
         obj["description"] = description
     if stage_name is not None:
         obj["stage"] = stage_name
+    if created_by is not None:
+        obj["created_by"] = created_by
     if cmd:
         if outs_from_stage:
             pipeline = calkit.dvc.read_pipeline()

calkit/cli/update.py

@@ -1258,6 +1258,45 @@ def update_dataset(
             help="URL the dataset was imported from.",
         ),
     ] = None,
+    imported_from_doi: Annotated[
+        str | None,
+        typer.Option(
+            "--imported-from-doi",
+            help="DOI the dataset was imported from, e.g. 10.5281/zenodo.1.",
+        ),
+    ] = None,
+    imported_from_git_url: Annotated[
+        str | None,
+        typer.Option(
+            "--imported-from-git-url",
+            help="Clone URL of the Git repo the dataset was imported from.",
+        ),
+    ] = None,
+    imported_from_git_rev: Annotated[
+        str | None,
+        typer.Option(
+            "--imported-from-git-rev",
+            help=(
+                "Commit hash it was taken from. A branch or tag isn't "
+                "accepted, since it would move."
+            ),
+        ),
+    ] = None,
+    imported_from_git_path: Annotated[
+        str | None,
+        typer.Option(
+            "--imported-from-git-path",
+            help="Path within that repo, if it isn't the whole thing.",
+        ),
+    ] = None,
+    imported_from_date: Annotated[
+        datetime | None,
+        typer.Option(
+            "--imported-from-date",
+            formats=["%Y-%m-%d"],
+            help="Date it was downloaded, as YYYY-MM-DD.",
+        ),
+    ] = None,
     stage: Annotated[
         str | None,
         typer.Option(
@@ -1267,23 +1306,90 @@ def update_dataset(
     ] = None,
 ) -> None:
     """Update a dataset entry in calkit.yaml."""
-    if imported_from_url is None and stage is None:
+    from pydantic import ValidationError
+
+    from calkit.models.core import (
+        Dataset,
+        _GitSource,
+        _ImportedFromDoi,
+        _ImportedFromGit,
+        _ImportedFromUrl,
+    )
+
+    # One source, since the entry records where the data came from rather
+    # than every place it could be found
+    source_options = {
+        "--imported-from-url": imported_from_url,
+        "--imported-from-doi": imported_from_doi,
+        "--imported-from-git-url": imported_from_git_url,
+    }
+    sources_given = [k for k, v in source_options.items() if v is not None]
+    if len(sources_given) > 1:
+        raise_error("Specify only one of " + ", ".join(source_options) + ".")
+    if not sources_given and (
+        imported_from_git_rev is not None
+        or imported_from_git_path is not None
+        or imported_from_date is not None
+    ):
+        raise_error(
+            "--imported-from-git-rev, --imported-from-git-path, and "
+            "--imported-from-date go with one of "
+            + ", ".join(source_options)
+            + "."
+        )
+    if not sources_given and stage is None:
         raise_error("No updates specified.")
+    imported_from: dict | None = None
+    if sources_given:
+        date = imported_from_date.date() if imported_from_date else None
+        try:
+            source: _ImportedFromUrl | _ImportedFromDoi | _ImportedFromGit
+            if imported_from_url is not None:
+                source = _ImportedFromUrl(url=imported_from_url, date=date)
+            elif imported_from_doi is not None:
+                source = _ImportedFromDoi(doi=imported_from_doi, date=date)
+            else:
+                if imported_from_git_rev is None:
+                    raise_error(
+                        "--imported-from-git-rev is required with "
+                        "--imported-from-git-url."
+                    )
+                source = _ImportedFromGit(
+                    git=_GitSource(
+                        repo_url=calkit.normalize_git_url(
+                            imported_from_git_url or ""
+                        ),
+                        rev=imported_from_git_rev,
+                        path=imported_from_git_path,
+                    ),
+                    date=date,
+                )
+        except ValidationError as e:
+            raise_error(
+                "Invalid import source: "
+                + "; ".join(str(err["msg"]) for err in e.errors())
+            )
+        imported_from = source.model_dump(exclude_none=True)
     ck_info = calkit.load_calkit_info()
     datasets = ck_info.get("datasets", [])
     for ds in datasets:
         if ds.get("path") == path:
-            if imported_from_url is not None:
-                ds["imported_from"] = {"url": imported_from_url}
-            if stage is not None:
-                ds["stage"] = stage
             break
     else:
-        entry: dict = {"path": path}
-        if imported_from_url is not None:
-            entry["imported_from"] = {"url": imported_from_url}
-        if stage is not None:
-            entry["stage"] = stage
-        datasets.append(entry)
+        ds = {"path": path}
+        datasets.append(ds)
         ck_info["datasets"] = datasets
+    if imported_from is not None:
+        ds["imported_from"] = imported_from
+    if stage is not None:
+        ds["stage"] = stage
+    # Checked as a whole, so an import added to a dataset someone collected
+    # is refused here rather than left for the next validation to find
+    try:
+        Dataset.model_validate(ds)
+    except ValidationError as e:
+        raise_error(
+            "Invalid dataset: "
+            + "; ".join(str(err["msg"]) for err in e.errors())
+        )
     calkit.save_calkit_info(ck_info)

calkit/core.py

@@ -1123,6 +1123,21 @@ def detect_project_name(
     return name
 
 
+def normalize_git_url(url: str) -> str:
+    """An SSH GitHub URL as its HTTPS form; anything else unchanged.
+
+    ``git@github.com:owner/repo.git`` and ``https://github.com/owner/repo``
+    name the same repo, and only the latter is a link, so that's the one
+    worth recording.
+    """
+    url = url.strip()
+    if url.startswith("git@github.com:"):
+        url = "https://github.com/" + url[len("git@github.com:") :]
+    if url.startswith("https://github.com/") and url.endswith(".git"):
+        url = url[: -len(".git")]
+    return url
+
+
 def detect_project_github_url(wdir: str | None = None) -> str | None:
     """Detect the GitHub URL for the current project."""
     try:
@@ -1134,8 +1149,7 @@ def detect_project_github_url(wdir: str | None = None) -> str | None:
         warnings.warn("Git remote is not a GitHub URL")
         return None
     url = url.removesuffix(".git")
-    if url.startswith("git@github.com:"):
-        url = url.replace("git@github.com:", "https://github.com/")
+    url = normalize_git_url(url)
     return url
 
 

calkit/dvc/core.py

@@ -8,6 +8,7 @@ import json
 import logging
 import os
 import sys
+from itertools import groupby
 from pathlib import Path
 from typing import Any, Literal
 
@@ -97,6 +98,60 @@ if sys.platform == "win32":
     _tolerate_lock_release_failures()
 
 
+def _hash_dirs_inside_nested_repos() -> None:
+    """Make DVC hash directories that live inside nested DVC repos.
+
+    DVC's ``DvcIgnoreFilter`` prunes nested DVC repos (e.g. isolated
+    subprojects) when walking, so the parent never indexes a subproject's
+    ``dvc.yaml`` and ``.dvc`` files as its own. The same walk hashes directory
+    deps and outs, though, and the subrepo check is order-dependent:
+    ``_get_trie_pattern`` hands the starting directory's ``dnames`` to every
+    ancestor it visits, so a walk that starts inside the nested repo never
+    registers it, while the root-level index walk does. Whichever touches the
+    ignore trie first wins for the rest of the process. In practice a parent
+    stage depending on a directory inside an isolated subproject hashes it as
+    *empty* on ``status`` but not always on ``repro``, so ``dvc.lock`` and
+    ``dvc status`` disagree and the stage is forever stale (or, worse, changes
+    in the subproject go unnoticed).
+
+    A walk that starts at or below a nested repo can only be a caller hashing
+    that path (the index walk always starts at the repo root), and the caller
+    named the path deliberately, so its contents must be hashed. Such walks
+    are routed through DVC's ``ignore_subrepos=False`` trie, which is built
+    separately, is deterministic, and still honors every ``.dvcignore`` on
+    the way down, including the nested repo's own.
+    """
+    from dvc.ignore import DvcIgnoreFilter
+    from dvc.repo import Repo
+
+    original_walk = DvcIgnoreFilter.walk
+
+    def in_nested_repo(dvcignore: Any, path: str) -> bool:
+        fs = dvcignore.fs
+        root_dir = dvcignore.root_dir
+        current = fs.abspath(path)
+        if not fs.isin(current, root_dir):
+            return False
+        while current != root_dir:
+            if fs.exists(fs.join(current, Repo.DVC_DIR)):
+                return True
+            parent = fs.parent(current)
+            if parent == current:
+                return False
+            current = parent
+        return False
+
+    def walk(self: Any, fs: Any, path: str, **kwargs: Any) -> Any:
+        if "ignore_subrepos" not in kwargs and in_nested_repo(self, path):
+            kwargs["ignore_subrepos"] = False
+        return original_walk(self, fs, path, **kwargs)
+
+    DvcIgnoreFilter.walk = walk
+
+
+_hash_dirs_inside_nested_repos()
+
+
 # Default seconds to wait for DVC's repo-level lock during a pipeline run.
 #
 # DVC's own default is only 3 seconds (``dvc.lock.DEFAULT_TIMEOUT``), after
@@ -354,6 +409,34 @@ def get_dvc_repo(wdir: str | None = None) -> dvc.repo.Repo:
     return dvc.repo.Repo(wdir)
 
 
+def commit_path(
+    dvc_repo: dvc.repo.Repo, path: str, allow_missing: bool = True
+) -> None:
+    """Commit a path to DVC, skipping the run cache.
+
+    This is like ``dvc_repo.commit(path, force=True)``, except stages are not
+    saved to the run cache, since doing so requires hashing all of their
+    dependencies, which will fail if any are missing from the workspace, e.g.,
+    if their data has never been pulled.
+    """
+    with dvc.repo.lock_repo(dvc_repo):
+        groups = groupby(
+            dvc_repo.stage.collect_granular(path),
+            key=lambda info: info.stage.dvcfile,
+        )
+        for dvcfile, stages_info_group in groups:
+            to_dump = []
+            for stage_info in stages_info_group:
+                stage = stage_info.stage
+                stage.save(allow_missing=allow_missing, run_cache=False)
+                stage.commit(
+                    filter_info=stage_info.filter_info,
+                    allow_missing=allow_missing,
+                )
+                to_dump.append(stage)
+            dvcfile.dump_stages(to_dump, update_pipeline=False)
+
+
 def ensure_dvc_lock_not_ignored(wdir: str | None = None) -> bool:
     """Ensure ``dvc.lock`` is not Git-ignored.