calkit-python 0.44.1__py3-none-any.whl → 0.45.1__py3-none-any.whl

calkit/__init__.py

@@ -10,7 +10,6 @@ from .core import *  # noqa: F403, I001
 if TYPE_CHECKING:
     from calkit import (  # noqa: F401
         calc,
-        check,
         conda,
         config,
         datasets,
@@ -37,7 +36,10 @@ if TYPE_CHECKING:
         ops,
         overleaf,
         pipeline,
+        procedures,
+        provenance,
         releases,
+        reproducibility,
         resources,
         schema,
         server,
@@ -59,7 +61,9 @@ _SUBMODULES = {
     "templates",
     "conda",
     "calc",
-    "check",
+    "procedures",
+    "provenance",
+    "reproducibility",
     "github",
     "invenio",
     "releases",

calkit/cli/check.py

@@ -197,7 +197,19 @@ def _check_julia_env(
     cmd += [
         f"--project={env_dir}",
         "-e",
-        "using Pkg; Pkg.instantiate();",
+        # Resolve only when the manifest no longer matches the project
+        # (e.g. a dependency was added, or the manifest was written by a
+        # different Julia version): plain instantiate would install it
+        # as-is and fail later at precompile. A current manifest is
+        # installed exactly as locked, with no registry involved---
+        # resolving one unconditionally can fail spuriously (and would
+        # rewrite the lock). Older Julias without the check keep the
+        # plain instantiate they always had.
+        "using Pkg; "
+        "if hasmethod(Pkg.is_manifest_current, Tuple{String}) && "
+        "Pkg.is_manifest_current(dirname(Base.active_project())) == false; "
+        "Pkg.resolve(); end; "
+        "Pkg.instantiate();",
     ]
     if julia_version:
         try:
@@ -296,7 +308,7 @@ def check_repro(
     ] = ".",
 ) -> None:
     """Check the reproducibility of a project."""
-    from calkit.check import check_reproducibility
+    from calkit.reproducibility import check_reproducibility
 
     res = check_reproducibility(wdir=wdir, log_func=typer.echo)
     calkit.echo(res.to_pretty())

calkit/cli/import_.py

@@ -208,7 +208,9 @@ def import_dataset(
             filter_paths=filter_paths,
         ),
     )
-    datasets.append(new_ds.model_dump())
+    # Nulls left out, so the entry reads as what was recorded rather than as
+    # a form with most of its fields blank
+    datasets.append(new_ds.model_dump(exclude_none=True))
     ck_info["datasets"] = datasets
     with open("calkit.yaml", "w") as f:
         calkit.ryaml.dump(ck_info, f)
@@ -465,13 +467,24 @@ def import_from_zenodo(
     if kind is not None:
         ck_info = calkit.load_calkit_info()
         items = ck_info.get(kind + "s", [])
+        # A record without a DOI (not every one has been minted one) is still
+        # somewhere in particular, so fall back to its URL rather than
+        # writing ``doi: null``, which nothing would then accept
+        doi = calkit.invenio.extract_doi(record)
+        if doi is not None:
+            imported_from: dict = {"doi": doi}
+        else:
+            imported_from = {
+                "url": record.get("links", {}).get("self_html")
+                or f"https://zenodo.org/records/{record_id}"
+            }
         item_record = {
             "path": dest_dir,
-            "imported_from": {
-                "doi": record.get("doi"),
-            },
-            "title": record.get("metadata", {}).get("title"),
+            "imported_from": imported_from,
         }
+        title = record.get("metadata", {}).get("title")
+        if title:
+            item_record["title"] = title
         items.append(item_record)
         ck_info[kind + "s"] = items
         with open("calkit.yaml", "w") as f:

calkit/cli/list.py

@@ -45,6 +45,7 @@ def _list_objects(
         "figures",
         "references",
         "publications",
+        "misc",
     ],
     json_output: bool = False,
 ):
@@ -241,6 +242,18 @@ def list_publications(
     _list_objects("publications", json_output)
 
 
+@list_app.command(name="misc")
+def list_misc(
+    json_output: Annotated[
+        bool, typer.Option("--json", help="Output result as JSON.")
+    ] = False,
+):
+    """List misc artifacts in the project, i.e., attributed paths that
+    aren't one of the typed kinds.
+    """
+    _list_objects("misc", json_output)
+
+
 @list_app.command(name="references|refs")
 def list_references(
     json_output: Annotated[

calkit/cli/main/core.py

@@ -125,6 +125,17 @@ def main(
         bool,
         typer.Option("--version", help="Show version and exit."),
     ] = False,
+    directory: Annotated[
+        Optional[str],
+        typer.Option(
+            "-C",
+            "--directory",
+            help=(
+                "Change to this directory before doing anything else, so a "
+                "project can be worked with from outside it."
+            ),
+        ),
+    ] = None,
     use_version: Annotated[
         Optional[str],
         typer.Option(
@@ -139,6 +150,13 @@ def main(
     if version:
         typer.echo(f"Calkit {calkit.__version__}")
         raise typer.Exit()
+    # Change directory first, so everything below---the project's .env, the
+    # config it reads, and whatever command runs next---sees the project
+    # rather than wherever the user happened to be standing.
+    if directory is not None:
+        if not os.path.isdir(directory):
+            raise_error(f"Directory does not exist: {directory}")
+        os.chdir(directory)
     if use_version:
         _exec_with_version(use_version)
     # Load the project's .env here rather than in the handful of commands
@@ -214,6 +232,21 @@ def _exec_with_version(version_spec: str) -> None:
     os.execvp(cmd[0], cmd)
 
 
+def project_is_initialized() -> bool:
+    """Determine whether the working directory is a Calkit project."""
+    import git
+    from git.exc import InvalidGitRepositoryError
+
+    # Parent directories count, since a project can be a self-contained
+    # directory inside a larger repo and so has no .git of its own. What
+    # makes it initialized is its own DVC repo and calkit.yaml.
+    try:
+        git.Repo(".", search_parent_directories=True)
+    except InvalidGitRepositoryError:
+        return False
+    return os.path.isfile(".dvc/config") and os.path.isfile("calkit.yaml")
+
+
 @app.command(name="init")
 def init(
     force: Annotated[
@@ -224,43 +257,54 @@ def init(
             help="Re-initialize even if the project is already initialized.",
         ),
     ] = False,
+    no_commit: Annotated[
+        bool,
+        typer.Option(
+            "--no-commit",
+            help="Stage the initial files rather than committing them.",
+        ),
+    ] = False,
 ):
     """Initialize the current working directory."""
-    import git
     from git.exc import InvalidGitRepositoryError
 
-    def _project_is_initialized() -> bool:
-        try:
-            git.Repo(".", search_parent_directories=False)
-        except InvalidGitRepositoryError:
-            return False
-        return os.path.isfile(".dvc/config") and os.path.isfile("calkit.yaml")
-
-    if _project_is_initialized() and not force:
+    if project_is_initialized() and not force:
         raise_error(
             "This project is already initialized. "
             "Use --force to re-initialize."
         )
-    subprocess.run(["git", "init"])
-    result = calkit.dvc.run_dvc_command(
-        ["init"] + (["--force"] if force else [])
-    )
+    # Only create a Git repo if we're not already inside one; a project
+    # can be a self-contained directory within a larger repo, and nesting
+    # a second repo there hides its contents from the outer one.
+    try:
+        calkit.git.get_repo()
+        # A directory the enclosing repo ignores is not part of it, so it
+        # gets its own repo rather than being treated as a subdirectory of
+        # one that will never track it.
+        needs_own_repo = calkit.dvc.enclosing_repo_ignores()
+    except InvalidGitRepositoryError:
+        needs_own_repo = True
+    if needs_own_repo:
+        subprocess.run(["git", "init"])
+    result = calkit.dvc.init(force=force)
     if result != 0:
         raise_error("Failed to initialize DVC")
     # Ensure autostage is enabled for DVC
     result = calkit.dvc.run_dvc_command(["config", "core.autostage", "true"])
     if result != 0:
         raise_error("Failed to configure DVC autostage")
-    # Commit the newly created .dvc directory
+    # Commit the newly created .dvc directory. Paths are made absolute
+    # because Git resolves them against the repo root, which is not this
+    # directory when the project is a subdirectory of a larger repo.
     repo = calkit.git.get_repo()
-    repo.git.add(".dvc")
-    if calkit.git.get_staged_files(repo=repo):
+    repo.git.add(os.path.abspath(".dvc"))
+    if not no_commit and calkit.git.get_staged_files(repo=repo):
         repo.git.commit("-m", "Initialize DVC")
     # Create an empty calkit.yaml if one doesn't already exist
     if not os.path.isfile("calkit.yaml"):
         calkit.schema.ensure_modeline("calkit.yaml")
-        repo.git.add("calkit.yaml")
-        if calkit.git.get_staged_files(repo=repo):
+        repo.git.add(os.path.abspath("calkit.yaml"))
+        if not no_commit and calkit.git.get_staged_files(repo=repo):
             repo.git.commit("-m", "Initialize Calkit")
     # TODO: Initialize `dvc.yaml`
     # TODO: Add a sane .gitignore file
@@ -485,7 +529,7 @@ def get_status(
         if not os.path.isfile(os.path.join(".dvc", "config")):
             typer.echo("Initializing DVC repository")
             try:
-                result = calkit.dvc.run_dvc_command(["init", "-q"])
+                result = calkit.dvc.init(quiet=True)
                 if result != 0:
                     raise subprocess.CalledProcessError(result, "dvc init")
             except subprocess.CalledProcessError as e:
@@ -691,7 +735,17 @@ def get_status(
                 if stale_stage.modified_inputs:
                     typer.echo("          modified inputs:")
                     for input_path in stale_stage.modified_inputs:
-                        typer.echo(f"            {input_path}")
+                        ignored = stale_stage.ignored_files_in_inputs.get(
+                            input_path
+                        )
+                        if ignored:
+                            warn(
+                                f"            {input_path} (contains "
+                                f"Git-ignored files: {', '.join(ignored)})",
+                                prefix="",
+                            )
+                        else:
+                            typer.echo(f"            {input_path}")
             if pipeline_status.always_run_stage_names:
                 typer.echo("Always-run stages:")
                 for stage_name in pipeline_status.always_run_stage_names:
@@ -702,6 +756,22 @@ def get_status(
                 typer.echo("Always-run stages:")
                 for stage_name in pipeline_status.always_run_stage_names:
                     typer.echo(f"        {typer.style(stage_name, fg='cyan')}")
+        # Ignored files aren't carried to other machines by Git, so the stage
+        # hashes its input differently there and looks stale, e.g., in CI
+        if pipeline_status and pipeline_status.ignored_files_in_inputs:
+            warn(
+                "Stage inputs contain Git-ignored files, which can make "
+                "stages stale on other machines, e.g., in CI:"
+            )
+            for (
+                stage_name,
+                inputs,
+            ) in pipeline_status.ignored_files_in_inputs.items():
+                warn(f"        {stage_name}:", prefix="")
+                for input_path, files in inputs.items():
+                    warn(f"          {input_path}:", prefix="")
+                    for path in files:
+                        warn(f"            {path}", prefix="")
 
 
 @app.command(name="diff")
@@ -814,9 +884,26 @@ def add(
             typer.echo(
                 "This is not a DVC repository; would initialize DVC here"
             )
+        elif (
+            calkit.dvc.dvc_init_needs_subdir()
+            and not os.path.isfile("calkit.yaml")
+            and not os.path.isfile("dvc.yaml")
+        ):
+            # Bootstrapping a project from nothing is fine, and is what
+            # this does at the root of a new repo. What isn't fine is doing
+            # it in some subdirectory of an existing repo, which would put
+            # a .dvc directory wherever the user happened to be standing.
+            raise_error(
+                f"No calkit.yaml or dvc.yaml in {os.getcwd()}, which is a "
+                "subdirectory of an existing Git repository, so this is "
+                "not a project. Run 'calkit init' first, or change to the "
+                "project directory."
+            )
         else:
             warn("DVC not initialized yet; initializing")
-            dvc_repo = dvc.repo.Repo.init()
+            dvc_repo = dvc.repo.Repo.init(
+                subdir=calkit.dvc.dvc_init_needs_subdir()
+            )
     if not dry_run:
         # Ensure autostage is enabled for DVC
         calkit.dvc.run_dvc_command(
@@ -857,9 +944,11 @@ def add(
             if dvc_repo is not None:
                 dvc_status = dvc_repo.data_status()
                 uncommitted = dvc_status["uncommitted"]
+                dvc_not_in_cache = set(dvc_status.get("not_in_cache", []))
                 dvc_uncommitted_all = uncommitted.get(
                     "modified", []
                 ) + uncommitted.get("deleted", [])
+                missing_from_cache = []
                 for dvc_uncommitted in dvc_uncommitted_all:
                     if os.path.exists(dvc_uncommitted):
                         if dry_run:
@@ -868,11 +957,11 @@ def add(
                             )
                         else:
                             typer.echo(f"Adding {dvc_uncommitted} to DVC")
-                            dvc_repo.commit(
-                                dvc_uncommitted,
-                                force=True,
-                                allow_missing=True,
-                            )
+                            calkit.dvc.commit_path(dvc_repo, dvc_uncommitted)
+                    elif dvc_uncommitted in dvc_not_in_cache:
+                        # The data isn't on this machine at all, so this isn't
+                        # a real deletion and there's nothing to commit
+                        missing_from_cache.append(dvc_uncommitted)
                     else:
                         if dry_run:
                             typer.echo(
@@ -882,11 +971,20 @@ def add(
                             typer.echo(
                                 f"Committing deleted {dvc_uncommitted} to DVC"
                             )
-                            dvc_repo.commit(
-                                dvc_uncommitted,
-                                force=True,
-                                allow_missing=True,
-                            )
+                            calkit.dvc.commit_path(dvc_repo, dvc_uncommitted)
+                if missing_from_cache:
+                    n_shown = 5
+                    paths_txt = ", ".join(missing_from_cache[:n_shown])
+                    if len(missing_from_cache) > n_shown:
+                        paths_txt += (
+                            f", and {len(missing_from_cache) - n_shown} more"
+                        )
+                    warn(
+                        f"Not committing {len(missing_from_cache)} missing "
+                        f"DVC-tracked path(s) since their data is not in the "
+                        f"cache ({paths_txt}); "
+                        "run 'calkit pull' to fetch their data"
+                    )
             if not disable_auto_ignore:
                 for untracked_file in untracked_git_files:
                     if (
@@ -1427,6 +1525,136 @@ STAGE_OUTPUT_START = "--- calkit stage output ---"
 STAGE_OUTPUT_END = "--- end calkit stage output ---"
 
 
+def _inject_markdown_stage_output(log_content: str) -> list[str]:
+    """Write each stage's output into its Markdown file's output blocks.
+
+    This is post-processing rather than a stage output: two stages in one
+    file would both claim the Markdown, and DVC forbids two stages
+    producing the same path. Output blocks are never read back when
+    extracting scripts, so writing one can't make its stage stale.
+
+    Returns the names of the DVC stages whose cached output changed. Each
+    of those depends on its cache, which was hashed before the stage ran,
+    so the caller has to record the new hash or the stage reads as stale.
+    """
+    import calkit.markdown
+
+    stdout_by_stage = _stage_stdout_from_log_content(log_content)
+    if not stdout_by_stage:
+        return []
+    ck_info = calkit.load_calkit_info()
+    md_stages = calkit.markdown.get_markdown_stages(ck_info)
+    if not md_stages:
+        return []
+    sep = calkit.markdown.STAGE_NAME_SEPARATOR
+    # Group by Markdown stage so each file is read and written once
+    by_stage: dict[str, dict[str, str]] = {}
+    for dvc_stage_name, output in stdout_by_stage.items():
+        if sep not in dvc_stage_name:
+            continue
+        md_stage_name, block_name = dvc_stage_name.rsplit(sep, 1)
+        if md_stage_name not in md_stages:
+            continue
+        by_stage.setdefault(md_stage_name, {})[block_name] = output
+    changed_stages = []
+    for md_stage_name, outputs in by_stage.items():
+        md_path = md_stages[md_stage_name]
+        if not os.path.isfile(md_path):
+            continue
+        with open(md_path, encoding="utf-8") as f:
+            text = f.read()
+        new_text, changed = calkit.markdown.set_output_blocks(text, outputs)
+        if changed:
+            with open(md_path, "w", encoding="utf-8", newline="\n") as f:
+                f.write(new_text)
+        # Keep each block's cache in step with the block itself. Read back
+        # from the written text rather than from what was injected, so the
+        # cache can only ever hold exactly what the file says.
+        cached = calkit.markdown.extract_outputs(
+            calkit.markdown.parse_markdown(new_text, path=md_path),
+            md_path,
+        )
+        before = {}
+        for block_name in cached:
+            cache_path = calkit.markdown.get_output_cache_path(
+                md_path, block_name
+            )
+            if os.path.isfile(cache_path):
+                with open(cache_path, encoding="utf-8") as f:
+                    before[block_name] = f.read()
+        cache_paths = calkit.markdown.write_output_caches(cached, md_path)
+        for block_name, cache_path in cache_paths.items():
+            with open(cache_path, encoding="utf-8") as f:
+                if f.read() != before.get(block_name):
+                    changed_stages.append(md_stage_name + sep + block_name)
+    return changed_stages
+
+
+def _inject_markdown_values() -> list[str]:
+    """Bring each Markdown file's value markers into line with its results.
+
+    Done after every run rather than only for stages that ran: the
+    results files are tracked outputs, so whatever they hold is what the
+    prose should say. Returns the paths of the files that changed.
+    """
+    import calkit.markdown
+
+    ck_info = calkit.load_calkit_info()
+    changed_paths = []
+    for md_path in set(calkit.markdown.get_markdown_stages(ck_info).values()):
+        if not os.path.isfile(md_path):
+            continue
+        with open(md_path, encoding="utf-8") as f:
+            text = f.read()
+        new_text, changed = calkit.markdown.set_values(text, md_path)
+        if changed:
+            with open(md_path, "w", encoding="utf-8", newline="\n") as f:
+                f.write(new_text)
+            changed_paths.append(md_path)
+    return changed_paths
+
+
+def _stage_stdout_from_log_content(log_content: str) -> dict[str, str]:
+    """Collect each stage's captured standard output from a run log.
+
+    The run log already brackets stage output with the markers above, so
+    this reads what a stage printed without capturing it a second time.
+    Stages that ran but printed nothing map to an empty string, which is
+    what lets a stale output block be emptied rather than left wrong.
+    """
+    res: dict[str, list[str]] = {}
+    current_stage_name: str | None = None
+    in_stage_output = False
+    for line in log_content.splitlines():
+        if line == STAGE_OUTPUT_START:
+            in_stage_output = True
+            continue
+        if line.endswith(STAGE_OUTPUT_END):
+            # The marker shares its line with output that lacked a final
+            # newline, so keep whatever came before it
+            remainder = line.removesuffix(STAGE_OUTPUT_END)
+            if remainder and current_stage_name is not None:
+                res.setdefault(current_stage_name, []).append(remainder)
+            in_stage_output = False
+            continue
+        if in_stage_output:
+            if current_stage_name is not None:
+                res.setdefault(current_stage_name, []).append(line)
+            continue
+        ls = line.split(" -", maxsplit=2)
+        if len(ls) < 3:
+            continue
+        message = ls[2].strip()
+        if message.startswith("Running stage "):
+            current_stage_name = (
+                message.removeprefix("Running stage ").rstrip(":").strip("'")
+            )
+            # Recorded now so a stage that printed nothing is still known
+            # to have run
+            res.setdefault(current_stage_name, [])
+    return {name: "\n".join(lines) for name, lines in res.items()}
+
+
 def _stage_run_info_from_log_content(
     log_content: str, run_finished: bool = False
 ) -> dict:
@@ -2025,6 +2253,17 @@ def run(
     if mock_scheduler:
         os.environ[calkit.cli.scheduler.MOCK_ENV_VAR] = "1"
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
+    # Refuse to run anywhere that isn't a project. Everything below
+    # initializes Git and DVC as needed, so without this check a mistyped
+    # path or a wrong working directory would silently scatter a .dvc
+    # directory into an unrelated folder.
+    if not os.path.isfile("calkit.yaml") and not os.path.isfile("dvc.yaml"):
+        os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
+        raise_error(
+            f"No calkit.yaml or dvc.yaml in {os.getcwd()}, so there is no "
+            "pipeline to run here. Run 'calkit init' to make this a "
+            "project, or change to the project directory."
+        )
     ck_info = calkit.load_calkit_info()
     # Ensure Git is initialized so DVC can be used.
     # Use search_parent_directories so running from a subproject folder
@@ -2069,6 +2308,14 @@ def run(
     except Exception as e:
         os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
         raise_error(str(e))
+    # Extract anything the project's Markdown files declare before the
+    # environments are checked, since an environment declared there must be
+    # in calkit.yaml before it can be created or entered.
+    try:
+        calkit.pipeline.sync_markdown(ck_info=ck_info)
+    except Exception as e:
+        os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
+        raise_error(f"Failed to read markdown stages: {e}")
     # Check all environments in the pipeline (with caching)
     # If any failed, warn the user that we might have problems running
     calkit.echo("📦 Checking environments")
@@ -2163,7 +2410,7 @@ def run(
     except Exception:
         if not quiet:
             typer.echo("Initializing DVC repo")
-        result = calkit.dvc.run_dvc_command(["init"])
+        result = calkit.dvc.init()
         if result != 0:
             raise_error("Failed to initialize DVC repo")
     # Convert deps into target stage names
@@ -2443,6 +2690,29 @@ def run(
         )
     else:
         failed = failed or res != 0
+    # Write what each stage printed back into any 'calkit output' blocks
+    # in the Markdown that declared it
+    try:
+        md_changed_stages = _inject_markdown_stage_output(log_content)
+    except Exception as e:
+        warn(f"Failed to write stage output into markdown: {e}")
+        md_changed_stages = []
+    if md_changed_stages:
+        # A stage depends on its output cache, which DVC hashed before the
+        # stage ran. Now that the cache holds what the stage just printed,
+        # record it in the lock, or the stage would read as stale and run
+        # again just to arrive at the same place.
+        res_commit = calkit.dvc.run_dvc_command(
+            ["commit", "--force", "--quiet"] + md_changed_stages
+        )
+        if res_commit != 0:
+            warn("Failed to record markdown output in dvc.lock")
+    # ...and bring any values the Markdown shows into line with the
+    # results files the stages produced
+    try:
+        _inject_markdown_values()
+    except Exception as e:
+        warn(f"Failed to write values into markdown: {e}")
     # Zip dvc-zip outputs for stages that actually ran
     if stage_run_info:
         from calkit.models.io import PathOutput
@@ -3247,15 +3517,14 @@ def run_procedure(
             return bool(value)
         return value
 
-    from calkit.models import Procedure
-
     ck_info = calkit.load_calkit_info()
     calkit.set_env_vars(ck_info=ck_info)
-    procs = ck_info.get("procedures", {})
-    if name not in procs:
+    if name not in ck_info.get("procedures", {}):
         raise_error(f"'{name}' is not defined as a procedure")
+    # Read from its own file if that's where it's kept, so the rest of this
+    # sees the same thing either way
     try:
-        proc = Procedure.model_validate(procs[name])
+        proc = calkit.procedures.load(name, ck_info=ck_info)
     except Exception as e:
         raise_error(f"Procedure '{name}' is invalid: {e}")
     git_repo = calkit.git.get_repo()