calkit-python 0.35.6__tar.gz → 0.35.7__tar.gz

{calkit_python-0.35.6 → calkit_python-0.35.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.35.6
+Version: 0.35.7
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.35.6 → calkit_python-0.35.7}/calkit/cli/main/core.py

@@ -55,7 +55,7 @@ from calkit.cli.office import office_app
 from calkit.cli.overleaf import overleaf_app
 from calkit.cli.slurm import slurm_app
 from calkit.cli.update import update_app
-from calkit.dvc import register_ck_scheme, run_dvc_command
+from calkit.dvc import get_dvc_repo, run_dvc_command
 from calkit.environments import get_env_lock_fpath
 from calkit.models import Procedure
 
@@ -252,10 +252,12 @@ def get_status(
 ):
     """View status (project, version control, and/or pipeline)."""
     ck_info = calkit.load_calkit_info()
-    try:
-        calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
-    except Exception as e:
-        warn(f"Failed to compile pipeline: {e.__class__.__name__}: {e}")
+    # If there's anything in ck_info and this isn't a Git repo, initialize one
+    if ck_info:
+        try:
+            git.Repo()
+        except InvalidGitRepositoryError:
+            git.Repo.init()
     valid_categories = ["project", "git", "dvc", "pipeline"]
     if categories is not None:
         for category in categories:
@@ -266,11 +268,22 @@ def get_status(
                 )
     else:
         categories = valid_categories
-    # Clean all notebooks in the pipeline
-    try:
-        calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
-    except Exception as e:
-        warn(f"Failed to clean notebooks: {e.__class__.__name__}: {e}")
+    pipeline_status = None
+    if "pipeline" in categories or "dvc" in categories:
+        pipeline_status = calkit.pipeline.get_status(
+            ck_info=ck_info,
+            targets=targets,
+            check_environments=True,
+            clean_notebooks=True,
+            compile_to_dvc=True,
+        )
+        for error in pipeline_status.errors:
+            warn(error)
+        if pipeline_status.failed_environment_checks:
+            warn(
+                "Failed pipeline environment checks for: "
+                + ", ".join(pipeline_status.failed_environment_checks)
+            )
     if "project" in categories:
         print_sep("Project")
         # Print latest status
@@ -307,10 +320,32 @@ def get_status(
         typer.echo()
     if "pipeline" in categories or "dvc" in categories:
         print_sep("Pipeline")
-        dvc_pipeline_cmd = ["status"]
-        if targets:
-            dvc_pipeline_cmd += targets
-        run_dvc_command(dvc_pipeline_cmd)
+        # Nicely format the results from pipeline status
+        if pipeline_status and pipeline_status.errors:
+            typer.echo("Pipeline status unavailable due to errors.")
+        elif pipeline_status and pipeline_status.is_stale:
+            for stage_name in pipeline_status.stale_stage_names:
+                stale_stage = pipeline_status.stale_stages.get(stage_name)
+                if stale_stage is None:
+                    continue
+                typer.echo(f"{typer.style(stage_name, fg='yellow')}:")
+                # Show stale outputs for this stage
+                if stale_stage.stale_outputs:
+                    typer.echo("  stale outputs:")
+                    for output_path in stale_stage.stale_outputs:
+                        typer.echo(f"    - {output_path}")
+                # Show modified outputs from this stage
+                if stale_stage.modified_outputs:
+                    typer.echo("  modified outputs:")
+                    for output_path in stale_stage.modified_outputs:
+                        typer.echo(f"    - {output_path}")
+                # Show modified inputs making the stage stale
+                if stale_stage.modified_inputs:
+                    typer.echo("  modified inputs:")
+                    for input_path in stale_stage.modified_inputs:
+                        typer.echo(f"    - {input_path}")
+        elif pipeline_status:
+            typer.echo("Pipeline is up to date.")
 
 
 @app.command(name="diff")
@@ -403,8 +438,7 @@ def add(
             raise_error("Not currently in a Git repo; run `calkit init` first")
         repo = git.Repo()
     try:
-        register_ck_scheme()
-        dvc_repo = dvc.repo.Repo()
+        dvc_repo = get_dvc_repo()
     except NotDvcRepoError:
         warn("DVC not initialized yet; initializing")
         dvc_repo = dvc.repo.Repo.init()
@@ -1139,10 +1173,8 @@ def run(
             os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
             raise_error(f"Pipeline compilation failed: {e}")
     # Initialize DVC repo if necessary
-    # Register the ck:// scheme before accessing DVC repo
-    register_ck_scheme()
     try:
-        dvc.repo.Repo()
+        get_dvc_repo()
     except Exception:
         if not quiet:
             typer.echo("Initializing DVC repo")
@@ -1203,8 +1235,7 @@ def run(
         git_staged_files_before = calkit.git.get_staged_files(repo=repo)
         git_untracked_files_before = calkit.git.get_untracked_files(repo=repo)
         # Get status of DVC repo before running
-        register_ck_scheme()
-        dvc_repo = dvc.repo.Repo()
+        dvc_repo = get_dvc_repo()
         dvc_status_before = dvc_repo.status()
         dvc_data_status_before = dvc_repo.data_status()
         dvc_data_status_before.pop("git", None)  # Remove git status

{calkit_python-0.35.6 → calkit_python-0.35.7}/calkit/cli/new.py

@@ -17,6 +17,7 @@ from git.exc import GitCommandError, InvalidGitRepositoryError, NoSuchPathError
 from typing_extensions import Annotated
 
 import calkit
+import calkit.pipeline
 from calkit.cli import raise_error, warn
 from calkit.cli.check import check_environment
 from calkit.cli.update import update_devcontainer
@@ -2523,6 +2524,26 @@ def new_release(
     repo = git.Repo()
     if name in repo.tags:
         raise_error(f"Git tag with name '{name}' already exists")
+    # Check that the pipeline is up-to-date
+    typer.echo("Checking pipeline is up-to-date for release")
+    status = calkit.pipeline.get_status(
+        ck_info=ck_info,
+        check_environments=True,
+        clean_notebooks=True,
+        compile_to_dvc=True,
+    )
+    if status.errors:
+        raise_error("Pipeline checks failed: " + "; ".join(status.errors))
+    if status.failed_environment_checks:
+        raise_error(
+            "Pipeline environment checks failed for: "
+            + ", ".join(status.failed_environment_checks)
+        )
+    if status.is_stale:
+        raise_error(
+            "Pipeline is not up-to-date; out-of-date stages: "
+            + ", ".join(status.stale_stage_names)
+        )
     release_dir = f".calkit/releases/{name}"
     release_files_dir = release_dir + "/files"
     os.makedirs(release_files_dir, exist_ok=True)
@@ -2543,8 +2564,14 @@ def new_release(
         all_paths = calkit.releases.ls_files()
         typer.echo(f"Adding files to {zip_path}")
         with zipfile.ZipFile(zip_path, "w") as zipf:
-            for fpath in all_paths:
-                zipf.write(fpath)
+            calkit.releases.add_paths_to_zip(zipf, all_paths)
+        typer.echo("Checking extracted project release archive")
+        try:
+            calkit.releases.check_project_release_archive(
+                zip_path, verbose=verbose
+            )
+        except Exception as e:
+            raise_error(str(e))
         title = ck_info.get("title")
         if title is None:
             warn("Project has no title")
@@ -2888,10 +2915,12 @@ def new_release(
             + new_lines[first_content_line_index:]
         )
         readme_txt = "\n".join(new_lines)
-        with open("README.md", "w") as f:
-            f.write(readme_txt)
         if not dry_run:
-            repo.git.add("README.md")
+            with open("README.md", "w") as f:
+                f.write(readme_txt)
+                repo.git.add("README.md")
+        else:
+            typer.echo(f"Would have updated README.md to:\n{readme_txt}")
     # Create Git tag
     git_tag_message = release_description
     if git_tag_message is None:
@@ -2919,13 +2948,15 @@ def new_release(
     ck_info["releases"] = releases
     # Create CITATION.cff file
     if release_type == "project":
-        typer.echo("Writing CITATION.cff")
         cff = calkit.releases.create_citation_cff(
             ck_info=ck_info, release_name=name, release_date=release_date
         )
-        with open("CITATION.cff", "w") as f:
-            calkit.ryaml.dump(cff, f)
-        if not dry_run:
+        if dry_run:
+            typer.echo(f"Would write CITATION.cff:\n{cff}")
+        else:
+            typer.echo("Writing CITATION.cff")
+            with open("CITATION.cff", "w") as f:
+                calkit.ryaml.dump(cff, f)
             repo.git.add("CITATION.cff")
     # Add to references so it can be cited
     typer.echo("Adding BibTeX entry to references")
@@ -2944,28 +2975,50 @@ def new_release(
                 reflib = bibtexparser.load(f)
         else:
             reflib = bibtexparser.bibdatabase.BibDatabase()
+        bibtex_doi = doi
+        if bibtex_doi is None and dry_run:
+            mock_suffix = "".join(
+                ch if (ch.isalnum() or ch in ".-_") else "-"
+                for ch in name.lower()
+            )
+            bibtex_doi = f"10.0000/calkit-dry-run.{mock_suffix}"
         invenio_bibtex = calkit.releases.create_bibtex(
             authors=authors,
             release_date=release_date,
             title=title,  # type: ignore
-            doi=doi,  # type: ignore
+            doi=bibtex_doi,
             record_id=record_id,  # type: ignore
             service=to,  # type: ignore
         )
-        new_entry = bibtexparser.loads(invenio_bibtex).entries[0]
+        new_entries = bibtexparser.loads(invenio_bibtex).entries
+        if not new_entries:
+            raise ValueError("Failed to parse generated BibTeX entry")
+        new_entry = new_entries[0]
         # Search through entries for one with the same DOI, and replace if
         # there is a match
-        existing_index = None
-        for n, entry in enumerate(reflib.entries):
-            if entry.get("doi") == doi:
-                typer.echo("Found matching DOI in existing references")
-                existing_index = n
-        if existing_index is not None:
-            _ = reflib.entries.pop(existing_index)
+        new_doi = new_entry.get("doi")
+        matching_indices = []
+        if new_doi:
+            matching_indices = [
+                n
+                for n, entry in enumerate(reflib.entries)
+                if entry.get("doi") == new_doi
+            ]
+        if matching_indices:
+            typer.echo(
+                "Found matching DOI in existing references "
+                f"({len(matching_indices)} entr"
+                f"{'y' if len(matching_indices) == 1 else 'ies'}); "
+                "replacing"
+            )
+            for n in reversed(matching_indices):
+                _ = reflib.entries.pop(n)
         reflib.entries.append(new_entry)
-        with open(ref_path, "w") as f:
-            bibtexparser.dump(reflib, f)
-        if not dry_run:
+        if dry_run:
+            typer.echo(f"Would write updated references to {ref_path}")
+        else:
+            with open(ref_path, "w") as f:
+                bibtexparser.dump(reflib, f)
             repo.git.add(ref_path)
     except Exception as e:
         warn(f"Failed to add to references: {e}")

{calkit_python-0.35.6 → calkit_python-0.35.7}/calkit/cli/update.py

@@ -12,6 +12,7 @@ import typer
 from typing_extensions import Annotated
 
 import calkit
+import calkit.pipeline
 from calkit.cli import raise_error
 
 update_app = typer.Typer(no_args_is_help=True)
@@ -156,6 +157,26 @@ def update_release(
     record_id = release.get("record_id")
     if record_id is None:
         raise_error("Release has no record ID")
+    if publish or reupload:
+        typer.echo("Checking pipeline is up-to-date for release update")
+        status = calkit.pipeline.get_status(
+            ck_info=ck_info,
+            check_environments=True,
+            clean_notebooks=True,
+            compile_to_dvc=True,
+        )
+        if status.errors:
+            raise_error("Pipeline checks failed: " + "; ".join(status.errors))
+        if status.failed_environment_checks:
+            raise_error(
+                "Pipeline environment checks failed for: "
+                + ", ".join(status.failed_environment_checks)
+            )
+        if status.is_stale:
+            raise_error(
+                "Pipeline is not up-to-date; out-of-date stages: "
+                + ", ".join(status.stale_stage_names)
+            )
     if publish:
         try:
             calkit.invenio.post(
@@ -245,8 +266,12 @@ def update_release(
         all_paths = calkit.releases.ls_files()
         typer.echo(f"Adding files to {zip_path}")
         with zipfile.ZipFile(zip_path, "w") as zipf:
-            for fpath in all_paths:
-                zipf.write(fpath)
+            calkit.releases.add_paths_to_zip(zipf, all_paths)
+        typer.echo("Checking project release archive")
+        try:
+            calkit.releases.check_project_release_archive(zip_path)
+        except Exception as e:
+            raise_error(str(e))
         try:
             files_in_record = [
                 entry["key"]

{calkit_python-0.35.6 → calkit_python-0.35.7}/calkit/dvc.py

@@ -9,6 +9,7 @@ import os
 from pathlib import Path
 from typing import Any
 
+import dvc.repo
 import git
 from dvc.utils.objects import cached_property
 from dvc_objects.fs.base import ObjectFileSystem
@@ -234,6 +235,12 @@ def run_dvc_cli(argv: list[str] | None = None) -> int:
     return dvc_main(argv)
 
 
+def get_dvc_repo(wdir: str | None = None) -> dvc.repo.Repo:
+    """Return a DVC repo with ``ck://`` scheme support registered."""
+    register_ck_scheme()
+    return dvc.repo.Repo(wdir)
+
+
 def run_dvc_command(argv: list[str], cwd: str | None = None) -> int:
     """Run a DVC command, optionally in a specific working directory.
 
@@ -367,14 +374,10 @@ def get_remotes(wdir: str | None = None) -> dict[str, str]:
     """Get a dictionary of DVC remotes, keyed by name, with URL as the
     value.
     """
-    import dvc.repo
     from dvc.exceptions import NotDvcRepoError
 
-    # Ensure DVC recognizes ck:// remotes when validating config.
-    register_ck_scheme()
-    repo_path = wdir or "."
     try:
-        repo = dvc.repo.Repo(repo_path)
+        repo = get_dvc_repo(wdir)
     except NotDvcRepoError:
         return {}
     try:
@@ -403,9 +406,7 @@ def list_files(wdir: str | None = None, recursive=True) -> list[dict]:
     """Return a list with all files in DVC, including their path and md5
     checksum.
     """
-    import dvc.repo
-
-    dvc_repo = dvc.repo.Repo(wdir)
+    dvc_repo = get_dvc_repo(wdir)
     return dvc_repo.ls(".", dvc_only=True, recursive=recursive)
 
 

{calkit_python-0.35.6 → calkit_python-0.35.7}/calkit/jupyterlab/routes.py

@@ -10,8 +10,6 @@ import sys
 import traceback
 from pathlib import Path
 
-import dvc
-import dvc.repo
 import git
 import tornado
 from dvc.exceptions import NotDvcRepoError
@@ -31,7 +29,7 @@ from calkit.cli.new import (
     new_venv,
 )
 from calkit.cli.notebooks import check_env_kernel
-from calkit.dvc import run_dvc_command
+from calkit.dvc import get_dvc_repo, run_dvc_command
 from calkit.environments import DEFAULT_PYTHON_VERSION
 from calkit.git import ensure_path_is_ignored
 from calkit.models.pipeline import JupyterNotebookStage
@@ -897,7 +895,7 @@ class PipelineStatusRouteHandler(APIHandler):
             calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
             # Clean all notebooks in the pipeline
             calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
-            dvc_repo = dvc.repo.Repo(os.getcwd())
+            dvc_repo = get_dvc_repo(os.getcwd())
             raw_status = dvc_repo.status()
             pipeline_status = {
                 k.split("dvc.yaml:")[-1]: v