calkit-python 0.47.1__py3-none-any.whl → 0.47.2__py3-none-any.whl

calkit/cli/check.py

@@ -1802,10 +1802,7 @@ def check_venv(
                 if verbose:
                     typer.echo(f"Using legacy lock file: {legacy_fpath}")
                 break
-    if _platform.system() == "Windows":
-        activate_cmd = f"{prefix}\\Scripts\\activate"
-    else:
-        activate_cmd = f". {prefix}/bin/activate"
+    activate_cmd = calkit.environments.get_venv_activate_cmd(prefix)
 
     def pip_install_and_freeze(reqs_arg: str) -> None:
         check_cmd = (

calkit/cli/main/core.py

@@ -6,7 +6,6 @@ import csv
 import json
 import logging
 import os
-import platform as _platform
 import posixpath
 import shlex
 import shutil
@@ -3543,10 +3542,7 @@ def run_in_env(
                 envs, path, env_name
             )
         shell_cmd = _to_shell_cmd(cmd)
-        if _platform.system() == "Windows":
-            activate_cmd = f"{prefix}\\Scripts\\activate"
-        else:
-            activate_cmd = f". {prefix}/bin/activate"
+        activate_cmd = calkit.environments.get_venv_activate_cmd(prefix)
         if verbose:
             typer.echo(f"Raw command: {cmd}")
             typer.echo(f"Shell command: {shell_cmd}")

calkit/cli/new.py

@@ -3434,6 +3434,18 @@ def new_release(
         str | None,
         typer.Option("--date", help="Release date. Will default to today."),
     ] = None,
+    include_pipeline: Annotated[
+        bool,
+        typer.Option(
+            "--pipeline",
+            help=(
+                "Include everything needed to reproduce the released path, "
+                "i.e., the pipeline, its lock file, and the stages, inputs, "
+                "and environments the path depends on. Stages unrelated to "
+                "the path are left out."
+            ),
+        ),
+    ] = False,
     no_docker_images: Annotated[
         bool,
         typer.Option(
@@ -3521,7 +3533,6 @@ def new_release(
     ] = False,
 ):
     """Create a new release."""
-    import bibtexparser
     import dotenv
 
     import calkit.pipeline
@@ -3536,6 +3547,20 @@ def new_release(
     repo = calkit.git.get_repo()
     if name in repo.tags:
         raise_error(f"Git tag with name '{name}' already exists")
+    # A release commits to calkit.yaml and pushes the branch it's on, neither
+    # of which works from a detached HEAD. Check before anything is uploaded,
+    # so a release can't get published and then fail on the way out.
+    will_push = (
+        not dry_run and not no_push and not no_commit and not draft_only
+    )
+    if repo.head.is_detached and will_push:
+        # Suggest creating a branch rather than checking one out, since in a
+        # worktree the branch they'd want may be checked out elsewhere
+        raise_error(
+            "HEAD is detached, so there is no branch to commit the release "
+            "record to and push; create a branch at this revision first, "
+            "e.g., with `git switch -c <branch>`"
+        )
     # Detect the release kind from the path unless it was given with --kind. A
     # "." path is always a project release; otherwise prefer a declared
     # artifact in calkit.yaml, falling back to auto-detection from the path
@@ -3589,10 +3614,24 @@ def new_release(
     # that produces the released artifact when releasing a single path.
     typer.echo("Checking pipeline is up-to-date for release")
     targets = None
+    # The stage that builds the released path, whose upstream stages define
+    # what a --pipeline release carries
+    pipeline_stage = ""
     if path != ".":
         stage_name = calkit.pipeline.get_stage_for_output(path, ck_info)
-        if stage_name is not None:
+        if stage_name is None:
+            if include_pipeline:
+                raise_error(
+                    f"No pipeline stage produces '{path}', "
+                    "so there is no pipeline to release along with it"
+                )
+        else:
             targets = [stage_name]
+            pipeline_stage = stage_name
+    elif include_pipeline:
+        # A project release carries the whole pipeline already
+        typer.echo("Project releases already include the pipeline")
+        include_pipeline = False
     status = calkit.pipeline.get_status(
         ck_info=ck_info,
         targets=targets,
@@ -3617,6 +3656,15 @@ def new_release(
         release_date = str(calkit.utcnow().date())
     typer.echo(f"Using release date: {release_date}")
     git_rev = repo.git.rev_parse(["--short", "HEAD"])
+    # This goes both beside the archive, which is the copy the archival
+    # service displays, and inside it, so an extracted copy still says what
+    # produced it. Rebuilt below once a more specific title is known.
+    release_readme = calkit.releases.create_release_readme(
+        release_kind=release_kind,
+        name=name,
+        git_rev=git_rev,
+        title=ck_info.get("title"),
+    )
     # Fields below are populated only for external (archival) releases;
     # internal releases leave them empty.
     doi = None
@@ -3636,9 +3684,15 @@ def new_release(
             stored_filename = f"{project_name}-{name}.zip"
             is_zip = True
         elif os.path.isfile(path):
-            _, ext = os.path.splitext(os.path.basename(path))
-            stored_filename = f"{project_name}-{name}{ext}"
-            is_zip = False
+            # Releasing the pipeline along with the artifact means shipping
+            # more than one file, so the artifact gets zipped up with it
+            if include_pipeline:
+                stored_filename = f"{project_name}-{name}.zip"
+                is_zip = True
+            else:
+                _, ext = os.path.splitext(os.path.basename(path))
+                stored_filename = f"{project_name}-{name}{ext}"
+                is_zip = False
         else:
             raise_error(f"Release path '{path}' does not exist")
         stored_path = os.path.join(release_dir, stored_filename)
@@ -3648,10 +3702,38 @@ def new_release(
             typer.echo(f"Would {action} {path} to {stored_path_posix}")
         else:
             os.makedirs(release_dir, exist_ok=True)
-            if is_zip:
+            overrides: dict[str, str] = {}
+            if include_pipeline:
+                typer.echo(f"Pruning project to what builds {path}")
+                try:
+                    overrides, paths = calkit.releases.prune_for_stage(
+                        ck_info, pipeline_stage
+                    )
+                except Exception as e:
+                    raise_error(
+                        f"Failed to prune project for stage "
+                        f"'{pipeline_stage}': {e}"
+                    )
+            elif is_zip:
                 paths = calkit.releases.ls_files() if path == "." else [path]
+            else:
+                paths = []
+            if is_zip:
                 typer.echo(f"Archiving {path} to {stored_path_posix}")
-                calkit.releases.zip_paths(stored_path, paths)
+                calkit.releases.zip_paths(
+                    stored_path,
+                    paths,
+                    overrides=overrides
+                    | {"CALKIT-RELEASE.md": release_readme},
+                )
+                if include_pipeline:
+                    typer.echo("Checking extracted release archive")
+                    try:
+                        calkit.releases.check_project_release_archive(
+                            stored_path, verbose=verbose
+                        )
+                    except Exception as e:
+                        raise_error(str(e))
             else:
                 typer.echo(f"Copying {path} to {stored_path_posix}")
                 shutil.copy2(path, stored_path)
@@ -3679,10 +3761,30 @@ def new_release(
         if path == ".":
             if release_kind is None:
                 release_kind = "project"
+            # Settle the title before building the archive, since the README
+            # that goes inside it is headed with the title
+            title = ck_info.get("title")
+            if title is None:
+                warn("Project has no title")
+                title = typer.prompt("Enter a title for the project")
+                ck_info["title"] = title
+                if not dry_run:
+                    with open("calkit.yaml", "w") as f:
+                        calkit.ryaml.dump(ck_info, f)
+            release_readme = calkit.releases.create_release_readme(
+                release_kind=release_kind,
+                name=name,
+                git_rev=git_rev,
+                title=title,
+            )
             zip_path = release_files_dir + "/archive.zip"
             all_paths = calkit.releases.ls_files()
             typer.echo(f"Adding files to {zip_path}")
-            calkit.releases.zip_paths(zip_path, all_paths)
+            calkit.releases.zip_paths(
+                zip_path,
+                all_paths,
+                overrides={"CALKIT-RELEASE.md": release_readme},
+            )
             typer.echo("Checking extracted project release archive")
             try:
                 calkit.releases.check_project_release_archive(
@@ -3690,14 +3792,6 @@ def new_release(
                 )
             except Exception as e:
                 raise_error(str(e))
-            title = ck_info.get("title")
-            if title is None:
-                warn("Project has no title")
-                title = typer.prompt("Enter a title for the project")
-                ck_info["title"] = title
-                if not dry_run:
-                    with open("calkit.yaml", "w") as f:
-                        calkit.ryaml.dump(ck_info, f)
         else:
             # TODO: Handle directories, e.g., datasets
             if not os.path.isfile(path):
@@ -3724,9 +3818,46 @@ def new_release(
                 )
             if title is None:
                 raise_error(f"{release_kind} at {path} has no title")
+            release_readme = calkit.releases.create_release_readme(
+                release_kind=release_kind,
+                name=name,
+                git_rev=git_rev,
+                title=title,
+            )
+            # Ship the artifact's provenance beside it: the stages that
+            # build it, their inputs and environments, and a pipeline and
+            # lock file pruned to match
+            if include_pipeline:
+                zip_path = release_files_dir + "/archive.zip"
+                typer.echo(f"Pruning project to what builds {path}")
+                try:
+                    overrides, all_paths = calkit.releases.prune_for_stage(
+                        ck_info, pipeline_stage
+                    )
+                except Exception as e:
+                    raise_error(
+                        f"Failed to prune project for stage "
+                        f"'{pipeline_stage}': {e}"
+                    )
+                typer.echo(f"Adding files to {zip_path}")
+                calkit.releases.zip_paths(
+                    zip_path,
+                    all_paths,
+                    overrides=overrides
+                    | {"CALKIT-RELEASE.md": release_readme},
+                )
+                typer.echo("Checking extracted project release archive")
+                try:
+                    calkit.releases.check_project_release_archive(
+                        zip_path, verbose=verbose
+                    )
+                except Exception as e:
+                    raise_error(str(e))
         # Save a metadata file with each DVC file's MD5 checksum
         dvc_md5s = calkit.releases.make_dvc_md5s(
-            zipfile="archive.zip" if path == "." else None,
+            zipfile=(
+                "archive.zip" if path == "." or include_pipeline else None
+            ),
             paths=None if path == "." else [path],
         )
         dvc_md5s_path = release_dir + "/dvc-md5s.yaml"
@@ -3738,7 +3869,7 @@ def new_release(
         # Archive the project's Docker images, so reproducing it doesn't
         # depend on a registry keeping them around, and leave breadcrumbs
         # behind so the environment check can fetch them back
-        if path == "." and not no_docker_images:
+        if (path == "." or include_pipeline) and not no_docker_images:
             typer.echo("Archiving Docker images")
             docker_images = calkit.releases.save_docker_images(
                 release_files_dir
@@ -3752,16 +3883,11 @@ def new_release(
                     calkit.ryaml.dump(docker_images, f)
                 if not dry_run:
                     repo.git.add(docker_images_path)
-        # Create a README for the Zenodo release
-        readme_txt = f"# {title}\n"
-        git_rev = repo.git.rev_parse(["--short", "HEAD"])
-        readme_txt += (
-            f"\nThis is a {release_kind} release ({name}) generated with "
-            f"Calkit v{calkit.__version__} from Git rev {git_rev}.\n"
-        )
+        # Write the same README beside the archive, since this is the copy
+        # the archival service renders on the record page
         readme_path = release_files_dir + "/README.md"
         with open(readme_path, "w") as f:
-            f.write(readme_txt)
+            f.write(release_readme)
         # Check size of files dir
         size = calkit.get_size(release_files_dir)
         typer.echo(f"Release size: {(size / 1e6):.1f} MB")
@@ -4114,6 +4240,7 @@ def new_release(
         description=release_description,
         internal=internal_release,
         stored_path=stored_path_posix,
+        includes_pipeline=include_pipeline,
     ).model_dump()
     releases[name] = release
     ck_info["releases"] = releases
@@ -4167,7 +4294,7 @@ def new_release(
                 record_id=record_id,  # type: ignore
                 service=to,  # type: ignore
             )
-            new_entries = bibtexparser.loads(invenio_bibtex).entries
+            new_entries = calkit.releases.parse_bibtex(invenio_bibtex)
             if not new_entries:
                 raise ValueError("Failed to parse generated BibTeX entry")
             new_entry = new_entries[0]
@@ -4179,9 +4306,9 @@ def new_release(
             replace_ids = []
             if new_doi:
                 try:
-                    existing_entries = bibtexparser.loads(
+                    existing_entries = calkit.releases.parse_bibtex(
                         existing_text
-                    ).entries
+                    )
                 except Exception as e:
                     warn(f"Could not parse existing references to dedupe: {e}")
                     existing_entries = []
@@ -4220,7 +4347,7 @@ def new_release(
     if not dry_run and calkit.git.get_staged_files() and not no_commit:
         repo.git.commit(["-m", f"Create new {release_kind} release {name}"])
     # Push with Git
-    if not dry_run and not no_push and not no_commit and not draft_only:
+    if will_push:
         repo.git.push(["origin", repo.active_branch.name, "--tags"])
         # Now create GitHub release (external releases only)
         if not internal_release and not no_github_release:

calkit/environments.py

@@ -1290,6 +1290,20 @@ def get_default_venv_prefix(envs: dict, path: str, name: str) -> str:
     return Path(base).as_posix()
 
 
+def get_venv_activate_cmd(prefix: str, system: str | None = None) -> str:
+    """Get the shell command that activates the virtualenv at ``prefix``.
+
+    Prefixes are kept POSIX-style, but cmd reads a forward slash as the start
+    of a switch, so it takes ``.calkit/envs/x/.venv`` for a command named
+    ``.calkit``. Hand Windows native separators instead.
+    """
+    if system is None:
+        system = platform.system()
+    if system == "Windows":
+        return prefix.replace("/", "\\") + "\\Scripts\\activate"
+    return f". {prefix}/bin/activate"
+
+
 def env_from_name_or_path(
     name_or_path: str | None = None,
     ck_info: dict | None = None,

calkit/invenio.py

@@ -1,6 +1,7 @@
 """Functionality for working with InvenioRDM instances like Zenodo."""
 
 import os
+import time
 from functools import partial
 from typing import Literal
 
@@ -57,6 +58,24 @@ def get_base_url(service: ServiceName = DEFAULT_SERVICE) -> str:
         raise ValueError(f"Unknown archival service '{service}'")
 
 
+# Pushing a release can mean sending hundreds of megabytes over a slow or
+# distant link, so give a response plenty of time to arrive rather than
+# letting requests wait forever with no timeout at all.
+CONNECT_TIMEOUT = 30
+READ_TIMEOUT = 600
+# Gateways in front of InvenioRDM return these when they're busy, which says
+# nothing about whether the request itself was valid, so it's worth sending
+# again after a pause.
+RETRY_STATUS_CODES = {429, 500, 502, 503, 504}
+MAX_ATTEMPTS = 4
+
+
+def get_timeout() -> tuple[float, float]:
+    """Get the connect and read timeouts to use for requests."""
+    read = os.getenv("CALKIT_INVENIO_TIMEOUT")
+    return CONNECT_TIMEOUT, float(read) if read else READ_TIMEOUT
+
+
 def _request(
     kind: Literal["get", "post", "put", "patch", "delete"],
     path: str,
@@ -73,15 +92,31 @@ def _request(
         params = {}
     if auth and "access_token" not in params:
         params = params | {"access_token": get_token(service=service)}
+    kwargs.setdefault("timeout", get_timeout())
     func = getattr(requests, kind)
-    resp = func(
-        get_base_url(service=service) + path,
-        params=params,
-        json=json,
-        data=data,
-        headers=headers,
-        **kwargs,
-    )
+    # A POST that timed out may still have been carried out on the far end,
+    # e.g., leaving a draft record behind, so only repeat requests that are
+    # safe to send twice
+    max_attempts = 1 if kind == "post" else MAX_ATTEMPTS
+    for attempt in range(1, max_attempts + 1):
+        try:
+            resp = func(
+                get_base_url(service=service) + path,
+                params=params,
+                json=json,
+                data=data,
+                headers=headers,
+                **kwargs,
+            )
+        except (requests.ConnectionError, requests.Timeout):
+            if attempt == max_attempts:
+                raise
+            time.sleep(2**attempt)
+            continue
+        if resp.status_code in RETRY_STATUS_CODES and attempt < max_attempts:
+            time.sleep(2**attempt)
+            continue
+        break
     if resp.status_code >= 400:
         msg = f"{resp.status_code}: "
         try:
@@ -92,6 +127,14 @@ def _request(
                 msg += f"\nErrors:\n{resp_json['errors']}"
         except ValueError:
             msg += resp.text
+        if kind == "post" and resp.status_code in RETRY_STATUS_CODES:
+            # The far end may have done the work anyway, and a blind retry
+            # would duplicate it, so say so rather than papering over it
+            msg += (
+                f"\nThis request was not retried automatically, since "
+                f"{service} may have carried it out despite the error. "
+                "Check for a leftover draft record before trying again."
+            )
         raise HTTPError(msg)
     resp.raise_for_status()
     if as_json:

calkit/models/core.py

@@ -1554,6 +1554,9 @@ class Release(BaseModel):
     # ".calkit/releases/v0/my-project-slides-v0.pdf". Only set for internal
     # releases, which store the artifact in the repo rather than ignoring it.
     stored_path: str | None = None
+    # Whether the release bundles the pipeline and inputs needed to
+    # rebuild its path, rather than just the artifact itself.
+    includes_pipeline: bool = False
 
 
 class StaticHtmlApp(BaseModel):

calkit/pipeline.py

@@ -2328,3 +2328,24 @@ def translate_run_targets(
             else:
                 parent_targets.append(f"{sp}/dvc.yaml")
     return parent_targets, isolated_sp_targets
+
+
+def get_upstream_stages(target: str, wdir: str | None = None) -> set[str]:
+    """Get a stage's name plus the names of all stages it depends on.
+
+    DVC already knows how to walk its own graph, including foreach/matrix
+    groups and stages defined in subdirectories, so we let it collect the
+    target with its dependencies rather than reimplementing the traversal.
+    Names of generated foreach stages come back as ``stage@item``, and both
+    ``calkit.yaml`` and ``dvc.yaml`` key off the base name, so we keep both.
+    """
+    import calkit.dvc
+
+    repo = calkit.dvc.get_dvc_repo(wdir)
+    names = set()
+    for stage in repo.stage.collect(target, with_deps=True):
+        name = getattr(stage, "name", None)
+        if name:
+            names.add(name)
+            names.add(name.split("@")[0])
+    return names