calkit-python 0.41.7__tar.gz → 0.41.9__tar.gz

{calkit_python-0.41.7 → calkit_python-0.41.9}/.gitignore

@@ -21,3 +21,4 @@ dist
 vscode-ext/out
 .claude
 TODO.md
+**/__pycache__/

{calkit_python-0.41.7 → calkit_python-0.41.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.41.7
+Version: 0.41.9
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.41.7 → calkit_python-0.41.9}/calkit/cli/main/core.py

@@ -2418,7 +2418,16 @@ def run_in_env(
             )
         # TODO: Prefix should only be in the env file or calkit.yaml, not both?
         prefix = env.get("prefix")
-        conda_cmd = ["conda", "run"]
+        # Conda is often not on the PATH (especially on Windows), so search
+        # typical install locations rather than relying on the bare name,
+        # which would fail with a confusing FileNotFoundError traceback.
+        conda_exe = calkit.conda.find_conda_exe()
+        if conda_exe is None:
+            raise_error(
+                "Cannot find Conda executable; "
+                "ensure Conda is installed and on the PATH"
+            )
+        conda_cmd = [conda_exe, "run"]
         if prefix is not None:
             conda_cmd += ["--prefix", os.path.abspath(prefix)]
         else:

{calkit_python-0.41.7 → calkit_python-0.41.9}/calkit/cli/new.py

@@ -7,6 +7,7 @@ import os
 import pathlib
 import shutil
 import subprocess
+import time
 from enum import Enum
 
 import typer
@@ -3142,7 +3143,7 @@ def new_release(
             )
             with open("LICENSE", "w") as f:
                 f.write(license_txt)
-                repo.git.add("LICENSE")
+            repo.git.add("LICENSE")
             license_ids = ["mit", "cc-by-4.0"]
         else:
             typer.echo(f"Detecting license(s) from {license_file}")
@@ -3325,22 +3326,23 @@ def new_release(
                 f"/records/{record_id}/draft/files/{filename}/commit",
                 service=to,  # type: ignore
             )
-        # Conditionally publish or reserve DOI based on --draft flag
-        if draft_only:
-            # Reserve a DOI for the draft record
-            typer.echo(f"Reserving DOI for {to} draft record ID {record_id}")
-            doi_resp = calkit.invenio.post(
-                f"/records/{record_id}/draft/pids/doi",
-                service=to,  # type: ignore
+        # Reserve a DOI on the draft before publishing. The publish action's
+        # response only echoes back the DOI under "pids" if one was reserved
+        # on the draft first (otherwise "pids" comes back empty), so always
+        # reserve here to get a stable identifier.
+        typer.echo(f"Reserving DOI for {to} draft record ID {record_id}")
+        doi_resp = calkit.invenio.post(
+            f"/records/{record_id}/draft/pids/doi",
+            service=to,  # type: ignore
+        )
+        if verbose:
+            typer.echo(f"DOI reservation response:\n{doi_resp}")
+        doi = calkit.invenio.extract_doi(doi_resp)
+        if doi is None:
+            raise_error(
+                f"Failed to reserve DOI for {to} draft record {record_id}"
             )
-            if verbose:
-                typer.echo(f"DOI response for draft:\n{doi_resp}")
-            try:
-                doi = doi_resp["pids"]["doi"]["identifier"]
-            except KeyError:
-                doi = doi_resp["doi"]
-            except Exception as e:
-                raise_error(f"Failed to reserve DOI for draft: {e}")
+        if draft_only:
             url = f"https://doi.org/{doi}"
             typer.echo(f"Created {to} draft with reserved DOI: {doi}")
         else:
@@ -3350,8 +3352,37 @@ def new_release(
                 f"/records/{record_id}/draft/actions/publish",
                 service=to,  # type: ignore
             )
+            if verbose:
+                typer.echo(f"Publish response:\n{invenio_dep}")
             record_id = invenio_dep["id"]
-            doi = invenio_dep["pids"]["doi"]["identifier"]
+            # Prefer the DOI from the publish response, but fall back to the
+            # reserved DOI (and a fetch of the published record) since the
+            # publish action returns 202 and may not echo "pids" immediately.
+            # Polling failures are tolerated: we already have the reserved DOI,
+            # so a transient error while the record settles should not abort
+            # the release.
+            published_doi = calkit.invenio.extract_doi(invenio_dep)
+            for _ in range(10):
+                if published_doi is not None:
+                    break
+                time.sleep(1)
+                try:
+                    record = calkit.invenio.get(
+                        f"/records/{record_id}",
+                        service=to,  # type: ignore
+                    )
+                except Exception as e:
+                    if verbose:
+                        typer.echo(f"Polling for published DOI failed: {e}")
+                    continue
+                published_doi = calkit.invenio.extract_doi(record)
+            if published_doi is not None:
+                doi = published_doi
+            else:
+                typer.echo(
+                    "Could not confirm minted DOI from published record; "
+                    f"falling back to reserved DOI {doi}"
+                )
             url = f"https://doi.org/{doi}"
             typer.echo(f"Published to {to} with DOI: {doi}")
     else:
@@ -3363,38 +3394,23 @@ def new_release(
         doi_base_url = calkit.releases.SERVICES[to]["url"]
         doi_md = (
             f"[![DOI]({doi_base_url}/badge/DOI/{doi}.svg)]"
-            f"(https://handle.stage.datacite.org/{doi})"
+            f"(https://doi.org/{doi})"
         )
         if os.path.isfile("README.md"):
             with open("README.md") as f:
                 readme_txt = f.read()
         else:
-            readme_txt = f"# {title}\n"
-        existing_lines = readme_txt.split("\n")
-        new_lines = []
-        first_content_line_index = None
-        for n, line in enumerate(existing_lines):
-            if line.startswith(doi_md[:6]):
-                pass  # Skip DOI lines
-            else:
-                if (
-                    n != 0
-                    and line.strip()
-                    and first_content_line_index is None
-                ):
-                    first_content_line_index = len(new_lines)
-                new_lines.append(line)
-        # Ensure first 3 lines are title, blank, DOI lines
-        new_lines = (
-            [new_lines[0]]
-            + ["", doi_md, ""]
-            + new_lines[first_content_line_index:]
+            readme_txt = ""
+        readme_txt = calkit.releases.add_doi_badge_to_readme(
+            readme_txt, badge=doi_md, title=title
         )
-        readme_txt = "\n".join(new_lines)
         if not dry_run:
             with open("README.md", "w") as f:
                 f.write(readme_txt)
-                repo.git.add("README.md")
+            # Stage only after the file is closed; otherwise the buffered
+            # contents may not be flushed to disk yet and Git would stage an
+            # empty file
+            repo.git.add("README.md")
         else:
             typer.echo(f"Would have updated README.md to:\n{readme_txt}")
     # Create Git tag
@@ -3449,11 +3465,13 @@ def new_release(
         references = reference_collections[0]
     ref_path = references.get("path", "references.bib")
     try:
+        # Read the existing references as raw text so we can append the new
+        # entry without reformatting the user's existing entries
         if os.path.isfile(ref_path):
             with open(ref_path) as f:
-                reflib = bibtexparser.load(f)
+                existing_text = f.read()
         else:
-            reflib = bibtexparser.bibdatabase.BibDatabase()
+            existing_text = ""
         bibtex_doi = doi
         if bibtex_doi is None and dry_run:
             mock_suffix = "".join(
@@ -3473,31 +3491,38 @@ def new_release(
         if not new_entries:
             raise ValueError("Failed to parse generated BibTeX entry")
         new_entry = new_entries[0]
-        # Search through entries for one with the same DOI, and replace if
-        # there is a match
+        # Search through existing entries for any with the same DOI, and
+        # replace them (by citation key) if there is a match. Tolerate parse
+        # errors here (e.g., non-standard or temporarily-invalid BibTeX) by
+        # skipping replacement detection and still appending the new entry.
         new_doi = new_entry.get("doi")
-        matching_indices = []
+        replace_ids = []
         if new_doi:
-            matching_indices = [
-                n
-                for n, entry in enumerate(reflib.entries)
+            try:
+                existing_entries = bibtexparser.loads(existing_text).entries
+            except Exception as e:
+                warn(f"Could not parse existing references to dedupe: {e}")
+                existing_entries = []
+            replace_ids = [
+                entry["ID"]
+                for entry in existing_entries
                 if entry.get("doi") == new_doi
             ]
-        if matching_indices:
+        if replace_ids:
             typer.echo(
                 "Found matching DOI in existing references "
-                f"({len(matching_indices)} entr"
-                f"{'y' if len(matching_indices) == 1 else 'ies'}); "
+                f"({len(replace_ids)} entr"
+                f"{'y' if len(replace_ids) == 1 else 'ies'}); "
                 "replacing"
             )
-            for n in reversed(matching_indices):
-                _ = reflib.entries.pop(n)
-        reflib.entries.append(new_entry)
+        new_text = calkit.releases.add_bibtex_entry(
+            existing_text, invenio_bibtex, replace_ids=replace_ids
+        )
         if dry_run:
             typer.echo(f"Would write updated references to {ref_path}")
         else:
             with open(ref_path, "w") as f:
-                bibtexparser.dump(reflib, f)
+                f.write(new_text)
             repo.git.add(ref_path)
     except Exception as e:
         warn(f"Failed to add to references: {e}")

{calkit_python-0.41.7 → calkit_python-0.41.9}/calkit/core.py

@@ -341,6 +341,14 @@ def check_dep_exists(
         return True
     if system_info is not None and system_info.get(f"{name}_version"):
         return True
+    # Conda and mamba are frequently installed but not on the PATH (most
+    # commonly on Windows), so search their typical install locations
+    # rather than relying on the bare name being directly executable.
+    if name in ("conda", "mamba"):
+        from calkit.conda import find_conda_exe, find_mamba_exe
+
+        exe = find_conda_exe() if name == "conda" else find_mamba_exe()
+        return exe is not None
     cmd = [name]
     # Executables with non-conventional CLIs
     if name == "matlab":

{calkit_python-0.41.7 → calkit_python-0.41.9}/calkit/invenio.py

@@ -107,6 +107,26 @@ put = partial(_request, "put")
 delete = partial(_request, "delete")
 
 
+def extract_doi(record: dict) -> str | None:
+    """Extract the DOI identifier from an InvenioRDM record or response.
+
+    Depending on the endpoint and whether the DOI has been minted yet, the
+    identifier can live in different places, so check all known locations and
+    return ``None`` if it isn't present yet.
+    """
+    pids = record.get("pids")
+    if isinstance(pids, dict):
+        doi = pids.get("doi")
+        if isinstance(doi, dict) and doi.get("identifier"):
+            return doi["identifier"]
+    if record.get("doi"):
+        return record["doi"]
+    metadata = record.get("metadata")
+    if isinstance(metadata, dict) and metadata.get("doi"):
+        return metadata["doi"]
+    return None
+
+
 def get_download_urls(
     record_id: int | str,
     service: ServiceName = DEFAULT_SERVICE,

{calkit_python-0.41.7 → calkit_python-0.41.9}/calkit/pipeline.py

@@ -96,10 +96,14 @@ class PipelineStatus(BaseModel):
     def always_run_stage_names(self) -> list[str]:
         # Only list stages whose sole change indicator is always_run; stages
         # that also have real changes are reported under stale_stage_names.
+        # Subproject stages are excluded: their always-changed status is a
+        # delegation detail, not a user-meaningful always-run stage of the
+        # parent project.
         return [
             name
             for name, stage in self.stale_stages.items()
             if stage.always_run
+            and not stage.is_subproject
             and not stage.modified_command
             and not stage.modified_inputs
             and not stage.modified_outputs
@@ -145,6 +149,12 @@ class StaleStage(BaseModel):
     modified_outputs: list[str] = Field(default_factory=list)
     modified_command: bool = False
     always_run: bool = False
+    # True for stages that originate from a subproject (either an individual
+    # "{sp}:{stage}" stage or a kept "{sp} (subproject)" wrapper). Subproject
+    # wrapper stages are marked always-changed purely as a delegation
+    # mechanism, so they should not be advertised as always-run stages of the
+    # parent project.
+    is_subproject: bool = False
 
     @staticmethod
     def _as_path_list(paths: object) -> list[str]:
@@ -227,6 +237,7 @@ class StaleStage(BaseModel):
         status_data: list | dict | str,
         configured_outputs: list[str] | None = None,
         path_prefix: str | None = None,
+        is_subproject: bool = False,
     ) -> "StaleStage":
         modified_inputs = []
         output_paths = []
@@ -312,6 +323,7 @@ class StaleStage(BaseModel):
             modified_outputs=modified_outputs,
             modified_command=modified_command,
             always_run=always_run,
+            is_subproject=is_subproject,
         )
 
 
@@ -546,6 +558,74 @@ def collapse_dvc_stages(
     return stage
 
 
+def _status_target_matches(
+    target: str,
+    display_name: str,
+    bare_name: str,
+    subproject: str | None,
+    stale_stage: StaleStage,
+) -> bool:
+    """Return True if a status ``target`` selects this stage.
+
+    Because the full DVC status is now computed and filtered locally (rather
+    than asking DVC to filter by target), this is where each ``target`` passed
+    to :func:`get_status` is resolved against a single stage. A ``target`` may
+    be a stage name in any of calkit's forms (``stage``, ``dvc.yaml:stage``,
+    ``subproject:stage``, or a whole ``subproject``) or a repo path, which is
+    matched against the stage's stale/modified inputs and outputs.
+
+    Iterated/matrix stages use a ``name@param`` key in DVC status, so the base
+    name (the part before ``@``) is matched too — targeting the base name
+    selects every iteration, consistent with how ``calkit run`` expands matrix
+    targets.
+    """
+    base_name = bare_name.split("@")[0]
+    # Stage-name matching is done against the raw target so it covers all of
+    # calkit's stage-target forms as well as DVC's native ``dvc.yaml:stage``
+    # and ``<subproject>/dvc.yaml:stage`` forms (the latter is what
+    # translate_run_targets emits for inline subprojects).
+    stage_aliases = {display_name, bare_name, base_name}
+    if subproject is not None:
+        subproject_name = Path(subproject).name
+        if target in {subproject, subproject_name}:
+            return display_name.startswith(
+                f"{subproject}:"
+            ) or display_name == (f"{subproject} (subproject)")
+        stage_aliases |= {
+            f"{subproject}:{bare_name}",
+            f"{subproject_name}:{bare_name}",
+            f"{subproject}:{base_name}",
+            f"{subproject_name}:{base_name}",
+            f"{subproject}/dvc.yaml:{bare_name}",
+            f"{subproject}/dvc.yaml:{base_name}",
+        }
+    else:
+        stage_aliases |= {
+            f"dvc.yaml:{bare_name}",
+            f"dvc.yaml:{base_name}",
+        }
+    if target in stage_aliases:
+        return True
+    # Otherwise treat the target as a repo path and match it against the
+    # stage's stale/modified inputs and outputs. A target is path-like if it
+    # exists on disk or contains a separator; this keeps root-level paths
+    # without separators (e.g. ``data`` or ``out.csv``) selectable, which a
+    # separator-only heuristic would miss. A target that looks like a stage
+    # selector (contains ``:``) but matched no stage above is not a path.
+    if ":" in target and not os.path.exists(target):
+        return False
+    is_pathlike = os.path.exists(target) or "/" in target or "\\" in target
+    if not is_pathlike:
+        return False
+    norm_target = Path(target).as_posix().rstrip("/")
+    candidate_paths = (
+        stale_stage.stale_outputs
+        + stale_stage.modified_inputs
+        + stale_stage.modified_outputs
+    )
+    return any(_paths_overlap(norm_target, p) for p in candidate_paths)
+
+
 def get_status(
     ck_info: dict | None = None,
     targets: list[str] | None = None,
@@ -628,7 +708,11 @@ def get_status(
             dvc_repo = calkit.dvc.get_dvc_repo()
             # calkit.dvc.core installs a filter on dvc.repo.status that drops
             # the noisy frozen-stage warning, so the call here stays quiet.
-            raw_status = dvc_repo.status(targets=targets)
+            # Ask DVC for the complete stale set, then filter locally.
+            # DVC's own target filtering can miss stale propagation from
+            # isolated subprojects, which makes targeted status disagree with
+            # the full-project status view.
+            raw_status = dvc_repo.status()
             raw_status = calkit.dvc.status_as_posix(raw_status)
         except Exception as e:
             result["errors"].append(
@@ -794,20 +878,47 @@ def get_status(
                 for output in stage_cfg.get("outputs", [])
             ]
             if subproject:
+                # Use posix separators so these match DVC's reported paths
+                # (which are always forward-slash, repo-root-relative). Using
+                # the OS-native separator here would, on Windows, produce a
+                # backslash variant that escapes path dedup and shows up as a
+                # duplicate stale output alongside DVC's posix path.
                 configured_outputs = [
-                    str(Path(subproject) / p) for p in raw_outputs
+                    (Path(subproject) / p).as_posix() for p in raw_outputs
                 ]
             else:
                 configured_outputs = raw_outputs
             # For isolated subprojects, DVC reports paths relative to the
             # subproject dir; prefix them so all paths are parent-relative.
+            # A stage is subproject-originating if it carries a subproject
+            # path ("{sp}:{stage}") or is a kept wrapper for a subproject
+            # ("{sp} (subproject)", whose bare name is a known subproject).
+            is_subproject = (
+                subproject is not None or bare_name in sp_by_stage_name
+            )
             ordered_stale_stages[display_name] = StaleStage.from_status_data(
                 status_data=status_data,
                 configured_outputs=configured_outputs,
                 path_prefix=subproject
                 if subproject in isolated_sp_paths
                 else None,
+                is_subproject=is_subproject,
             )
+        if targets:
+            ordered_stale_stages = {
+                display_name: stale_stage
+                for display_name, stale_stage in ordered_stale_stages.items()
+                if any(
+                    _status_target_matches(
+                        target,
+                        display_name,
+                        raw_stale_stages[display_name][0],
+                        raw_stale_stages[display_name][1],
+                        stale_stage,
+                    )
+                    for target in targets
+                )
+            }
         result["stale_stages"] = ordered_stale_stages
         return PipelineStatus(
             has_pipeline=result["has_pipeline"],

{calkit_python-0.41.7 → calkit_python-0.41.9}/calkit/releases.py

@@ -86,6 +86,109 @@ def create_bibtex(
     )
 
 
+def _find_bibtex_entry_span(
+    text: str, entry_id: str
+) -> tuple[int, int] | None:
+    """Locate the raw character span of a BibTeX entry by its citation key.
+
+    Returns ``(start, end)`` offsets into ``text`` where ``end`` is exclusive
+    (just past the entry's closing brace), or ``None`` if no entry with that
+    key is found.
+    """
+    # Match e.g. ``@misc{the-key,`` allowing arbitrary surrounding whitespace
+    pattern = re.compile(
+        r"@\w+\s*\{\s*" + re.escape(entry_id) + r"\s*,", re.IGNORECASE
+    )
+    match = pattern.search(text)
+    if match is None:
+        return None
+    # Walk from the entry's opening brace to its matching close, counting depth
+    depth = 0
+    i = text.index("{", match.start())
+    while i < len(text):
+        if text[i] == "{":
+            depth += 1
+        elif text[i] == "}":
+            depth -= 1
+            if depth == 0:
+                return match.start(), i + 1
+        i += 1
+    return None
+
+
+def add_bibtex_entry(
+    existing_text: str, new_entry: str, replace_ids: list[str] | None = None
+) -> str:
+    """Append a BibTeX entry to existing references without reformatting them.
+
+    Any entries whose citation keys are in ``replace_ids`` are removed first
+    (so a re-released version replaces its prior entry); all other existing
+    content is preserved byte-for-byte. ``new_entry`` is appended at the end.
+    """
+    text = existing_text
+    # Remove replaced entries from the raw text, last-to-first so earlier spans
+    # stay valid as we splice. De-duplicate spans in case a key matched more
+    # than once.
+    spans = set()
+    for entry_id in replace_ids or []:
+        span = _find_bibtex_entry_span(text, entry_id)
+        if span is not None:
+            spans.add(span)
+    for start, end in sorted(spans, reverse=True):
+        # Consume any trailing newlines so we don't leave a gap behind
+        while end < len(text) and text[end] in "\r\n":
+            end += 1
+        text = text[:start] + text[end:]
+    # Append the new entry, leaving the existing content untouched and adding
+    # only the separator newlines needed for exactly one blank line between the
+    # last existing entry and the new one
+    if not text.strip():
+        return new_entry.strip() + "\n"
+    if text.endswith("\n\n"):
+        separator = ""
+    elif text.endswith("\n"):
+        separator = "\n"
+    else:
+        separator = "\n\n"
+    return text + separator + new_entry.strip() + "\n"
+
+
+def add_doi_badge_to_readme(
+    readme_text: str, badge: str, title: str | None
+) -> str:
+    """Insert (or refresh) a DOI badge near the top of a README.
+
+    The badge is placed directly beneath the title with exactly one blank line
+    on either side. Any pre-existing DOI badge line is replaced rather than
+    duplicated, the title is never duplicated, and all other existing content
+    is preserved.
+    """
+    # Drop any existing DOI badge line(s); the current badge is re-added below
+    lines = [
+        line
+        for line in (readme_text.split("\n") if readme_text else [])
+        if not line.startswith("[![DOI")
+    ]
+    # The first non-blank line is treated as the title; synthesize one from the
+    # project title if the README has no content yet
+    while lines and not lines[0].strip():
+        lines.pop(0)
+    if lines:
+        title_line = lines[0]
+        rest = lines[1:]
+    else:
+        title_line = f"# {title}"
+        rest = []
+    # Drop leading blank lines from the remaining content so the spacing around
+    # the badge is controlled entirely here
+    while rest and not rest[0].strip():
+        rest.pop(0)
+    new_lines = [title_line, "", badge]
+    if rest:
+        new_lines += [""] + rest
+    return "\n".join(new_lines)
+
+
 CITATION_CFF_TEMPLATE = """
 cff-version: 1.2.0
 message: If you use this software, please cite it using these metadata.

{calkit_python-0.41.7 → calkit_python-0.41.9}/calkit/tests/cli/main/test_subprojects.py

@@ -296,3 +296,56 @@ def test_isolated_subproject_external_dep(tmp_dir):
         check_environments=False, compile_to_dvc=False
     )
     assert not status.is_stale
+
+
+def test_targeted_status_uses_full_subproject_status_for_stage_targets(
+    tmp_dir, monkeypatch
+):
+    os.makedirs("solver/.dvc", exist_ok=True)
+    root_ck = {
+        "subprojects": [{"path": "solver"}],
+        "pipeline": {
+            "stages": {
+                "post-process": {
+                    "outputs": [{"path": "final.txt", "storage": "git"}]
+                }
+            }
+        },
+    }
+    solver_ck = {
+        "pipeline": {
+            "stages": {
+                "package-paper": {
+                    "outputs": [{"path": "paper.txt", "storage": "git"}]
+                }
+            }
+        }
+    }
+
+    class FakeRepo:
+        def status(self, targets=None):
+            if targets is None:
+                return {
+                    "solver/dvc.yaml:package-paper": ["always changed"],
+                    "post-process": [
+                        {"changed deps": {"solver/docs/figs": "modified"}}
+                    ],
+                }
+            return {}
+
+    monkeypatch.setattr(
+        calkit,
+        "load_calkit_info",
+        lambda wdir=None: solver_ck if wdir == "solver" else root_ck,
+    )
+    monkeypatch.setattr(calkit.dvc, "get_dvc_repo", lambda *args: FakeRepo())
+    monkeypatch.setattr(calkit.dvc, "status_as_posix", lambda status: status)
+    status = calkit.pipeline.get_status(
+        ck_info=root_ck,
+        targets=["post-process"],
+        check_environments=False,
+        clean_notebooks=False,
+        compile_to_dvc=False,
+    )
+    assert status.is_stale
+    assert status.stale_stage_names == ["post-process"]