calkit-python 0.35.5__tar.gz → 0.35.6__tar.gz

{calkit_python-0.35.5 → calkit_python-0.35.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.35.5
+Version: 0.35.6
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

calkit_python-0.35.6/calkit/git.py

@@ -0,0 +1,297 @@
+"""Git-related functionality."""
+
+from __future__ import annotations
+
+import os
+from os import PathLike
+from pathlib import Path
+
+import git
+
+
+def get_staged_files(
+    path: str | None = None, repo: git.Repo | None = None
+) -> list[str]:
+    """Get a list of staged files for the repo at ``path`` or the provided
+    repo.
+    """
+    if repo is None:
+        repo = git.Repo(path)
+    cmd = ["--staged", "--name-only"]
+    if path is not None:
+        cmd.append(path)
+    diff = repo.git.diff(cmd)
+    paths = diff.split("\n")
+    return [p for p in paths if p]
+
+
+def get_changed_files(
+    path: str | None = None, repo: git.Repo | None = None
+) -> list[str]:
+    """Get a list of files that have been changed but not staged."""
+    if repo is None:
+        repo = git.Repo(path)
+    return [
+        item.a_path
+        for item in repo.index.diff(None)
+        if item.a_path is not None
+    ]
+
+
+def get_untracked_files(
+    path: str | None = None, repo: git.Repo | None = None
+) -> list[str]:
+    """Get a list of untracked files."""
+    if repo is None:
+        repo = git.Repo(path)
+    return repo.untracked_files
+
+
+def get_staged_files_with_status(
+    path: str | None = None, repo: git.Repo | None = None
+) -> list[dict]:
+    if repo is None:
+        repo = git.Repo(path)
+    cmd = ["--staged", "--name-status"]
+    if path is not None:
+        cmd.append(path)
+    diff = repo.git.diff(cmd)
+    paths = diff.split("\n")
+    res = []
+    for pathi in paths:
+        # Make sure line is not empty, e.g., a trailing newline
+        if pathi:
+            status, p = pathi.split("\t")
+            res.append({"status": status, "path": p})
+    return res
+
+
+def ls_files(repo: git.Repo, *args, **kwargs) -> list[str]:
+    """Get a list of all files tracked by git."""
+    output = repo.git.ls_files(*args, **kwargs)
+    return [f for f in output.split("\n") if f]
+
+
+def _resolve_repo_and_ignore_path(
+    repo: git.Repo, path: str | PathLike
+) -> tuple[git.Repo, str]:
+    """Resolve which repo should own ignore rules for ``path``."""
+    # Normalize target path to absolute from the current repo root.
+    repo_root = Path(repo.working_dir).resolve()
+    path_obj = Path(path)
+    if path_obj.is_absolute():
+        abs_path = path_obj.resolve()
+    else:
+        abs_path = (repo_root / path_obj).resolve()
+    # If the path is inside a submodule, use that repo and relative path.
+    for submodule in repo.submodules:
+        submodule_root = (repo_root / submodule.path).resolve()
+        if abs_path == submodule_root:
+            continue
+        if abs_path.is_relative_to(submodule_root):
+            sub_repo = submodule.module()
+            rel_path = abs_path.relative_to(submodule_root).as_posix()
+            return sub_repo, rel_path
+    # Fall back to a repo-relative path when possible.
+    try:
+        rel_path = abs_path.relative_to(repo_root).as_posix()
+    except ValueError:
+        rel_path = path_obj.as_posix()
+    return repo, rel_path
+
+
+def _get_matching_gitignore_details(
+    repo: git.Repo, path: str
+) -> tuple[Path | None, str | None]:
+    """Return the repo-local gitignore file and pattern matching ``path``."""
+    try:
+        check_ignore = repo.git.check_ignore("-v", "--", path)
+    except git.GitCommandError:
+        return None, None
+    line = check_ignore.splitlines()[0]
+    try:
+        source_info, _ = line.split("\t", 1)
+        source_path, _, pattern = source_info.rsplit(":", 2)
+    except ValueError:
+        return None, None
+    if not source_path.endswith(".gitignore"):
+        return None, pattern
+    gitignore_path = (Path(repo.working_dir) / source_path).resolve()
+    try:
+        gitignore_path.relative_to(Path(repo.working_dir).resolve())
+    except ValueError:
+        return None, pattern
+    return gitignore_path, pattern
+
+
+def ensure_path_is_ignored(
+    repo: git.Repo, path: str | PathLike
+) -> None | bool:
+    """Ensure that the given path is ignored by Git.
+
+    Returns True if ``.gitignore`` was modified.
+    """
+    # Resolve whether the ignore rule belongs to this repo or a submodule.
+    target_repo, target_path = _resolve_repo_and_ignore_path(repo, path)
+    # No-op if Git already ignores this path.
+    if target_repo.ignored(target_path):
+        return
+    # Read gitignore first to check if the path is already ignored
+    # If not, we don't want to add a line for it since it was added
+    # TODO: Add an option to remove cached (`git rm --cached`)
+    gitignore_path = os.path.join(target_repo.working_dir, ".gitignore")
+    if os.path.isfile(gitignore_path):
+        with open(gitignore_path) as f:
+            gitignore_txt = f.read()
+        lines = gitignore_txt.splitlines()
+        if target_path in lines:
+            return
+    with open(gitignore_path, "a") as f:
+        if (
+            os.path.isfile(gitignore_path)
+            and os.path.getsize(gitignore_path) > 0
+        ):
+            f.write("\n")
+        f.write(f"{target_path}\n")
+        return True
+
+
+def ensure_path_is_not_ignored(
+    repo: git.Repo, path: str | PathLike
+) -> None | bool:
+    """Ensure a path is not ignored by Git."""
+    # Resolve whether the unignore rule belongs to this repo or a submodule.
+    target_repo, target_path = _resolve_repo_and_ignore_path(repo, path)
+    # No-op if Git does not ignore this path.
+    if not target_repo.ignored(target_path):
+        return
+    matching_gitignore_path, matched_pattern = _get_matching_gitignore_details(
+        target_repo, target_path
+    )
+    if matching_gitignore_path is not None:
+        gitignore_path = matching_gitignore_path.as_posix()
+        path_for_gitignore = (
+            (Path(target_repo.working_dir) / target_path)
+            .resolve()
+            .relative_to(matching_gitignore_path.parent.resolve())
+            .as_posix()
+        )
+    else:
+        gitignore_path = os.path.join(target_repo.working_dir, ".gitignore")
+        path_for_gitignore = target_path
+    if not os.path.isfile(gitignore_path):
+        with open(gitignore_path, "w") as f:
+            f.write(f"!{path_for_gitignore}\n")
+        return True
+    with open(gitignore_path) as f:
+        gitignore_txt = f.read()
+    lines = gitignore_txt.splitlines()
+    direct_rule_variants = [path_for_gitignore, f"/{path_for_gitignore}"]
+    if matched_pattern is not None and matched_pattern.startswith("/"):
+        no_ignore_line = f"!/{path_for_gitignore}"
+    else:
+        no_ignore_line = f"!{path_for_gitignore}"
+    path_parts = Path(path_for_gitignore).parts
+
+    def ancestor_requires_recursive_unignore() -> bool:
+        """Return True if any ancestor-level ignore rule would block this path.
+
+        This includes explicit directory ignores (e.g. 'dir/' or '/dir/')
+        as well as ancestor-based glob patterns like 'dir/*' or '/dir/*',
+        i.e., any rule that would prevent reaching the nested path without
+        adding recursive unignore patterns.
+        """
+        for i in range(1, len(path_parts)):
+            ancestor = "/".join(path_parts[:i])
+            if (
+                ancestor in lines
+                or f"/{ancestor}" in lines
+                or f"{ancestor}/" in lines
+                or f"/{ancestor}/" in lines
+                or f"{ancestor}/*" in lines
+                or f"/{ancestor}/*" in lines
+            ):
+                return True
+        return False
+
+    if len(path_parts) == 1:
+        # Simple (non-nested) path: remove the direct ignore rule, or add a
+        # negation if the ignore comes from a glob or other pattern
+        direct_rule = next(
+            (rule for rule in direct_rule_variants if rule in lines), None
+        )
+        if direct_rule is not None:
+            lines.remove(direct_rule)
+        else:
+            # Remove any stale negation and re-append at the end so it takes
+            # precedence over any later re-ignore rule
+            if no_ignore_line in lines:
+                lines.remove(no_ignore_line)
+            lines.append(no_ignore_line)
+    else:
+        # Nested path: only apply recursive un-ignore rules when an ancestor
+        # directory is explicitly ignored
+        # Otherwise, remove a direct ignore
+        # rule for this path or add a simple negation if needed
+        removed_direct_rule = False
+        direct_rule = next(
+            (rule for rule in direct_rule_variants if rule in lines), None
+        )
+        if direct_rule is not None:
+            lines.remove(direct_rule)
+            removed_direct_rule = True
+        if ancestor_requires_recursive_unignore():
+            # Git will not traverse into a directory excluded by a "dir/"
+            # pattern, so a bare "!dir/sub/file" negation has no effect.
+            # We need to:
+            #   1. Convert any "ancestor/" (or "ancestor") exclude to
+            #      "ancestor/*" so git traverses the directory while still
+            #      ignoring direct children by default.
+            #   2. Add "!ancestor/" rules for intermediate directories.
+            #   3. Add "ancestor/*" re-ignore rules for each intermediate dir.
+            #   4. Add "!target_path" for the specific file.
+            for i in range(1, len(path_parts)):
+                ancestor = "/".join(path_parts[:i])
+                reignore_glob = f"{ancestor}/*"
+                if f"{ancestor}/" in lines:
+                    idx = lines.index(f"{ancestor}/")
+                    lines[idx] = reignore_glob
+                elif f"/{ancestor}/" in lines:
+                    idx = lines.index(f"/{ancestor}/")
+                    lines[idx] = f"/{ancestor}/*"
+                elif ancestor in lines:
+                    idx = lines.index(ancestor)
+                    lines[idx] = reignore_glob
+                elif f"/{ancestor}" in lines:
+                    idx = lines.index(f"/{ancestor}")
+                    lines[idx] = f"/{ancestor}/*"
+                no_ignore_dir = f"!{ancestor}/"
+                anchored_no_ignore_dir = f"!/{ancestor}/"
+                # The first ancestor does not need an explicit un-ignore once
+                # converted to "ancestor/*". Deeper ancestors do.
+                if i > 1:
+                    # Remove stale entry and re-append so it takes precedence
+                    if no_ignore_dir in lines:
+                        lines.remove(no_ignore_dir)
+                    elif anchored_no_ignore_dir in lines:
+                        lines.remove(anchored_no_ignore_dir)
+                    lines.append(no_ignore_dir)
+                if (
+                    reignore_glob not in lines
+                    and f"/{ancestor}/*" not in lines
+                ):
+                    lines.append(reignore_glob)
+            # Remove stale negation and re-append at the end so it takes
+            # precedence over any later re-ignore rule
+            if no_ignore_line in lines:
+                lines.remove(no_ignore_line)
+            lines.append(no_ignore_line)
+        elif not removed_direct_rule:
+            # The path may be ignored by a non-directory pattern (e.g., glob);
+            # remove stale negation and append at end so it takes precedence
+            if no_ignore_line in lines:
+                lines.remove(no_ignore_line)
+            lines.append(no_ignore_line)
+    with open(gitignore_path, "w") as f:
+        f.write(os.linesep.join(lines))
+    return True

{calkit_python-0.35.5 → calkit_python-0.35.6}/calkit/pipeline.py

@@ -241,25 +241,15 @@ def to_dvc(
                 outputs += stage.notebook_outputs
             elif stage.kind == "sbatch":
                 outputs.append(stage.log_output)
-            # Build the set of current output paths so we can detect stale
-            # .gitignore entries from the previous version of the stage
-            current_out_paths = set()
-            for out in outputs:
-                if isinstance(out, PathOutput):
-                    current_out_paths.add(out.path)
-                elif isinstance(out, str):
-                    current_out_paths.add(out)
+            # Build the set of current DVC output paths so we can detect stale
+            # .gitignore entries from the previous version of the stage,
+            # including synthesized outputs like LaTeX PDFs
+            current_out_paths = set(calkit.dvc.out_paths_from_stage(dvc_stage))
             # If this stage already existed, un-ignore any outputs that have
             # been renamed or removed so .gitignore does not accumulate stale
             # entries (e.g., after a capitalization change in the path)
             old_stage = existing_dvc_stages.get(stage_name, {})
-            for old_out in old_stage.get("outs", []):
-                if isinstance(old_out, str):
-                    old_path = old_out
-                elif isinstance(old_out, dict):
-                    old_path = list(old_out.keys())[0]
-                else:
-                    continue
+            for old_path in calkit.dvc.out_paths_from_stage(old_stage):
                 if old_path not in current_out_paths:
                     calkit.git.ensure_path_is_not_ignored(repo, path=old_path)
             # Deal with any gitignore changes necessary

{calkit_python-0.35.5 → calkit_python-0.35.6}/calkit/tests/test_git.py

@@ -105,7 +105,7 @@ def test_ensure_path_is_not_ignored_nested(tmp_dir):
 
     When a parent directory is excluded with a trailing slash (e.g.,
     ``results/``), git will not traverse into it, so a simple negation like
-    ``!results/StageName/end.json`` has no effect.  The fix converts the
+    ``!results/StageName/end.json`` has no effect. The fix converts the
     directory exclude to a glob pattern and adds intermediate un-ignore rules.
     """
     repo = git.Repo.init()
@@ -124,15 +124,13 @@ def test_ensure_path_is_not_ignored_nested(tmp_dir):
     assert result is True
     with open(".gitignore") as f:
         lines = f.read().splitlines()
-    # The plain directory exclude should be replaced with a glob
-    assert "results/" not in lines
-    assert "results/*" in lines
-    # Intermediate directory must be un-ignored so git traverses into it
-    assert "!results/StageName/" in lines
-    # The intermediate directory's other contents must be re-ignored
-    assert "results/StageName/*" in lines
-    # The specific file must be explicitly un-ignored
-    assert "!results/StageName/end.json" in lines
+    # Keep the rules minimal while preserving the required behavior
+    assert lines == [
+        "results/*",
+        "!results/StageName/",
+        "results/StageName/*",
+        "!results/StageName/end.json",
+    ]
     # Verify git no longer considers the target file as ignored
     assert not repo.ignored("results/StageName/end.json")
     # Other files in results/ must still be ignored
@@ -148,3 +146,54 @@ def test_ensure_path_is_not_ignored_nested(tmp_dir):
         repo, path="results/StageName/end.json"
     )
     assert result2 is None
+
+
+def test_ensure_path_is_not_ignored_nested_direct_path_rule(tmp_dir):
+    """Unignoring a directly ignored nested path should only remove that
+    rule.
+    """
+    repo = git.Repo.init()
+    target = "pubs/applied-ocean-research-model/references.bib"
+    sibling = "pubs/applied-ocean-research-model/paper.pdf"
+    with open(".gitignore", "w") as f:
+        f.write(f"{target}\n")
+    os.makedirs("pubs/applied-ocean-research-model", exist_ok=True)
+    with open(target, "w") as f:
+        f.write("@article{test}\n")
+    with open(sibling, "w") as f:
+        f.write("pdf\n")
+    # Only the direct target path should be ignored initially
+    assert repo.ignored(target)
+    assert not repo.ignored(sibling)
+    result = calkit.git.ensure_path_is_not_ignored(repo, path=target)
+    assert result is True
+    with open(".gitignore") as f:
+        lines = f.read().splitlines()
+    # Remove only the direct rule, with no recursive ancestor entries
+    assert target not in lines
+    assert f"!{target}" not in lines
+    assert "!pubs/" not in lines
+    assert "pubs/*" not in lines
+    assert "!pubs/applied-ocean-research-model/" not in lines
+    assert "pubs/applied-ocean-research-model/*" not in lines
+    assert not repo.ignored(target)
+    assert not repo.ignored(sibling)
+
+
+def test_ensure_path_is_not_ignored_nested_gitignore_direct_path_rule(tmp_dir):
+    repo = git.Repo.init()
+    os.makedirs("paper", exist_ok=True)
+    target = "paper/main.pdf"
+    with open("paper/.gitignore", "w") as f:
+        f.write("/main.pdf\n")
+    with open(target, "w") as f:
+        f.write("pdf\n")
+    assert repo.ignored(target)
+    result = calkit.git.ensure_path_is_not_ignored(repo, path=target)
+    assert result is True
+    with open("paper/.gitignore") as f:
+        lines = f.read().splitlines()
+    assert "/main.pdf" not in lines
+    assert "!/main.pdf" not in lines
+    assert not os.path.exists(".gitignore")
+    assert not repo.ignored(target)

{calkit_python-0.35.5 → calkit_python-0.35.6}/calkit/tests/test_pipeline.py

@@ -1,5 +1,6 @@
 """Tests for ``calkit.pipeline``."""
 
+import os
 import subprocess
 
 import git
@@ -653,9 +654,12 @@ def test_shell_script_stage_allows_non_composite_slurm_env():
 
 
 def test_gitignore_updated_when_stage_output_renamed(tmp_dir):
-    """When a stage output path is renamed (e.g., from a capitalization change) the
-    old path should be removed from .gitignore and the new path should be
-    added.
+    """When a stage output path is renamed, stale .gitignore entries are
+    replaced.
+
+    Use the .gitignore contents for verification because case-only renames can
+    still appear ignored on case-insensitive filesystems like the default macOS
+    setup.
     """
     subprocess.check_call(["calkit", "init"])
     # Stage 1: initial calkit.yaml with output 'b_sparsity_plot.pdf' stored in DVC
@@ -682,6 +686,9 @@ def test_gitignore_updated_when_stage_output_renamed(tmp_dir):
     # Verify DVC has added the old output path to .gitignore
     repo = git.Repo(".")
     assert repo.ignored("b_sparsity_plot.pdf")
+    with open(".gitignore") as f:
+        lines = f.read().splitlines()
+    assert "/b_sparsity_plot.pdf" in lines
     # Stage 2: rename output (capitalization change) to 'B_sparsity_plot.pdf'
     ck_info["pipeline"]["stages"]["plot"]["command"] = (
         "touch B_sparsity_plot.pdf"
@@ -692,6 +699,52 @@ def test_gitignore_updated_when_stage_output_renamed(tmp_dir):
     with open("calkit.yaml", "w") as f:
         calkit.ryaml.dump(ck_info, f)
     subprocess.check_call(["calkit", "run"])
-    # Old (stale) path should no longer be ignored by git
-    assert not repo.ignored("b_sparsity_plot.pdf")
+    with open(".gitignore") as f:
+        lines = f.read().splitlines()
+    assert "/b_sparsity_plot.pdf" not in lines
+    assert "/B_sparsity_plot.pdf" in lines
     assert repo.ignored("B_sparsity_plot.pdf")
+
+
+def test_gitignore_not_unignored_latex_pdf_output(tmp_dir):
+    repo = git.Repo.init()
+    subprocess.check_call(["calkit", "init"])
+    os.makedirs("paper", exist_ok=True)
+    with open("paper/.gitignore", "w") as f:
+        f.write("/main.pdf\n")
+    ck_info = {
+        "environments": {
+            "tex": {
+                "kind": "conda",
+                "path": "environment.yaml",
+            }
+        },
+        "pipeline": {
+            "stages": {
+                "build-paper": {
+                    "kind": "latex",
+                    "environment": "tex",
+                    "target_path": "paper/main.tex",
+                    "force": True,
+                    "inputs": [
+                        "paper/references.bib",
+                        "paper/aasjournal.bst",
+                        "paper/aastex631.cls",
+                        "paper/results.tex",
+                        "paper/diagrams",
+                        "paper/figures",
+                    ],
+                }
+            }
+        },
+    }
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
+    assert not os.path.exists(".gitignore")
+    assert repo.ignored("paper/main.pdf")
+    calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
+    assert not os.path.exists(".gitignore")
+    with open("paper/.gitignore") as f:
+        assert f.read().splitlines() == ["/main.pdf"]
+    assert repo.ignored("paper/main.pdf")