calkit-python 0.35.4__tar.gz → 0.35.5__tar.gz

{calkit_python-0.35.4 → calkit_python-0.35.5}/Makefile

@@ -37,12 +37,17 @@ test-docs: sync-docs ## Test if documentation can be built without warnings or e
 	@uv run mkdocs build -s
 
 .PHONY: sync-docs
-sync-docs: ## Sync documentation content from docs/*.md into README.md.
+sync-docs: cli-reference ## Sync documentation content from docs/*.md into README.md.
 	@echo "🚀 Syncing documentation"
 	@uv run python scripts/sync-docs.py
 
+.PHONY: cli-reference
+cli-reference: ## Generate docs/cli-reference.md from CLI help output.
+	@echo "🚀 Generating CLI reference"
+	@uv run python scripts/generate-cli-reference.py
+
 .PHONY: docs
-docs: ## Build and serve the documentation.
+docs: sync-docs ## Build and serve the documentation.
 	@uv run mkdocs serve --livereload
 
 .PHONY: import-profile

{calkit_python-0.35.4 → calkit_python-0.35.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.35.4
+Version: 0.35.5
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.35.4 → calkit_python-0.35.5}/calkit/conda.py

@@ -9,6 +9,7 @@ import shutil
 import subprocess
 import warnings
 from pathlib import Path
+from typing import cast
 
 import toml
 from packaging.specifiers import SpecifierSet
@@ -123,7 +124,13 @@ def _editable_package_name_from_dir(dir_path: str) -> str:
     elif os.path.isfile(os.path.join(dir_path, "pyproject.toml")):
         # Read pyproject.toml to get the package name
         with open(os.path.join(dir_path, "pyproject.toml")) as f:
-            pyproject = toml.load(f)
+            try:
+                pyproject = toml.load(f)
+            except Exception as e:
+                raise type(e)(
+                    f"Failed to load pyproject.toml from {dir_path}; "
+                    "check that it is valid TOML"
+                ) from e
         if "project" in pyproject:
             if "name" in pyproject["project"]:
                 return pyproject["project"]["name"]
@@ -139,6 +146,7 @@ def _check_single(
     """
     # If this is an editable install it needs to be handled specially
     # It also needs to be relative to the env spec dir
+    editable = False
     if req.startswith("-e ") or req.startswith("--editable "):
         req = req.split(" ", 1)[1]
         if "#" in req:
@@ -147,6 +155,7 @@ def _check_single(
         # Create path relative to env spec dir
         req = os.path.join(env_spec_dir, req)
         req = _editable_package_name_from_dir(req)
+        editable = True
     # If this is a Git version, we can't check it
     # TODO: Clone Git repos to check?
     if "@git" in req:
@@ -182,7 +191,7 @@ def _check_single(
         # TODO: Check exact version only
         return True
     spec = SpecifierSet(req_spec)
-    return spec.contains(version)
+    return spec.contains(version, prereleases=editable)
 
 
 def _check_list(
@@ -193,6 +202,10 @@ def _check_list(
     if "::" in req:
         req = req.split("::", 1)[1]
     for installed in actual:
+        if not isinstance(installed, str):
+            raise ValueError(
+                f"Expected installed package to be a string, got {installed}"
+            )
         if _check_single(
             req, installed, env_spec_dir=env_spec_dir, conda=conda
         ):
@@ -200,6 +213,46 @@ def _check_list(
     return False
 
 
+def _split_env_dependencies(
+    dependencies: list[str | dict[str, str | list[str]]],
+) -> tuple[list[str], list[str]]:
+    """Split an environment dependency list into conda and pip deps.
+
+    Conda environment files commonly include both the plain ``"pip"`` package
+    marker and a nested ``{"pip": [...]}`` section. This helper normalizes the
+    latter so callers do not need to assume it is the final list entry or that
+    the pip section is already represented as a list.
+    """
+    conda_deps = []
+    pip_deps = []
+    for dep in dependencies:
+        if isinstance(dep, dict):
+            dep_pip = dep.get("pip", [])
+            if isinstance(dep_pip, str):
+                dep_pip = [dep_pip]
+            elif dep_pip is None:
+                dep_pip = []
+            pip_deps.extend(dep_pip)
+        else:
+            conda_deps.append(dep)
+    return conda_deps, pip_deps
+
+
+def _get_pip_dependency_list(
+    dependencies: list[str | dict[str, str | list[str]]],
+) -> list[str]:
+    """Return a mutable pip dependency list from an env dependency list."""
+    for dep in dependencies:
+        if isinstance(dep, dict) and "pip" in dep:
+            dep_pip = dep["pip"]
+            if isinstance(dep_pip, str):
+                dep["pip"] = [dep_pip]
+            elif dep_pip is None:
+                dep["pip"] = []
+            return cast(list[str], dep["pip"])
+    return []
+
+
 class EnvCheckResult(BaseModel):
     env_exists: bool | None = None
     env_needs_export: bool | None = None
@@ -393,18 +446,12 @@ def check_env(
                 ryaml.dump(env_check, f)
         # Determine if the env matches
         env_needs_rebuild = False
-        if isinstance(env_check["dependencies"][-1], dict):
-            existing_conda_deps = env_check["dependencies"][:-1]
-            existing_pip_deps = env_check["dependencies"][-1]["pip"]
-        else:
-            existing_conda_deps = env_check["dependencies"]
-            existing_pip_deps = []
-        if isinstance(env_spec["dependencies"][-1], dict):
-            required_conda_deps = env_spec["dependencies"][:-1]
-            required_pip_deps = env_spec["dependencies"][-1]["pip"]
-        else:
-            required_conda_deps = env_spec["dependencies"]
-            required_pip_deps = []
+        existing_conda_deps, existing_pip_deps = _split_env_dependencies(
+            env_check["dependencies"]
+        )
+        required_conda_deps, required_pip_deps = _split_env_dependencies(
+            env_spec["dependencies"]
+        )
         if relaxed:
             log_func("Running in relaxed mode; combining pip and conda deps")
             for dep in existing_pip_deps:
@@ -516,20 +563,23 @@ def check_env(
         # Note that this needs to be relative to the env lock directory,
         # since that's how pip will interpret it
         editable_pip_deps = {}
-        if isinstance(env_spec["dependencies"][-1], dict):
-            # Map editable install dir to package name we'd see in lock
-            required_pip_deps = env_spec["dependencies"][-1]["pip"]
-            for dep in required_pip_deps:
-                if dep.startswith("-e ") or dep.startswith("--editable "):
-                    dir_path = dep.split(" ", 1)[1]
-                    if "#" in dir_path:
-                        dir_path = dir_path.split("#", 1)[0]
-                    dir_path = dir_path.strip()
-                    dir_path = os.path.join(env_spec_dir, dir_path)
-                    pkg_name = _editable_package_name_from_dir(dir_path)
-                    editable_pip_deps[pkg_name] = dir_path
-        if isinstance(env_export["dependencies"][-1], dict):
-            export_pip_deps = env_export["dependencies"][-1]["pip"]
+        required_pip_deps = _get_pip_dependency_list(env_spec["dependencies"])
+        for dep in required_pip_deps:
+            if dep.startswith("-e ") or dep.startswith("--editable "):
+                dir_path = dep.split(" ", 1)[1]
+                if "#" in dir_path:
+                    dir_path = dir_path.split("#", 1)[0]
+                dir_path = dir_path.strip()
+                dir_path = os.path.join(env_spec_dir, dir_path)
+                pkg_name = _editable_package_name_from_dir(dir_path)
+                if verbose:
+                    log_func(
+                        f"Found editable pip dependency '{pkg_name}' "
+                        f"at '{dir_path}'"
+                    )
+                editable_pip_deps[pkg_name] = dir_path
+        export_pip_deps = _get_pip_dependency_list(env_export["dependencies"])
+        if export_pip_deps:
             for i, dep in enumerate(export_pip_deps):
                 dep_name = re.split("[=<>]+", dep, maxsplit=1)[0]
                 if dep_name in editable_pip_deps:

{calkit_python-0.35.4 → calkit_python-0.35.5}/calkit/overleaf.py

@@ -317,6 +317,32 @@ class OverleafSyncPaths:
             - self.files_in_overleaf_last_sync
         )
 
+    @cached_property
+    def dvc_files(self) -> set[str]:
+        """Files tracked by DVC within the Overleaf project folder.
+
+        These paths are relative to the project directory (i.e., relative to
+        the Overleaf repo root). Files tracked by DVC may not exist on disk if
+        they haven't been pulled, but should still be kept on Overleaf rather
+        than deleted.
+        """
+        try:
+            import calkit.dvc
+
+            dvc_paths = calkit.dvc.list_paths(
+                wdir=str(self.main_repo.working_dir), recursive=True
+            )
+        except Exception as e:
+            warnings.warn(f"Could not list DVC files: {e}")
+            return set()
+        prefix = Path(self.path_in_project).as_posix().rstrip("/") + "/"
+        result = set()
+        for p in dvc_paths:
+            p_posix = Path(p).as_posix()
+            if p_posix.startswith(prefix):
+                result.add(p_posix[len(prefix) :])
+        return result
+
     @cached_property
     def files_to_keep_on_overleaf(self) -> set[str]:
         """Files that should be preserved on Overleaf.
@@ -324,9 +350,14 @@ class OverleafSyncPaths:
         This includes:
         1. All files being copied from local
         2. Any files newly added on Overleaf since last sync
+        3. Any files tracked by DVC within the project path (these may not
+           exist on disk if not pulled, but should not be deleted from
+           Overleaf)
         """
         return (
-            set(self.files_to_copy_to_overleaf) | self.newly_added_on_overleaf
+            set(self.files_to_copy_to_overleaf)
+            | self.newly_added_on_overleaf
+            | self.dvc_files
         )
 
     @cached_property

{calkit_python-0.35.4 → calkit_python-0.35.5}/calkit/pipeline.py

@@ -117,6 +117,20 @@ def to_dvc(
     except Exception as e:
         raise ValueError(f"Pipeline is not defined properly: {e}")
     dvc_stages = {}
+    # Read existing dvc.yaml now so we can clean up stale .gitignore entries
+    # when stage outputs are renamed or removed
+    if write:
+        dvc_yaml_path = os.path.join(wdir, "dvc.yaml") if wdir else "dvc.yaml"
+        if os.path.isfile(dvc_yaml_path):
+            with open(dvc_yaml_path) as f:
+                existing_dvc_yaml = calkit.ryaml.load(f)
+        else:
+            existing_dvc_yaml = {}
+        if existing_dvc_yaml is None:
+            existing_dvc_yaml = {}
+        existing_dvc_stages = existing_dvc_yaml.get("stages", {})
+    else:
+        existing_dvc_stages = {}
     # First, gather up any env lock paths we might need for DVC deps
     used_envs = set(
         [stage.inner_environment for stage in pipeline.stages.values()]
@@ -227,6 +241,27 @@ def to_dvc(
                 outputs += stage.notebook_outputs
             elif stage.kind == "sbatch":
                 outputs.append(stage.log_output)
+            # Build the set of current output paths so we can detect stale
+            # .gitignore entries from the previous version of the stage
+            current_out_paths = set()
+            for out in outputs:
+                if isinstance(out, PathOutput):
+                    current_out_paths.add(out.path)
+                elif isinstance(out, str):
+                    current_out_paths.add(out)
+            # If this stage already existed, un-ignore any outputs that have
+            # been renamed or removed so .gitignore does not accumulate stale
+            # entries (e.g., after a capitalization change in the path)
+            old_stage = existing_dvc_stages.get(stage_name, {})
+            for old_out in old_stage.get("outs", []):
+                if isinstance(old_out, str):
+                    old_path = old_out
+                elif isinstance(old_out, dict):
+                    old_path = list(old_out.keys())[0]
+                else:
+                    continue
+                if old_path not in current_out_paths:
+                    calkit.git.ensure_path_is_not_ignored(repo, path=old_path)
             # Deal with any gitignore changes necessary
             for out in outputs:
                 if isinstance(out, PathOutput) and out.storage is None:
@@ -268,14 +303,8 @@ def to_dvc(
                         else:
                             dvc_stages[stage_name]["deps"].append(out)
     if write:
-        if os.path.isfile("dvc.yaml"):
-            with open("dvc.yaml") as f:
-                dvc_yaml = calkit.ryaml.load(f)
-        else:
-            dvc_yaml = {}
-        if dvc_yaml is None:
-            dvc_yaml = {}
-        existing_stages = dvc_yaml.get("stages", {})
+        dvc_yaml = existing_dvc_yaml
+        existing_stages = existing_dvc_stages
         for stage_name, stage in existing_stages.items():
             # Skip private stages (ones whose names start with an underscore)
             # and stages that are automatically generated

{calkit_python-0.35.4 → calkit_python-0.35.5}/calkit/tests/cli/test_overleaf.py

@@ -181,6 +181,44 @@ def test_overleaf(tmp_dir):
     subprocess.run(["calkit", "overleaf", "sync", "--verbose"], check=True)
     print("Overleaf Git show after adding fig2 back:", ol_repo.git.show())
     assert "ol-project/figs/fig2.txt" in ls_files(repo)
+    # Test that if a file is deleted from Git but added to DVC, it is not
+    # deleted from Overleaf (the file still logically exists in the DVC repo)
+    with open(
+        os.path.join(repo.working_dir, "ol-project", "figs", "fig3.txt"), "w"
+    ) as f:
+        f.write("Fig3 created in main repo")
+    repo.git.add("ol-project/figs/fig3.txt")
+    repo.git.commit(["-m", "Add figure 3"])
+    assert "ol-project/figs/fig3.txt" in ls_files(repo)
+    subprocess.run(["calkit", "overleaf", "sync", "--verbose"], check=True)
+    ol_repo_git_show = ol_repo.git.show()
+    assert "diff --git a/figs/fig3.txt b/figs/fig3.txt" in ol_repo_git_show
+    # Now move from Git to DVC: first remove from Git index (keeping file on
+    # disk), then add to DVC so it gets moved to DVC cache
+    repo.git.rm(["--cached", "ol-project/figs/fig3.txt"])
+    subprocess.run(
+        ["dvc", "add", "ol-project/figs/fig3.txt"],
+        check=True,
+        cwd=repo.working_dir,
+    )
+    # Commit the DVC pointer file (fig3.txt is now tracked by DVC, not Git)
+    repo.git.add("ol-project/figs/fig3.txt.dvc", "ol-project/figs/.gitignore")
+    repo.git.commit(["-m", "Move figure 3 from git to DVC"])
+    assert "ol-project/figs/fig3.txt" not in ls_files(repo)
+    # Also remove the local file to simulate the file not being pulled from
+    # DVC (i.e., only the DVC pointer exists locally, not the actual file)
+    fig3_path = os.path.join(
+        repo.working_dir, "ol-project", "figs", "fig3.txt"
+    )
+    if os.path.exists(fig3_path):
+        os.remove(fig3_path)
+    assert not os.path.exists(fig3_path)
+    subprocess.run(["calkit", "overleaf", "sync", "--verbose"], check=True)
+    ol_repo_git_show = ol_repo.git.show()
+    print("Git show in OL repo after moving fig3 to DVC:\n", ol_repo_git_show)
+    # The file should not have been deleted from Overleaf
+    assert "deleted file mode" not in ol_repo_git_show
+    assert "--- a/figs/fig3.txt" not in ol_repo_git_show
 
 
 def test_extract_title_from_tex(tmp_dir):

{calkit_python-0.35.4 → calkit_python-0.35.5}/calkit/tests/test_conda.py

@@ -1,12 +1,19 @@
 """Tests for the ``conda`` module."""
 
 import os
+import shutil
 import subprocess
 
 import pytest
 
 import calkit
-from calkit.conda import _check_list, _check_single, check_env
+from calkit.conda import (
+    _check_list,
+    _check_single,
+    _get_pip_dependency_list,
+    _split_env_dependencies,
+    check_env,
+)
 
 ENV_NAME = "main"
 
@@ -37,6 +44,25 @@ def test_check_list():
     assert not _check_list("pandas", installed, env_spec_dir=".", conda=False)
 
 
+def test_split_env_dependencies():
+    dependencies = [
+        "python=3.12",
+        "pip",
+        "numpy=2",
+        {"pip": ["sqlalchemy==2.0.39"]},
+    ]
+    conda_deps, pip_deps = _split_env_dependencies(dependencies)
+    assert conda_deps == ["python=3.12", "pip", "numpy=2"]
+    assert pip_deps == ["sqlalchemy==2.0.39"]
+
+
+def test_get_pip_dependency_list():
+    dependencies = ["python=3.12", "pip", {"pip": "sqlalchemy==2.0.39"}]
+    pip_deps = _get_pip_dependency_list(dependencies)
+    assert pip_deps == ["sqlalchemy==2.0.39"]
+    assert dependencies[-1]["pip"] == ["sqlalchemy==2.0.39"]
+
+
 def delete_env(name: str):
     subprocess.check_call(["conda", "env", "remove", "-y", "-n", name])
 
@@ -262,7 +288,7 @@ def test_check_prefix_env(tmp_dir, conda_env_prefix):
     )
 
 
-def test_check_editable(tmp_dir, conda_env_name):
+def test_check_env_editable(tmp_dir, conda_env_name):
     subprocess.check_call(["calkit", "init"])
     # Create a dummy package named 'src' to install in editable mode
     os.makedirs("src", exist_ok=True)
@@ -306,6 +332,98 @@ setup(
         lock = calkit.ryaml.load(f)
     pip_deps = lock["dependencies"][-1]["pip"]
     assert "-e ." in pip_deps
+    # Now let's make sure we get proper output if the editable package is
+    # has an invalid pyproject.toml
+    os.remove("setup.py")
+    shutil.rmtree("src.egg-info")
+    toml_txt = """[build-system]
+requires = ["setuptools>=61.0.0", "wheel", "setuptools-scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "src-thing"
+dynamic = ["version"]
+authors = [
+  {name = "Someone"}
+]
+description = "Test"
+
+dependencies = [
+    "numpy>=1.21",
+    "scipy>=1.7",
+    "pandas>=1.5",
+    "matplotlib>=3.5",
+    "h5netcdf>=0.12",
+    "h5py>=3.0",
+    "xarray>=2023.0",
+    "streamlit>=1.0"
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+packages = ["src-thing"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+"src-thing" = [] # Explicitly state no package data
+
+[tool.setuptools_scm]
+local_scheme = "no-local-version"
+fallback_version = "0+unknown"
+"""
+    with open("pyproject.toml", "w") as f:
+        f.write(toml_txt)
+    with pytest.raises(Exception, match="Failed to load pyproject.toml"):
+        res = check_env()
+    # Fix it and make sure it runs with relaxed mode
+    toml_txt = """[build-system]
+requires = ["setuptools>=61.0.0", "wheel", "setuptools-scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "src-thing"
+dynamic = ["version"]
+authors = [
+  {name = "Someone"}
+]
+description = "Test"
+
+dependencies = []
+
+[tool.setuptools]
+packages = ["src"]
+
+[tool.setuptools_scm]
+local_scheme = "no-local-version"
+fallback_version = "0+unknown"
+"""
+    with open("pyproject.toml", "w") as f:
+        f.write(toml_txt)
+    res = check_env(relaxed=True)
+    assert res.env_exists
+    assert res.env_needs_rebuild
+    assert res.env_needs_export
+    # Make sure we can import the editable package
+    os.makedirs("subdir")
+    subprocess.check_call(
+        [
+            "conda",
+            "run",
+            "-n",
+            conda_env_name,
+            "python",
+            "-c",
+            "import src; print('src file:', src.__file__);",
+        ],
+        cwd="subdir",
+    )
+    # Check again and make sure we don't need a rebuild since the editable
+    # package is still valid
+    res = check_env(relaxed=True)
+    assert res.env_exists
+    assert not res.env_needs_rebuild
 
 
 def test_find_conda_exe():

{calkit_python-0.35.4 → calkit_python-0.35.5}/calkit/tests/test_pipeline.py

@@ -2,8 +2,10 @@
 
 import subprocess
 
+import git
 import pytest
 
+import calkit
 import calkit.pipeline
 from calkit.environments import get_env_lock_fpath
 from calkit.pipeline import stages_are_similar
@@ -648,3 +650,48 @@ def test_shell_script_stage_allows_non_composite_slurm_env():
         "calkit slurm batch --name run --environment mycluster "
         "-- scripts/run.sh a b"
     )
+
+
+def test_gitignore_updated_when_stage_output_renamed(tmp_dir):
+    """When a stage output path is renamed (e.g., from a capitalization change) the
+    old path should be removed from .gitignore and the new path should be
+    added.
+    """
+    subprocess.check_call(["calkit", "init"])
+    # Stage 1: initial calkit.yaml with output 'b_sparsity_plot.pdf' stored in DVC
+    ck_info = {
+        "pipeline": {
+            "stages": {
+                "plot": {
+                    "kind": "command",
+                    "environment": "_system",
+                    "command": "touch b_sparsity_plot.pdf",
+                    "outputs": [
+                        {
+                            "path": "b_sparsity_plot.pdf",
+                            "storage": "dvc",
+                        }
+                    ],
+                }
+            }
+        }
+    }
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    subprocess.check_call(["calkit", "run"])
+    # Verify DVC has added the old output path to .gitignore
+    repo = git.Repo(".")
+    assert repo.ignored("b_sparsity_plot.pdf")
+    # Stage 2: rename output (capitalization change) to 'B_sparsity_plot.pdf'
+    ck_info["pipeline"]["stages"]["plot"]["command"] = (
+        "touch B_sparsity_plot.pdf"
+    )
+    ck_info["pipeline"]["stages"]["plot"]["outputs"] = [
+        {"path": "B_sparsity_plot.pdf", "storage": "dvc"}
+    ]
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    subprocess.check_call(["calkit", "run"])
+    # Old (stale) path should no longer be ignored by git
+    assert not repo.ignored("b_sparsity_plot.pdf")
+    assert repo.ignored("B_sparsity_plot.pdf")