calkit-python 0.45.1__py3-none-any.whl → 0.45.2__py3-none-any.whl

calkit/cli/main/core.py

@@ -286,9 +286,18 @@ def init(
         needs_own_repo = True
     if needs_own_repo:
         subprocess.run(["git", "init"])
-    result = calkit.dvc.init(force=force)
-    if result != 0:
-        raise_error("Failed to initialize DVC")
+    # A DVC repo can already exist here without a calkit.yaml, e.g., on
+    # platforms that set one up for us, so leave it alone rather than
+    # failing on DVC's refusal to initialize over an existing .dvc. The
+    # marker is .dvc/config, as elsewhere: a .dvc directory without one is
+    # an interrupted initialization, which has nothing worth keeping and
+    # which DVC will only initialize over with --force.
+    if os.path.isfile(os.path.join(".dvc", "config")) and not force:
+        typer.echo("DVC is already initialized")
+    else:
+        result = calkit.dvc.init(force=force or os.path.isdir(".dvc"))
+        if result != 0:
+            raise_error("Failed to initialize DVC")
     # Ensure autostage is enabled for DVC
     result = calkit.dvc.run_dvc_command(["config", "core.autostage", "true"])
     if result != 0:

calkit/dvc/core.py

@@ -98,6 +98,60 @@ if sys.platform == "win32":
     _tolerate_lock_release_failures()
 
 
+def _hash_dirs_inside_nested_repos() -> None:
+    """Make DVC hash directories that live inside nested DVC repos.
+
+    DVC's ``DvcIgnoreFilter`` prunes nested DVC repos (e.g. isolated
+    subprojects) when walking, so the parent never indexes a subproject's
+    ``dvc.yaml`` and ``.dvc`` files as its own. The same walk hashes directory
+    deps and outs, though, and the subrepo check is order-dependent:
+    ``_get_trie_pattern`` hands the starting directory's ``dnames`` to every
+    ancestor it visits, so a walk that starts inside the nested repo never
+    registers it, while the root-level index walk does. Whichever touches the
+    ignore trie first wins for the rest of the process. In practice a parent
+    stage depending on a directory inside an isolated subproject hashes it as
+    *empty* on ``status`` but not always on ``repro``, so ``dvc.lock`` and
+    ``dvc status`` disagree and the stage is forever stale (or, worse, changes
+    in the subproject go unnoticed).
+
+    A walk that starts at or below a nested repo can only be a caller hashing
+    that path (the index walk always starts at the repo root), and the caller
+    named the path deliberately, so its contents must be hashed. Such walks
+    are routed through DVC's ``ignore_subrepos=False`` trie, which is built
+    separately, is deterministic, and still honors every ``.dvcignore`` on
+    the way down, including the nested repo's own.
+    """
+    from dvc.ignore import DvcIgnoreFilter
+    from dvc.repo import Repo
+
+    original_walk = DvcIgnoreFilter.walk
+
+    def in_nested_repo(dvcignore: Any, path: str) -> bool:
+        fs = dvcignore.fs
+        root_dir = dvcignore.root_dir
+        current = fs.abspath(path)
+        if not fs.isin(current, root_dir):
+            return False
+        while current != root_dir:
+            if fs.exists(fs.join(current, Repo.DVC_DIR)):
+                return True
+            parent = fs.parent(current)
+            if parent == current:
+                return False
+            current = parent
+        return False
+
+    def walk(self: Any, fs: Any, path: str, **kwargs: Any) -> Any:
+        if "ignore_subrepos" not in kwargs and in_nested_repo(self, path):
+            kwargs["ignore_subrepos"] = False
+        return original_walk(self, fs, path, **kwargs)
+
+    DvcIgnoreFilter.walk = walk
+
+
+_hash_dirs_inside_nested_repos()
+
+
 # Default seconds to wait for DVC's repo-level lock during a pipeline run.
 #
 # DVC's own default is only 3 seconds (``dvc.lock.DEFAULT_TIMEOUT``), after

calkit/tests/cli/main/test_core.py

@@ -149,6 +149,25 @@ def test_init(tmp_dir):
     # --force allows re-initialization without clobbering calkit.yaml
     subprocess.check_call(["calkit", "init", "--force"], cwd="sub")
     assert calkit.load_calkit_info(wdir="sub") == ck_info
+    # An existing DVC repo with no calkit.yaml is left alone rather than
+    # failing on DVC's refusal to initialize over an existing .dvc (#1013)
+    os.remove(os.path.join("sub", "calkit.yaml"))
+    assert os.path.isdir(os.path.join("sub", ".dvc"))
+    result = subprocess.run(
+        ["calkit", "init"],
+        cwd="sub",
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0
+    assert "already initialized" in result.stdout.lower()
+    assert os.path.isfile(os.path.join("sub", "calkit.yaml"))
+    assert calkit.load_calkit_info(wdir="sub") == {}
+    # A .dvc directory left behind by an interrupted init has no config in
+    # it, so it is re-initialized rather than taken for a working repo
+    os.remove(os.path.join("sub", ".dvc", "config"))
+    subprocess.check_call(["calkit", "init"], cwd="sub")
+    assert os.path.isfile(os.path.join("sub", ".dvc", "config"))
     # --no-commit stages the initial files without committing, so a
     # bootstrap (as run by xr) can be rolled back by restoring files
     os.makedirs("nocommit")

calkit/tests/cli/main/test_subprojects.py

@@ -4,6 +4,7 @@ import os
 import subprocess
 
 import calkit
+import calkit.dvc
 import calkit.pipeline
 
 
@@ -375,6 +376,106 @@ def test_isolated_subproject_external_dep(tmp_dir):
     assert not status.is_stale
 
 
+def test_isolated_subproject_dir_dep(tmp_dir):
+    # A parent stage depending on a *directory* inside an isolated
+    # subproject: DVC prunes nested repos when walking, so without calkit's
+    # workaround the directory hashes as empty on status (but not always on
+    # repro), leaving the stage forever stale and blind to subproject changes.
+    def write_sub(b_content):
+        # Create the output directory here, since mkdir -p doesn't work in
+        # cmd.exe on Windows
+        os.makedirs("sub/figs", exist_ok=True)
+        write_ck_info(
+            "sub/calkit.yaml",
+            {
+                "pipeline": {
+                    "stages": {
+                        "make-figs": make_stage(
+                            "echo a > figs/a.txt && "
+                            f"echo {b_content} > figs/b.txt",
+                            outputs=[
+                                {"path": "figs/a.txt", "storage": "git"},
+                                {"path": "figs/b.txt", "storage": "git"},
+                            ],
+                        )
+                    }
+                }
+            },
+        )
+        calkit.pipeline.to_dvc(wdir="sub", write=True, manage_gitignore=False)
+
+    subprocess.check_call(["calkit", "init"])
+    init_isolated_subproject("sub", {})
+    write_sub("b")
+    write_ck_info(
+        "calkit.yaml",
+        {
+            "subprojects": [{"path": "sub"}],
+            "pipeline": {
+                "stages": {
+                    "consume": make_stage(
+                        "cat sub/figs/* > parent.txt",
+                        inputs=["sub/figs/"],
+                        outputs=[{"path": "parent.txt", "storage": "git"}],
+                    )
+                }
+            },
+        },
+    )
+    subprocess.check_call(["calkit", "run"])
+    # Windows' echo leaves trailing spaces, so compare tokens
+    with open("parent.txt") as f:
+        assert f.read().split() == ["a", "b"]
+    # The lock must record the directory's real contents, not an empty dir
+    with open("dvc.lock") as f:
+        lock = calkit.ryaml.load(f)
+    figs_dep = [
+        d
+        for d in lock["stages"]["consume"]["deps"]
+        if d["path"] == "sub/figs/"
+    ][0]
+    assert figs_dep["nfiles"] == 2
+    # And status must agree with the lock
+    status = calkit.pipeline.get_status(
+        check_environments=False, compile_to_dvc=False
+    )
+    assert not status.is_stale
+    # Hashing must not depend on whether the index walk (which registers the
+    # subrepo in DVC's ignore trie) ran first
+    repo = calkit.dvc.get_dvc_repo()
+    list(repo.index.stages)
+    found = sorted(
+        os.path.relpath(p)
+        for p in repo.dvcignore.find(repo.fs, os.path.abspath("sub/figs"))
+    )
+    assert found == [
+        os.path.join("sub", "figs", "a.txt"),
+        os.path.join("sub", "figs", "b.txt"),
+    ]
+    # The index walk still keeps the subproject's own pipeline out of the
+    # parent's index
+    assert {s.addressing for s in repo.index.stages} == {"sub", "consume"}
+    # A change inside the subproject propagates to the parent stage
+    write_sub("c")
+    subprocess.check_call(["calkit", "run"])
+    with open("parent.txt") as f:
+        assert f.read().split() == ["a", "c"]
+    status = calkit.pipeline.get_status(
+        check_environments=False, compile_to_dvc=False
+    )
+    assert not status.is_stale
+    # Editing a file in the directory makes the parent stage stale
+    with open("sub/figs/a.txt", "w") as f:
+        f.write("z\n")
+    status = calkit.pipeline.get_status(
+        check_environments=False, compile_to_dvc=False
+    )
+    assert status.is_stale
+    assert [
+        p.rstrip("/") for p in status.stale_stages["consume"].modified_inputs
+    ] == ["sub/figs"]
+
+
 def test_targeted_status_uses_full_subproject_status_for_stage_targets(
     tmp_dir, monkeypatch
 ):

{calkit_python-0.45.1.dist-info → calkit_python-0.45.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: calkit-python
-Version: 0.45.1
+Version: 0.45.2
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.45.1.dist-info → calkit_python-0.45.2.dist-info}/RECORD

@@ -54,10 +54,10 @@ calkit/cli/scheduler.py,sha256=GQf_Hw2qVSrFH4SzSQCOdbZzZ-n2EdKgR2NoqNIqWlw,53064
 calkit/cli/sync.py,sha256=UGen0Yey6BARooI5K83eiArw4YN2ZQcRQjCocaGy6E4,2539
 calkit/cli/update.py,sha256=4CbMxL3jS2l2mF32174IYLjN0DpoqCOag814UekKaVk,48411
 calkit/cli/main/__init__.py,sha256=qu1POZPyqs33ZKfasOxv_Wc-EzVcEgK3Dt_vwFL8Bi8,65
-calkit/cli/main/core.py,sha256=zQoqy6uy1__2CjJpc4xm8wq5KExXvEN5rgi0vDmk9Dw,152154
+calkit/cli/main/core.py,sha256=VtLEXS7xIYvnucXx7vJwMBuSi_WsfGg7y7GorpGfrO4,152737
 calkit/cli/main/xr.py,sha256=cYQBNVlRTzp6ba3EwEwZylOxVpk9ThWhy6IAD7_ahWw,37594
 calkit/dvc/__init__.py,sha256=-B6tilCb_jw7QFmJ2srILez24wDhNKUzIBhZUMt01dE,381
-calkit/dvc/core.py,sha256=_viZIkl5qO6Bnipru0w2zaEc2xwXuEOSYCDvlDoEaU0,36282
+calkit/dvc/core.py,sha256=0yfq_p3CrejFxGEl19za9M2ZdywByZPS4MtndK_R2Mo,38637
 calkit/dvc/zip.py,sha256=nf4EzOyc5nI_OA3M6S7wwZpL7N4EQz9yCf8FbnPjlxM,21857
 calkit/jupyterlab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 calkit/jupyterlab/routes.py,sha256=sARrFjWnHVV-LunrgqO4DCt_x7XLhVPAqA1eaUFf7HQ,51443
@@ -142,9 +142,9 @@ calkit/tests/cli/test_scheduler.py,sha256=YfJi_fcRlQt1bdFyo3b8BFelA1PcH-wn5lAVXi
 calkit/tests/cli/test_sync.py,sha256=jmhfgoMIjdihIuu9UVyHSizBDTF0hlBt2VWqEdVkAKk,4375
 calkit/tests/cli/test_update.py,sha256=i7dsfrDVD-0VA7knspzbb-6UZWHgGPVkbB6CsT0czMM,14878
 calkit/tests/cli/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-calkit/tests/cli/main/test_core.py,sha256=a22eVhjPeK4cyFIm9uAf_jOzDStJ-GRi7cKlxdQOY4E,92505
+calkit/tests/cli/main/test_core.py,sha256=LFS-3p01qVWlmG4RmB5CcRqMRVfllk0eLibWIqMb27E,93415
 calkit/tests/cli/main/test_lock.py,sha256=1iX8iSbGtr4O9OkaAi4d3gP3_QzNl6Rr6HLtbEpS6iU,9847
-calkit/tests/cli/main/test_subprojects.py,sha256=RKWHKoqWLgJnsBBbD5Q-DJHQ5agZdOS7ahzfPzkr878,14942
+calkit/tests/cli/main/test_subprojects.py,sha256=Qs5NvwhQEFlZUvAtOyCWN243PONz8N79np5HwlgUl7E,18709
 calkit/tests/cli/main/test_xr.py,sha256=iauAWKXdzaaRFu5W91l4ZdH_ZU3h2NAn3v5p-EhAbco,28798
 calkit/tests/dvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 calkit/tests/dvc/test_core.py,sha256=LCmtZO_vLkW4GHf8nfeZvouLifVIc3VuuArkdKf6vD8,18215
@@ -159,22 +159,22 @@ calkit/tests/models/test_pipeline.py,sha256=hglkTBry1ONKbzN1HmlpefbrqAbc-668C0MQ
 calkit/agent_skills/add-pipeline-stage/SKILL.md,sha256=kpYYb-rJsS4B9p1Ub3gL21irCG5ZZ4DvWZpdJeEwPZk,3772
 calkit/agent_skills/conventions/SKILL.md,sha256=osbAtYU0AwROZHgmOoQYbqYr3JwG_v-cJAmandm7yN4,9738
 calkit/agent_skills/create-pipeline/SKILL.md,sha256=2FGw1iDQVRk5FUq79GP3lPdgvgof2Wi1O9O-abGGtgs,5422
-calkit_python-0.45.1.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/package.json,sha256=tYRdVVqmPBoUJzNrMc7Da9Lk-NB54vhnplKRSlOTMIE,6224
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=NBG3t7gFYb6_2J-yPU29qS7Ck9pYgm8-wC5Q1LrzQeI,6082
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/909.6d8285ce7c45878ac508.js,sha256=bYKFznxFh4rFCPs8tWnbZgaFGp72DuPZGdkzYCoCaSU,114583
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.d7a43c7948f690d37d19.js,sha256=16Q8eUj2kNN9Gaeoe3hOSEUPMJ5IUHQnJF1h9DRUIJY,8737
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
-calkit_python-0.45.1.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
-calkit_python-0.45.1.dist-info/METADATA,sha256=RpRntx5V2HE6ajwpb2e1Lw4vbvOMokNZKs9cC7NZl_A,12818
-calkit_python-0.45.1.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-calkit_python-0.45.1.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
-calkit_python-0.45.1.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
-calkit_python-0.45.1.dist-info/RECORD,,
+calkit_python-0.45.2.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/package.json,sha256=tYRdVVqmPBoUJzNrMc7Da9Lk-NB54vhnplKRSlOTMIE,6224
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=NBG3t7gFYb6_2J-yPU29qS7Ck9pYgm8-wC5Q1LrzQeI,6082
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/909.6d8285ce7c45878ac508.js,sha256=bYKFznxFh4rFCPs8tWnbZgaFGp72DuPZGdkzYCoCaSU,114583
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.d7a43c7948f690d37d19.js,sha256=16Q8eUj2kNN9Gaeoe3hOSEUPMJ5IUHQnJF1h9DRUIJY,8737
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
+calkit_python-0.45.2.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
+calkit_python-0.45.2.dist-info/METADATA,sha256=Rr27zNPVQ_KH7uaw8FcHthxM9g6dHF_jsYw6aTp8qhA,12818
+calkit_python-0.45.2.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+calkit_python-0.45.2.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
+calkit_python-0.45.2.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
+calkit_python-0.45.2.dist-info/RECORD,,