calkit-python 0.39.0__tar.gz → 0.39.1__tar.gz

{calkit_python-0.39.0 → calkit_python-0.39.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.39.0
+Version: 0.39.1
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.39.0 → calkit_python-0.39.1}/calkit/__init__.py

@@ -1,9 +1,47 @@
+from __future__ import annotations
+
 from importlib.metadata import version as _version
+from typing import TYPE_CHECKING
 
 __version__ = _version("calkit-python")
 
 from .core import *  # noqa: F403, I001
 
+if TYPE_CHECKING:
+    from calkit import (  # noqa: F401
+        calc,
+        check,
+        cloud,
+        conda,
+        config,
+        datasets,
+        detect,
+        docker,
+        dvc,
+        environments,
+        fs,
+        git,
+        github,
+        gui,
+        invenio,
+        julia,
+        jupyter,
+        licenses,
+        magics,
+        matlab,
+        models,
+        notebooks,
+        office,
+        ops,
+        overleaf,
+        pipeline,
+        releases,
+        server,
+        templates,
+    )
+
+    from .notebooks import declare_notebook  # noqa: F401
+
 # Lazy-load submodules to speed up imports
 _SUBMODULES = {
     "git",

{calkit_python-0.39.0 → calkit_python-0.39.1}/calkit/cli/main/core.py

@@ -1053,10 +1053,32 @@ def pull(
             if calkit.dvc.detect_calkit_remote_type(name, url) == "http":
                 typer.echo(f"Checking authentication for DVC remote: {name}")
                 calkit.dvc.set_remote_auth(remote_name=name)
+    if (
+        not no_recursive
+        and "--recursive" not in dvc_args
+        and "-R" not in dvc_args
+    ):
+        dvc_args.append("--recursive")
     result = calkit.dvc.run_dvc_command(["pull"] + dvc_args)
     if result != 0:
         raise_error("DVC pull failed")
     calkit.dvc.zip.sync_all(direction="to-workspace")
+    if not no_recursive:
+        # Pull DVC in isolated subprojects (those with their own .dvc folder)
+        ck_info = calkit.load_calkit_info()
+        for sp in ck_info.get("subprojects", []):
+            if not isinstance(sp, dict) or not sp.get("path"):
+                continue
+            sp_path = sp["path"]
+            if not os.path.isdir(os.path.join(sp_path, ".dvc")):
+                continue
+            typer.echo(f"DVC pulling subproject: {sp_path}")
+            sp_result = calkit.dvc.run_dvc_command(
+                ["pull"] + dvc_args, cwd=sp_path
+            )
+            if sp_result != 0:
+                raise_error(f"DVC pull failed for subproject: {sp_path}")
+            calkit.dvc.zip.sync_all(direction="to-workspace", wdir=sp_path)
 
 
 @app.command(name="push")

{calkit_python-0.39.0 → calkit_python-0.39.1}/calkit/dvc/zip.py

@@ -46,14 +46,19 @@ ZIP_COMPRESS_LEVEL = 1
 ZIP_USE_SYSTEM_CLI = False
 
 
-def _check_local_dir() -> Path:
-    if not os.path.isdir(LOCAL_DIR):
-        os.makedirs(LOCAL_DIR, exist_ok=True)
-    gitignore_path = os.path.join(LOCAL_DIR, ".gitignore")
+def _local_dir(wdir: str | None = None) -> str:
+    return os.path.join(wdir, LOCAL_DIR) if wdir else LOCAL_DIR
+
+
+def _check_local_dir(wdir: str | None = None) -> Path:
+    local_dir = _local_dir(wdir)
+    if not os.path.isdir(local_dir):
+        os.makedirs(local_dir, exist_ok=True)
+    gitignore_path = os.path.join(local_dir, ".gitignore")
     if not os.path.isfile(gitignore_path):
         with open(gitignore_path, "w") as f:
             f.write("*\n")
-    return Path(LOCAL_DIR)
+    return Path(local_dir)
 
 
 def is_zip_candidate(path: str) -> bool:
@@ -111,10 +116,15 @@ def make_zip_path(workspace_path: str) -> str:
     return os.path.join(ZIPS_DIR, "files", workspace_path + ".zip")
 
 
-def get_zip_path_map() -> dict[str, str]:
+def _path_map_path(wdir: str | None = None) -> str:
+    return os.path.join(wdir, PATH_MAP_PATH) if wdir else PATH_MAP_PATH
+
+
+def get_zip_path_map(wdir: str | None = None) -> dict[str, str]:
     """Get a mapping of input paths to zip paths."""
-    if os.path.isfile(PATH_MAP_PATH):
-        with open(PATH_MAP_PATH, "r") as f:
+    path = _path_map_path(wdir)
+    if os.path.isfile(path):
+        with open(path, "r") as f:
             return json.load(f)
     return {}
 
@@ -207,19 +217,22 @@ def calc_dir_sig(path: str) -> str:
     return f"{file_count}-{total_size}-{latest_mtime}"
 
 
-def get_hash(path: str, alg="md5") -> str | None:
+def get_hash(path: str, alg="md5", wdir: str | None = None) -> str | None:
     """Get the hash of a path, using/updating the cache if applicable."""
     try:
         st = os.stat(path)
     except OSError:
         return None
-    _check_local_dir()
+    _check_local_dir(wdir)
     # Normalize path as posix
     path = Path(path).as_posix()
     mtime = st.st_mtime_ns
     size = st.st_size
     dir_sig = calc_dir_sig(path)
-    with SqliteDict(HASH_CACHE_PATH) as cache:
+    hash_cache_path = (
+        os.path.join(wdir, HASH_CACHE_PATH) if wdir else HASH_CACHE_PATH
+    )
+    with SqliteDict(hash_cache_path) as cache:
         record = None
         raw = cache.get(path)
         if raw is not None:
@@ -250,38 +263,44 @@ def get_hash(path: str, alg="md5") -> str | None:
     return hash_val
 
 
-def get_sync_record(workspace_path: str) -> SyncRecord | None:
+def _sync_records_path(wdir: str | None = None) -> str:
+    return os.path.join(wdir, SYNC_RECORDS_PATH) if wdir else SYNC_RECORDS_PATH
+
+
+def get_sync_record(
+    workspace_path: str, wdir: str | None = None
+) -> SyncRecord | None:
     """Get a sync record for a given workspace path."""
-    _check_local_dir()
-    with SqliteDict(SYNC_RECORDS_PATH) as db:
+    _check_local_dir(wdir)
+    with SqliteDict(_sync_records_path(wdir)) as db:
         raw = db.get(workspace_path)
     if raw is not None:
         return SyncRecord.model_validate(raw)
     return None
 
 
-def write_sync_record(record: SyncRecord):
+def write_sync_record(record: SyncRecord, wdir: str | None = None):
     """Write a sync record."""
-    _check_local_dir()
-    with SqliteDict(SYNC_RECORDS_PATH) as db:
+    _check_local_dir(wdir)
+    with SqliteDict(_sync_records_path(wdir)) as db:
         db[record.workspace_path] = record.model_dump()
         db.commit()
 
 
-def delete_sync_record(workspace_path: str):
+def delete_sync_record(workspace_path: str, wdir: str | None = None):
     """Delete a sync record."""
-    _check_local_dir()
-    with SqliteDict(SYNC_RECORDS_PATH) as db:
+    _check_local_dir(wdir)
+    with SqliteDict(_sync_records_path(wdir)) as db:
         if workspace_path in db:
             del db[workspace_path]
             db.commit()
 
 
-def cleanup_sync_records():
+def cleanup_sync_records(wdir: str | None = None):
     """Remove sync records for paths no longer in the path map."""
-    _check_local_dir()
-    pm = get_zip_path_map()
-    with SqliteDict(SYNC_RECORDS_PATH) as db:
+    _check_local_dir(wdir)
+    pm = get_zip_path_map(wdir)
+    with SqliteDict(_sync_records_path(wdir)) as db:
         stale = [k for k in db if k not in pm]
         if stale:
             for k in stale:
@@ -289,8 +308,8 @@ def cleanup_sync_records():
             db.commit()
 
 
-def get_zip_path(workspace_path: str) -> str:
-    pm = get_zip_path_map()
+def get_zip_path(workspace_path: str, wdir: str | None = None) -> str:
+    pm = get_zip_path_map(wdir)
     workspace_path = Path(workspace_path).as_posix()
     if workspace_path in pm:
         return pm[workspace_path]
@@ -408,14 +427,15 @@ class SyncStatus(BaseModel):
 
 
 def get_sync_status(
-    workspace_path: str, zip_path: str | None = None
+    workspace_path: str,
+    zip_path: str | None = None,
+    wdir: str | None = None,
 ) -> SyncStatus:
-    # First get cached information and see if we need to rehash
-    workspace_hash = get_hash(workspace_path)
+    workspace_hash = get_hash(workspace_path, wdir=wdir)
     if zip_path is None:
-        zip_path = get_zip_path(workspace_path)
-    zip_hash = get_hash(zip_path)
-    last_sync_record = get_sync_record(workspace_path)
+        zip_path = get_zip_path(workspace_path, wdir=wdir)
+    zip_hash = get_hash(zip_path, wdir=wdir)
+    last_sync_record = get_sync_record(workspace_path, wdir=wdir)
     if last_sync_record is not None:
         workspace_changed = workspace_hash != last_sync_record.workspace_hash
         zip_changed = zip_hash != last_sync_record.zip_hash
@@ -439,9 +459,23 @@ def sync_one(
     workspace_path: str,
     zip_path: str | None = None,
     direction: Literal["to-zip", "to-workspace", "both"] = "both",
+    wdir: str | None = None,
 ) -> SyncRecord | None:
     """Process a single zip."""
-    status = get_sync_status(workspace_path, zip_path)
+    if wdir:
+        workspace_path = os.path.join(wdir, workspace_path)
+        if zip_path is not None:
+            zip_path = os.path.join(wdir, zip_path)
+    return _sync_one(workspace_path, zip_path, direction, wdir=wdir)
+
+
+def _sync_one(
+    workspace_path: str,
+    zip_path: str | None = None,
+    direction: Literal["to-zip", "to-workspace", "both"] = "both",
+    wdir: str | None = None,
+) -> SyncRecord | None:
+    status = get_sync_status(workspace_path, zip_path, wdir=wdir)
     workspace_changed = status.workspace_changed
     zip_changed = status.zip_changed
     workspace_hash = status.workspace_hash
@@ -454,7 +488,7 @@ def sync_one(
     # Both deleted — clear the stale sync record so a future recreated side
     # is treated as a fresh first sync rather than a spurious conflict
     if workspace_deleted and zip_deleted:
-        delete_sync_record(workspace_path)
+        delete_sync_record(workspace_path, wdir=wdir)
         return None
     # Neither side exists and no sync record — nothing to do (e.g., a
     # pipeline output that hasn't been produced yet on a fresh run)
@@ -478,28 +512,28 @@ def sync_one(
         if os.path.exists(zip_path):
             typer.echo(f"Deleting '{zip_path}' (workspace was deleted)")
             os.remove(zip_path)
-        delete_sync_record(workspace_path)
+        delete_sync_record(workspace_path, wdir=wdir)
         return None
     # Propagate zip deletion to workspace
     if zip_deleted and direction in ["to-workspace", "both"]:
         if os.path.exists(workspace_path):
             typer.echo(f"Deleting '{workspace_path}' (zip was deleted)")
             shutil.rmtree(workspace_path)
-        delete_sync_record(workspace_path)
+        delete_sync_record(workspace_path, wdir=wdir)
         return None
     # Zip was deleted but workspace exists and direction is to-zip:
     # restore zip
     if zip_deleted and direction == "to-zip":
         typer.echo(f"Rezipping '{workspace_path}' (zip was deleted)")
         zip_(workspace_path=workspace_path, zip_path=zip_path)
-        run_dvc_command(["add", zip_path])
-        zip_hash = get_hash(zip_path)
+        run_dvc_command(["add", zip_path], cwd=wdir)
+        zip_hash = get_hash(zip_path, wdir=wdir)
     # Workspace was deleted but zip exists and direction is to-workspace:
     # restore workspace
     if workspace_deleted and direction == "to-workspace":
         typer.echo(f"Unzipping to '{workspace_path}' (workspace was deleted)")
         unzip(workspace_path=workspace_path, zip_path=zip_path)
-        workspace_hash = get_hash(workspace_path)
+        workspace_hash = get_hash(workspace_path, wdir=wdir)
     # If hashes have changed since last check, we need to synchronize the
     # path with its zip file (unzip if zip is newer, rezip if path is newer)
     # If both have changed, we have a conflict and the user needs to decide
@@ -514,13 +548,13 @@ def sync_one(
     if workspace_changed and (direction in ["to-zip", "both"]):
         typer.echo(f"Zipping '{workspace_path}' (workspace has changed)")
         zip_(workspace_path=workspace_path, zip_path=zip_path)
-        run_dvc_command(["add", zip_path])
-        zip_hash = get_hash(zip_path)
+        run_dvc_command(["add", zip_path], cwd=wdir)
+        zip_hash = get_hash(zip_path, wdir=wdir)
     # If we unzip, we need to update the hash
     if zip_changed and (direction in ["to-workspace", "both"]):
         typer.echo(f"Unzipping to '{workspace_path}' (zip has changed)")
         unzip(workspace_path=workspace_path, zip_path=zip_path)
-        workspace_hash = get_hash(workspace_path)
+        workspace_hash = get_hash(workspace_path, wdir=wdir)
     assert workspace_hash is not None and zip_hash is not None
     record = SyncRecord(
         workspace_path=workspace_path,
@@ -529,7 +563,7 @@ def sync_one(
         workspace_hash=workspace_hash,
         zip_hash=zip_hash,
     )
-    write_sync_record(record)
+    write_sync_record(record, wdir=wdir)
     return record
 
 
@@ -547,11 +581,15 @@ def sync_some(
         )
 
 
-def sync_all(direction: Literal["to-zip", "to-workspace", "both"] = "both"):
+def sync_all(
+    direction: Literal["to-zip", "to-workspace", "both"] = "both",
+    wdir: str | None = None,
+):
     """Process all project zips."""
-    for workspace_path, zip_path in get_zip_path_map().items():
+    for workspace_path, zip_path in get_zip_path_map(wdir=wdir).items():
         sync_one(
             workspace_path=workspace_path,
             zip_path=zip_path,
             direction=direction,
+            wdir=wdir,
         )

{calkit_python-0.39.0 → calkit_python-0.39.1}/calkit/pipeline.py

@@ -311,6 +311,98 @@ def _expand_matrix(input_dict: dict[str, list]) -> list[dict]:
     return final_list
 
 
+def _paths_overlap(a: str, b: str) -> bool:
+    """Return True if path `a` and path `b` overlap in the file tree.
+
+    Two paths overlap if they are equal or one is an ancestor directory of the
+    other, e.g. ``data/`` overlaps with ``data/file.csv``.
+    """
+    a = a.rstrip("/")
+    b = b.rstrip("/")
+    if a == b:
+        return True
+    return b.startswith(a + "/") or a.startswith(b + "/")
+
+
+def collapse_dvc_stages(
+    stages: dict,
+    cmd: str | None = None,
+    wdir: str | None = None,
+    desc: str | None = None,
+) -> dict:
+    """Collapse a dict of DVC stages into a single wrapper stage.
+
+    Collects all deps and outs from ``stages``, expands matrix templates, and
+    returns a complete DVC stage dict. ``cmd``, ``wdir``, and ``desc`` are
+    optional; when provided they are included in the returned dict.
+
+    Rules applied:
+    - A dep that tree-overlaps any out is dropped (e.g. a folder dep that
+      contains output files would cause a DVC conflict).
+    - An out that tree-overlaps any remaining dep is also dropped (covers
+      exact-match aliasing and folder/child relationships).
+    - Outputs are wrapped with ``{cache: false, persist: true}`` so the parent
+      doesn't double-cache files already managed by the subproject's DVC.
+    """
+    all_outs: set[str] = set()
+    all_deps: set[str] = set()
+    for stage_cfg in stages.values():
+        matrix = stage_cfg.get("matrix")
+        replacements = _expand_matrix(matrix) if matrix else [{}]
+        for out in stage_cfg.get("outs", []):
+            raw = out if isinstance(out, str) else list(out.keys())[0]
+            for r in replacements:
+                expanded = raw
+                for var, val in r.items():
+                    expanded = expanded.replace(f"${{item.{var}}}", str(val))
+                all_outs.add(Path(expanded).as_posix())
+        for dep in stage_cfg.get("deps", []):
+            if not isinstance(dep, str):
+                continue
+            for r in replacements:
+                expanded = dep
+                for var, val in r.items():
+                    expanded = expanded.replace(f"${{item.{var}}}", str(val))
+                all_deps.add(Path(expanded).as_posix())
+    external_deps = all_deps - all_outs
+    outs_raw = sorted(all_outs)
+    # Track whether any dep is dropped because it is a strict folder ancestor
+    # of an output (not an exact match — those are already removed by the set
+    # subtraction above). When this happens the wrapper cannot fully express
+    # the input boundary, so we mark it always_changed.
+    # A dep is a strict ancestor of an out when the out path starts with the
+    # dep path followed by "/". The reverse (dep inside an output folder) is
+    # just a normal internal path drop and does not require always_changed.
+    has_ancestor_dep_drop = any(
+        o.startswith(d.rstrip("/") + "/")
+        for d in external_deps
+        for o in outs_raw
+    )
+    deps = sorted(
+        d
+        for d in external_deps
+        if not any(_paths_overlap(d, o) for o in outs_raw)
+    )
+    dep_set = set(deps)
+    outs = [
+        {o: {"cache": False, "persist": True}}
+        for o in outs_raw
+        if not any(_paths_overlap(o, d) for d in dep_set)
+    ]
+    stage: dict = {}
+    if cmd is not None:
+        stage["cmd"] = cmd
+    if wdir is not None:
+        stage["wdir"] = wdir
+    if has_ancestor_dep_drop:
+        stage["always_changed"] = True
+    stage["deps"] = deps
+    stage["outs"] = outs
+    if desc is not None:
+        stage["desc"] = desc
+    return stage
+
+
 def get_status(
     ck_info: dict | None = None,
     targets: list[str] | None = None,
@@ -617,6 +709,8 @@ def to_dvc(
     dvc.yaml files in the tree, so no synthetic parent stages are needed.
 
     Returns a dictionary of DVC stages for the `stages` key of a dvc.yaml.
+    Returns ``{}`` when the project has no pipeline and no subprojects (e.g.
+    a subproject that is a library with no defined pipeline stages).
     """
     import calkit.dvc.zip
     from calkit.environments import get_env_lock_fpath
@@ -624,7 +718,7 @@ def to_dvc(
     if ck_info is None:
         ck_info = calkit.load_calkit_info(wdir=wdir)
     if "pipeline" not in ck_info and "subprojects" not in ck_info:
-        raise ValueError("No pipeline or subprojects found in calkit.yaml")
+        return {}
     # Compile subproject pipelines recursively.
     # For isolated subprojects (those with their own .dvc/ directory), DVC
     # won't cross the .dvc/ boundary during --all-pipelines discovery, so we
@@ -661,67 +755,22 @@ def to_dvc(
                 stacklevel=2,
             )
             continue
-        # Collect all outputs and all deps from the subproject's compiled stages.
-        # For matrix stages the template strings (${item.foo}) must be expanded
-        # so the wrapper stage has concrete paths.
-        all_sp_outs: set[str] = set()
-        all_sp_deps: set[str] = set()
-        for stage_cfg in sp_dvc_stages.values():
-            matrix = stage_cfg.get("matrix")
-            replacements = _expand_matrix(matrix) if matrix else [{}]
-            for out in stage_cfg.get("outs", []):
-                raw = out if isinstance(out, str) else list(out.keys())[0]
-                for r in replacements:
-                    expanded = raw
-                    for var, val in r.items():
-                        expanded = expanded.replace(
-                            f"${{item.{var}}}", str(val)
-                        )
-                    # Normalize to strip leading "./" so "data.txt" and
-                    # "./data.txt" are treated as the same path.
-                    all_sp_outs.add(Path(expanded).as_posix())
-            for dep in stage_cfg.get("deps", []):
-                if not isinstance(dep, str):
-                    continue
-                for r in replacements:
-                    expanded = dep
-                    for var, val in r.items():
-                        expanded = expanded.replace(
-                            f"${{item.{var}}}", str(val)
-                        )
-                    all_sp_deps.add(Path(expanded).as_posix())
-        # External deps are inputs the subproject reads from outside itself.
-        external_deps = all_sp_deps - all_sp_outs
-        # The wrapper stage has `wdir: sp`, so DVC resolves deps/outs relative
-        # to that directory — do NOT prefix with the subproject path.
-        wrapper_deps = sorted(
-            d for d in external_deps if not d.startswith(".calkit/env-locks/")
-        )
-        # Wrap outputs with cache: false + persist: true so the parent doesn't
-        # double-cache files already managed by the subproject's DVC, and the
-        # nested dvc repro run is responsible for file persistence.
-        wrapper_outs_raw = sorted(
-            o for o in all_sp_outs if not o.startswith(".calkit/env-locks/")
-        )
-        # Defensive deduplication: if any output path also appears as a dep
-        # (which can happen due to path aliasing), keep it as a dep only.
-        wrapper_dep_set = set(wrapper_deps)
-        wrapper_outs = [
-            {o: {"cache": False, "persist": True}}
-            for o in wrapper_outs_raw
-            if o not in wrapper_dep_set
-        ]
         sp_stage_name = Path(sp).name
-        wrapper_stages[sp_stage_name] = {
-            "cmd": "calkit dvc repro",
-            "wdir": sp,
-            "deps": wrapper_deps,
-            "outs": wrapper_outs,
-            "desc": (
+        wrapper = collapse_dvc_stages(
+            sp_dvc_stages,
+            cmd="calkit dvc repro",
+            wdir=sp,
+            desc=(
                 f"Automatically generated wrapper for subproject '{sp}'. "
                 "Changes made here will be overwritten."
             ),
-        }
+        )
+        # calkit.yaml is the source of truth for the subproject pipeline;
+        # tracking it ensures the wrapper goes stale when the pipeline
+        # definition changes.
+        if "calkit.yaml" not in wrapper["deps"]:
+            wrapper["deps"] = sorted(wrapper["deps"] + ["calkit.yaml"])
+        wrapper_stages[sp_stage_name] = wrapper
     if "pipeline" not in ck_info:
         if write and wrapper_stages:
             dvc_yaml_fpath = (