pivot 0.3.0__tar.gz → 0.3.2__tar.gz

{pivot-0.3.0 → pivot-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pivot
-Version: 0.3.0
+Version: 0.3.2
 Summary: High-performance Python pipeline tool with automatic code change detection
 Author: Sami Jawhar
 Author-email: Sami Jawhar <sami@metr.org>

{pivot-0.3.0 → pivot-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pivot"
-version = "0.3.0"
+version = "0.3.2"
 description = "High-performance Python pipeline tool with automatic code change detection"
 requires-python = ">=3.13,<3.14"
 readme = "README.md"

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/cli/checkout.py

@@ -67,11 +67,13 @@ def _get_stage_output_info() -> dict[str, HashInfo]:
     return result
 
 
-def _on_disk_matches_expected(path: pathlib.Path, expected_hash: HashInfo) -> bool:
+def _on_disk_matches_expected(
+    path: pathlib.Path, expected_hash: HashInfo, cache_dir: pathlib.Path
+) -> bool:
     if is_dir_hash(expected_hash):
         if not path.is_dir():
             return False
-        actual_hash, _ = cache.hash_directory(path)
+        actual_hash, _ = cache.hash_directory(path, cache_dir=cache_dir)
     else:
         if not path.is_file():
             return False
@@ -119,7 +121,7 @@ def _restore_path_locked(
     if path.exists():
         match behavior:
             case CheckoutBehavior.ERROR:
-                if _on_disk_matches_expected(path, output_hash):
+                if _on_disk_matches_expected(path, output_hash, cache_dir):
                     return ("skipped", path.name)
                 raise click.ClickException(
                     f"'{path.name}' already exists with different content. "
@@ -150,7 +152,12 @@ def _restore_path_locked(
                         return ("missing", path.name)
                     return ("restored" if outcome["restored"] else "skipped", path.name)
             case CheckoutBehavior.FORCE:
-                cache.remove_output(path)
+                missing = cache.missing_cache_objects(output_hash, cache_dir, checkout_modes)
+                if missing:
+                    raise click.ClickException(
+                        f"Cannot overwrite '{path.name}': cache object missing: {', '.join(missing)}. "
+                        + "The workspace output was not changed."
+                    )
             case _:  # pyright: ignore[reportUnnecessaryComparison] - defensive for future enum values
                 raise ValueError(f"Unhandled checkout behavior: {behavior}")  # pyright: ignore[reportUnreachable]
 

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/cli/commit.py

@@ -20,7 +20,9 @@ def commit_command(ctx: click.Context, stages: tuple[str, ...]) -> None:
     quiet = cli_ctx["quiet"]
 
     stage_names = cli_helpers.resolve_stage_names(list(stages) if stages else None)
-    committed, failed = commit.commit_stages(cli_helpers.get_pipeline(), stage_names)
+    committed, failed = commit.commit_stages(
+        cli_helpers.get_pipeline(), stage_names, warn_left_out_symlinks=not quiet
+    )
 
     if not quiet:
         if committed:

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/cli/console.py

@@ -117,8 +117,12 @@ class Console:
             self._echo(f"    {key}")
 
             if change_type == "modified":
-                self._echo(f"      Old: {self._color(str(old_val) if old_val else 'N/A', 'red')}")
-                self._echo(f"      New: {self._color(str(new_val) if new_val else 'N/A', 'green')}")
+                self._echo(
+                    f"      Old: {self._color(str(old_val) if old_val is not None else 'N/A', 'red')}"
+                )
+                self._echo(
+                    f"      New: {self._color(str(new_val) if new_val is not None else 'N/A', 'green')}"
+                )
             elif change_type == "added":
                 self._echo(f"      {self._color('(added)', 'green')} {new_val}")
             else:

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/cli/doctor.py

@@ -4,10 +4,11 @@ import enum
 import os
 import subprocess
 import sys
-from typing import TYPE_CHECKING, Literal, TypedDict
+from typing import TYPE_CHECKING, Literal, TypedDict, cast
 from urllib.parse import urlparse, urlunparse
 
 import click
+import pydantic
 
 from pivot.cli import decorators as cli_decorators
 from pivot.cli import helpers as cli_helpers
@@ -87,6 +88,32 @@ def _sanitize_url(url: str) -> str:
     return urlunparse(parsed._replace(netloc=netloc))
 
 
+def _find_invalid_remote(
+    remotes: dict[object, object],
+) -> tuple[object, object, str]:
+    """Return the first remote entry that fails the configuration validator."""
+    from pivot.config import models
+
+    for name, remote in remotes.items():
+        try:
+            models.PivotConfig.model_validate({"remotes": {name: remote}})
+        except pydantic.ValidationError as error:
+            detail = error.errors()[0]
+            return name, remote, detail["msg"].removeprefix("Value error, ")
+    raise AssertionError("invalid remotes mapping contains no invalid entry")
+
+
+def _format_config_validation_error(error: pydantic.ValidationError) -> str:
+    """Format a config validation error for the doctor command's user-facing boundary."""
+    detail = error.errors()[0]
+    location = ".".join(str(part) for part in detail["loc"])
+    value = detail["input"]
+    if location == "remotes" and isinstance(value, dict) and value:
+        name, remote, message = _find_invalid_remote(cast("dict[object, object]", value))
+        return f"Invalid config remotes.{name}: {remote!r}; {message}"
+    return f"Invalid config {location}: {value!r}; expected {detail['msg'].removeprefix('Value error, ')}"
+
+
 def _relative_path(path: pathlib.Path, project_root: pathlib.Path) -> str:
     """Return path relative to project root, or absolute if outside."""
     try:
@@ -316,7 +343,10 @@ def doctor(ctx: click.Context, as_json: bool, check_remote: bool) -> None:
     checks.append(project_check)
 
     checks.append(_check_pipeline_config(project_root))
-    checks.append(_check_cache_directory(project_root))
+    try:
+        checks.append(_check_cache_directory(project_root))
+    except pydantic.ValidationError as e:
+        raise click.ClickException(_format_config_validation_error(e)) from e
     checks.append(_check_git_repository(project_root))
 
     if check_remote:

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/cli/track.py

@@ -86,9 +86,14 @@ def _track_single_path(
     stage_outputs_resolved: dict[str, pathlib.Path],
     existing_tracked: dict[str, track_mod.PvtData],
     force: bool,
+    *,
+    warn_left_out_symlinks: bool,
 ) -> TrackResult:
     """Track a single file or directory.
 
+    ``warn_left_out_symlinks`` logs the warning naming the symlinks a tracked directory leaves
+    out; the caller turns it off under ``--quiet``, like this command's other warning.
+
     Returns:
         TrackResult with optional warning and the tracked path for the caller to handle output.
     """
@@ -149,7 +154,9 @@ def _track_single_path(
 
     # Hash and cache
     if path.is_dir():
-        tree_hash, manifest = cache.hash_directory(path)
+        tree_hash, manifest = cache.hash_directory(
+            path, cache_dir=cache_dir, warn_left_out_symlinks=warn_left_out_symlinks
+        )
         total_size = sum(e["size"] for e in manifest)
         num_files = len(manifest)
 
@@ -212,7 +219,14 @@ def track(ctx: click.Context, paths: tuple[str, ...], force: bool) -> None:
     existing_tracked = track_mod.discover_pvt_files(project_root)
 
     for path_str in paths:
-        result = _track_single_path(path_str, cache_dir, stage_outputs, existing_tracked, force)
+        result = _track_single_path(
+            path_str,
+            cache_dir,
+            stage_outputs,
+            existing_tracked,
+            force,
+            warn_left_out_symlinks=not quiet,
+        )
         if not quiet:
             if result["warning"]:
                 click.echo(result["warning"], err=True)

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/engine/engine.py

@@ -1787,9 +1787,11 @@ class Engine:
             return lock_service.acquire_many(lock_requests)
 
         lock_handle = await anyio.to_thread.run_sync(_acquire_artifact_locks)
-        await self._set_stage_state(stage_name, StageExecutionState.PREPARING)
-
         try:
+            lock_data = await anyio.to_thread.run_sync(_read_lock)
+            if lock_data is None:
+                return False
+            await self._set_stage_state(stage_name, StageExecutionState.PREPARING)
             return await self._try_skip_with_locks(
                 stage_name,
                 stage_info,

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/engine/watch.py

@@ -84,6 +84,8 @@ class WatchCoordinator:
 
     _graph: nx.DiGraph[str]
     _artifact_paths: dict[pathlib.Path, pivot_types.ArtifactIdentity]
+    _resolved_output_paths: dict[tuple[pathlib.Path, str], list[pivot_types.ArtifactIdentity]]
+    _resolved_input_paths: dict[pathlib.Path, list[pivot_types.ArtifactIdentity]]
 
     def __init__(
         self,
@@ -92,6 +94,29 @@ class WatchCoordinator:
     ) -> None:
         self._graph = graph
         self._artifact_paths = artifact_paths
+        self._rebuild_resolved_path_indexes()
+
+    def _rebuild_resolved_path_indexes(self) -> None:
+        """Rebuild the input and output aliases used for physical watch paths."""
+        self._resolved_input_paths = dict[pathlib.Path, list[pivot_types.ArtifactIdentity]]()
+        self._resolved_output_paths = dict[
+            tuple[pathlib.Path, str], list[pivot_types.ArtifactIdentity]
+        ]()
+        for artifact_path, identity in self._artifact_paths.items():
+            if engine_graph.get_producer(self._graph, identity) is None:
+                self._resolved_input_paths.setdefault(
+                    artifact_path.resolve(), list[pivot_types.ArtifactIdentity]()
+                ).append(identity)
+                continue
+            key = (artifact_path.parent.resolve(), artifact_path.name)
+            self._resolved_output_paths.setdefault(
+                key, list[pivot_types.ArtifactIdentity]()
+            ).append(identity)
+
+    def _refresh_resolved_path_indexes_for(self, path: pathlib.Path) -> None:
+        """Refresh aliases after a symlink or watched directory changes."""
+        if path.is_symlink() or path.is_dir():
+            self._rebuild_resolved_path_indexes()
 
     @property
     def graph(self) -> nx.DiGraph[str]:
@@ -104,6 +129,10 @@ class WatchCoordinator:
 
     def resolve_identity(self, path: pathlib.Path) -> pivot_types.ArtifactIdentity | None:
         """Resolve a filesystem event path to the matching artifact identity."""
+        self._refresh_resolved_path_indexes_for(path)
+        outputs = self._matching_resolved_output_identities(path)
+        if outputs:
+            return outputs[0]
         current = project.normalize_path(path)
         while True:
             identity = self._artifact_paths.get(current)
@@ -114,6 +143,38 @@ class WatchCoordinator:
                 return None
             current = parent
 
+    def _matching_identities(self, path: pathlib.Path) -> list[pivot_types.ArtifactIdentity]:
+        """Return identities whose resolved paths contain a filesystem event path."""
+        current = project.normalize_path(path)
+        identities = list[pivot_types.ArtifactIdentity]()
+        while True:
+            identity = self._artifact_paths.get(current)
+            if identity is not None:
+                identities.append(identity)
+            parent = current.parent
+            if parent == current:
+                return identities
+            current = parent
+
+    def _matching_resolved_input_identities(
+        self, path: pathlib.Path
+    ) -> list[pivot_types.ArtifactIdentity]:
+        resolved_path = path.resolve()
+        identities = list[pivot_types.ArtifactIdentity]()
+        current = resolved_path
+        while True:
+            identities.extend(self._resolved_input_paths.get(current, []))
+            parent = current.parent
+            if parent == current:
+                return identities
+            current = parent
+
+    def _matching_resolved_output_identities(
+        self, path: pathlib.Path
+    ) -> list[pivot_types.ArtifactIdentity]:
+        """Return output identities by resolved parent directory and logical leaf name."""
+        return self._resolved_output_paths.get((path.parent.resolve(), path.name), [])
+
     def should_filter_path(
         self,
         path: pathlib.Path,
@@ -134,13 +195,23 @@ class WatchCoordinator:
         """Get all stages affected by the changed paths, including downstream."""
         affected = set[str]()
         for path in paths:
-            identity = self.resolve_identity(path)
-            if identity is None:
-                continue
-            consumers = engine_graph.get_consumers(self._graph, identity)
-            affected.update(consumers)
-            for stage in consumers:
-                affected.update(engine_graph.get_downstream_stages(self._graph, stage))
+            self._refresh_resolved_path_indexes_for(path)
+            output_identities = set(self._matching_resolved_output_identities(path))
+            if output_identities:
+                identities = output_identities
+            else:
+                logical_identities = set(self._matching_identities(path))
+                identities = logical_identities.copy()
+                if all(
+                    engine_graph.get_producer(self._graph, identity) is None
+                    for identity in logical_identities
+                ):
+                    identities.update(self._matching_resolved_input_identities(path))
+            for identity in identities:
+                consumers = engine_graph.get_consumers(self._graph, identity)
+                affected.update(consumers)
+                for stage in consumers:
+                    affected.update(engine_graph.get_downstream_stages(self._graph, stage))
         return sorted(affected)
 
     def get_producer(self, path: pathlib.Path) -> str | None:

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/executor/commit.py

@@ -24,6 +24,8 @@ def commit_stages(
     pipeline: registry.PipelineLike,
     stage_names: list[str] | None = None,
     force: bool = False,
+    *,
+    warn_left_out_symlinks: bool = True,
 ) -> tuple[list[str], list[str]]:
     """Commit current workspace state for stages.
 
@@ -37,6 +39,8 @@ def commit_stages(
         pipeline: The pipeline whose stages are committed.
         stage_names: Specific stages to commit. None means all stale stages.
         force: If True, commit even if lock file is unchanged.
+        warn_left_out_symlinks: Log the warning naming symlinks a directory output leaves
+            out. A quiet command turns it off.
 
     Returns:
         Tuple of (committed, failed) stage name lists.
@@ -150,7 +154,12 @@ def commit_stages(
                 identity_key = types.identity_key(out.identity)
                 try:
                     out_path = store.resolve_display_path(out)
-                    hash_info = store.commit(out, out_path, state_dir=default_state_dir)
+                    hash_info = store.commit(
+                        out,
+                        out_path,
+                        state_dir=default_state_dir,
+                        warn_left_out_symlinks=warn_left_out_symlinks,
+                    )
                     file_hash_entries.extend(worker.output_file_hash_entries(out_path, hash_info))
                     output_hashes_by_key[identity_key] = hash_info
                     output_hashes_by_identity[out.identity] = hash_info

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/executor/core.py

@@ -409,7 +409,7 @@ def verify_tracked_files(project_root: pathlib.Path, checkout_missing: bool = Fa
                 if path.is_file():
                     current_hash, _ = cache.hash_file(path, state_db)
                 elif path.is_dir():
-                    current_hash, _ = cache.hash_directory(path, state_db)
+                    current_hash, _ = cache.hash_directory(path, state_db, cache_dir=cache_dir)
                 else:
                     # Path doesn't exist
                     raise FileNotFoundError(data_path)

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/executor/worker.py

@@ -411,7 +411,9 @@ def execute_stage(
                     project_root,
                 )
                 if no_commit:
-                    output_hashes = _hash_output_paths(output_paths)
+                    output_hashes = _hash_output_paths(
+                        output_paths, cache_dir=_store_cache_dir(store)
+                    )
                 else:
                     output_hashes = _commit_outputs_with_store(
                         output_paths, store, project_state_dir
@@ -556,7 +558,7 @@ def restore_outputs_to_hashes(
             if expected_hash is not None:
                 output_path = store.resolve_display_path(ref)
                 try:
-                    actual_hash = hash_output(output_path, state_db)
+                    actual_hash = hash_output(output_path, state_db, cache_dir=store.cache_dir)
                 except OSError:
                     actual_hash = None
                 if actual_hash == expected_hash:
@@ -613,10 +615,23 @@ def _check_skip_or_run(
     return "unchanged", "", input_hash, decision["run_cache_allowed"]
 
 
-def hash_output(path: pathlib.Path, state_db: state.StateDB | None = None) -> HashInfo:
-    """Compute output hash without saving to cache."""
+def _store_cache_dir(store: store_mod.Store) -> pathlib.Path | None:
+    """The cache object root whose symlinks count as directory content for ``store``."""
+    return store.cache_dir if isinstance(store, store_mod.WorkspaceStore) else None
+
+
+def hash_output(
+    path: pathlib.Path,
+    state_db: state.StateDB | None = None,
+    *,
+    cache_dir: pathlib.Path | None = None,
+) -> HashInfo:
+    """Compute output hash without saving to cache.
+
+    ``cache_dir`` is the cache object root whose symlinks count as directory content.
+    """
     if path.is_dir():
-        tree_hash, manifest = cache.hash_directory(path, state_db)
+        tree_hash, manifest = cache.hash_directory(path, state_db, cache_dir=cache_dir)
         return DirHash(hash=tree_hash, manifest=manifest)
     file_hash, _ = cache.hash_file(path, state_db)
     return FileHash(hash=file_hash)
@@ -1040,13 +1055,17 @@ def _run_stage_function_with_store(
 def _hash_output_paths(
     outputs_by_ref: dict[types.ArtifactRef, pathlib.Path],
     state_db: state.StateDB | None = None,
+    *,
+    cache_dir: pathlib.Path | None = None,
 ) -> dict[str, HashInfo]:
     output_hashes = dict[str, HashInfo]()
     for ref, path in outputs_by_ref.items():
         if not path.exists():
             raise exceptions.OutputMissingError(f"Stage did not produce output: {path}")
         if path.is_dir():
-            tree_hash, manifest = cache.hash_directory(path, state_db)
+            tree_hash, manifest = cache.hash_directory(
+                path, state_db, cache_dir=cache_dir, warn_left_out_symlinks=True
+            )
             output_hashes[types.identity_key(ref.identity)] = DirHash(
                 hash=tree_hash, manifest=manifest
             )

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/skip.py

@@ -190,7 +190,7 @@ def params_equal(old: object, new: object) -> bool:
     (``parameters.get_effective_params``), so the text is the canonical encoding. Python
     equality alone would call a NaN param changed on every run, because nan != nan.
     """
-    return old == new or json.dumps(old, sort_keys=True) == json.dumps(new, sort_keys=True)
+    return json.dumps(old, sort_keys=True) == json.dumps(new, sort_keys=True)
 
 
 def diff_dep_hashes(

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/status.py

@@ -348,6 +348,7 @@ def get_tracked_files_status(
     results = list[TrackedFileInfo]()
 
     # Use state_db for hash caching (mtime-based)
+    cache_dir = config.get_cache_dir() / "files"
     with state_mod.StateDB(config.get_state_dir()) as state_db:
         for i, (abs_path_str, track_data) in enumerate(sorted(tracked.items()), 1):
             path = pathlib.Path(abs_path_str)
@@ -355,7 +356,7 @@ def get_tracked_files_status(
 
             try:
                 if path.is_dir():
-                    current_hash, _ = cache.hash_directory(path, state_db)
+                    current_hash, _ = cache.hash_directory(path, state_db, cache_dir=cache_dir)
                 else:
                     current_hash, _ = cache.hash_file(path, state_db)
             except FileNotFoundError:

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/storage/cache.py

@@ -45,8 +45,8 @@ def _restore_target_token(target_path: pathlib.Path) -> str:
     return hashlib.sha256(str(logical_path).encode()).hexdigest()[:32]
 
 
-def _restore_staging_path(path: pathlib.Path, token: str | None = None) -> pathlib.Path:
-    """Return the deterministic staging path for a file checkout."""
+def restore_staging_path(path: pathlib.Path, token: str | None = None) -> pathlib.Path:
+    """Return the deterministic staging path for a file restore."""
     return path.parent / f"{_RESTORE_TEMP_PREFIX}{token or _restore_target_token(path)}.file"
 
 
@@ -92,26 +92,37 @@ def atomic_write_file(
     dest: pathlib.Path,
     write_fn: Callable[[int], None],
     mode: int = 0o644,
+    *,
+    staging_path: pathlib.Path | None = None,
 ) -> None:
-    """Atomically write to dest using temp file + rename pattern.
+    """Atomically write to dest using a temporary file and replacement.
+
+    A caller holding a restore lock can supply its target-tagged staging path so a
+    later restore of the same target cleans up a process killed before replacement.
 
     Args:
         dest: Target file path.
         write_fn: Function that receives the file descriptor and writes content.
                   MUST close fd (typically via os.fdopen which takes ownership).
         mode: File permissions (default 0o644).
+        staging_path: Optional deterministic temporary path beside ``dest``.
     """
     dest.parent.mkdir(parents=True, exist_ok=True)
-    fd, tmp_path = tempfile.mkstemp(dir=dest.parent, suffix=".tmp")
-    tmp = pathlib.Path(tmp_path)
+    if staging_path is None:
+        fd, tmp_path = tempfile.mkstemp(dir=dest.parent, suffix=".tmp")
+        tmp = pathlib.Path(tmp_path)
+    else:
+        tmp = staging_path
+        _clear_path(tmp)
+        fd = os.open(tmp, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
     fd_closed = False
     try:
         write_fn(fd)
         fd_closed = True  # write_fn took ownership via os.fdopen
-        os.chmod(tmp_path, mode)
+        os.chmod(tmp, mode)
         tmp.replace(dest)
     except BaseException:
-        tmp.unlink(missing_ok=True)
+        _clear_path(tmp)
         raise
     finally:
         # Only close fd if write_fn didn't (e.g., exception before os.fdopen)
@@ -159,39 +170,83 @@ def hash_file(
     return file_hash, file_stat
 
 
-def _scandir_recursive(path: pathlib.Path) -> Generator[os.DirEntry[str]]:
+def _cache_object_key(path: pathlib.Path, cache_dir: pathlib.Path | None) -> str | None:
+    """Return the cache key a symlink resolves to, if its target is an object in ``cache_dir``.
+
+    Cache objects live at ``<cache_dir>/XX/YYYY...`` named by their content hash; a file-by-file
+    symlink checkout restores each directory member as a link to one. A link anywhere else,
+    even to a hash-named file, is a user symlink.
+    """
+    if cache_dir is None:
+        return None
+    key = _get_symlink_cache_hash(path, cache_dir)
+    if key is None or not is_valid_hash(key) or not path.is_file():
+        return None
+    return key
+
+
+def _scandir_recursive(
+    path: pathlib.Path, cache_dir: pathlib.Path | None, left_out: list[pathlib.Path]
+) -> Generator[os.DirEntry[str]]:
     """Yield every file under `path` recursively, using os.scandir() for efficiency.
 
     A declared directory artifact is hashed and cached whole: no name is ignored, so a change
     to any file in a directory input reruns its consumers, and every file in a directory output
     survives cache and restore. Ignore rules belong to code fingerprinting and watch mode.
+    A symlink to an object in ``cache_dir`` (left by a symlink checkout) counts as the file it
+    points to; any other symlink is not stored and is appended to ``left_out``.
     DirEntry objects cache stat results, avoiding redundant syscalls.
-    Symlinks are skipped to prevent loops.
     PermissionError propagates to caller - partial hashes would be incorrect.
 
     Args:
         path: Directory to scan
+        cache_dir: Cache object root whose links count as content, or None for no cache
+        left_out: Collects the symlinks that are not part of the artifact
     """
     with os.scandir(path) as entries:
         for entry in entries:
             if entry.is_symlink():
-                continue
-            if entry.is_file():
+                if _cache_object_key(pathlib.Path(entry.path), cache_dir) is not None:
+                    yield entry
+                else:
+                    left_out.append(pathlib.Path(entry.path))
+            elif entry.is_file():
                 yield entry
             elif entry.is_dir():
-                yield from _scandir_recursive(pathlib.Path(entry.path))
+                yield from _scandir_recursive(pathlib.Path(entry.path), cache_dir, left_out)
+
+
+def _warn_left_out_symlinks(path: pathlib.Path, left_out: list[pathlib.Path]) -> None:
+    """Log one warning naming every symlink a directory artifact does not store."""
+    absolute = path.absolute()
+    cwd = pathlib.Path.cwd()
+    shown = absolute.relative_to(cwd) if absolute.is_relative_to(cwd) else path
+    names = ", ".join(sorted(str(link.relative_to(path)) for link in left_out))
+    one = len(left_out) == 1
+    logger.warning(
+        "Directory artifact %s leaves out %s %s (a symlink inside a directory artifact is not "
+        + "stored); replace %s with a real file or directory",
+        shown,
+        "symlink" if one else "symlinks",
+        names,
+        "it" if one else "each",
+    )
 
 
 def hash_directory(
     path: pathlib.Path,
     state_db: state_mod.StateDB | None = None,
+    *,
+    cache_dir: pathlib.Path | None = None,
+    warn_left_out_symlinks: bool = False,
 ) -> tuple[str, list[DirManifestEntry]]:
     """Compute tree hash of directory, returning hash and manifest.
 
     Symlink handling:
-    - Symlinks INSIDE directories are skipped (prevents infinite loops)
+    - A symlink to an object in ``cache_dir`` is hashed by its target's content under its own
+      relpath; the content must match the object's name
+    - Any other symlink inside the directory is left out of the manifest
     - Base path may be symlinked (resolved for consistency)
-    - Content-based hashing only (symlink metadata excluded from fingerprints)
 
     Note: For portability, paths are stored as normalized (symlinks preserved)
     in lock files, but resolved here for consistent hashing.
@@ -199,20 +254,30 @@ def hash_directory(
     Args:
         path: Directory to hash
         state_db: Optional StateDB for caching file hashes
+        cache_dir: The configured cache's object root (``<cache.dir>/files``). Pass it
+            wherever one exists, so recording and re-hashing agree on symlink checkouts;
+            None means no cache, and then no symlink is content.
+        warn_left_out_symlinks: Log one warning naming the symlinks left out. Set where a
+            manifest is recorded (a stage output, a tracked directory), not on every re-hash.
     """
     _t = metrics.start()
     manifest = list[DirManifestEntry]()
+    left_out = list[pathlib.Path]()
     resolved_base = path.resolve()
 
-    for entry in sorted(_scandir_recursive(path), key=lambda e: e.path):
+    for entry in sorted(_scandir_recursive(path, cache_dir, left_out), key=lambda e: e.path):
         file_path = pathlib.Path(entry.path)
-        # Verify file is still within the directory (paranoid check)
-        if not file_path.resolve().is_relative_to(resolved_base):
+        # Verify a regular file is still within the directory (paranoid check). A cache-object
+        # link resolves into the cache by design.
+        if not entry.is_symlink() and not file_path.resolve().is_relative_to(resolved_base):
             continue
         try:
             rel = file_path.relative_to(path)
             file_stat = entry.stat(follow_symlinks=True)
             file_hash, _ = hash_file(file_path, state_db)
+            if entry.is_symlink() and file_hash != _cache_object_key(file_path, cache_dir):
+                left_out.append(file_path)
+                continue
             manifest_entry: DirManifestEntry = {
                 "relpath": str(rel),
                 "hash": file_hash,
@@ -223,6 +288,9 @@ def hash_directory(
         except FileNotFoundError:
             continue  # File deleted between scan and hash
 
+    if left_out and warn_left_out_symlinks:
+        _warn_left_out_symlinks(path, left_out)
+
     manifest_json = json.dumps(manifest, sort_keys=True, separators=(",", ":"))
     tree_hash = xxhash.xxh64(manifest_json.encode()).hexdigest()
 
@@ -513,6 +581,20 @@ def _resolve_checkout_modes(
     return DEFAULT_CHECKOUT_MODE_ORDER.copy()
 
 
+def missing_cache_objects(
+    output_hash: HashInfo, cache_dir: pathlib.Path, checkout_modes: list[CheckoutMode]
+) -> list[str]:
+    """Return hashes required to restore an output that are absent from the cache."""
+    if is_dir_hash(output_hash):
+        tree_path = get_cache_path(cache_dir, output_hash["hash"])
+        if checkout_modes[0] == CheckoutMode.SYMLINK and tree_path.exists():
+            return []
+        hashes = [entry["hash"] for entry in output_hash["manifest"]]
+    else:
+        hashes = [output_hash["hash"]]
+    return [file_hash for file_hash in hashes if not get_cache_path(cache_dir, file_hash).exists()]
+
+
 def save_to_cache(
     path: pathlib.Path,
     cache_dir: pathlib.Path,
@@ -578,7 +660,7 @@ def _store_file(
     cache_path = get_cache_path(cache_dir, file_hash)
     copy_to_cache(path, cache_path)
     with restore_lock(path, state_dir) as lock_token:
-        staging_path = _restore_staging_path(path, lock_token)
+        staging_path = restore_staging_path(path, lock_token)
         _clear_path(staging_path)
         _checkout_with_fallback(
             path,
@@ -604,10 +686,10 @@ def _save_directory_to_cache(
             cache_dir_path = get_cache_path(cache_dir, existing_hash)
             if cache_dir_path.exists():
                 # Already correctly linked - compute manifest from actual content
-                _, manifest = hash_directory(path, state_db)
+                _, manifest = hash_directory(path, state_db, cache_dir=cache_dir)
                 return DirHash(hash=existing_hash, manifest=manifest)
 
-    tree_hash, manifest = hash_directory(path, state_db)
+    tree_hash, manifest = hash_directory(path, state_db, cache_dir=cache_dir)
     output_hash = DirHash(hash=tree_hash, manifest=manifest)
     _store_directory(
         path,
@@ -656,7 +738,7 @@ def _store_directory(
             for entry in manifest:
                 file_path = path / entry["relpath"]
                 cache_path = get_cache_path(cache_dir, entry["hash"])
-                staging_path = _restore_staging_path(file_path)
+                staging_path = restore_staging_path(file_path)
                 _clear_path(staging_path)
                 _checkout_with_fallback(
                     file_path,
@@ -777,7 +859,7 @@ def _restore_file_from_cache(
                 lock_token=held_token,
             )
 
-    staging_path = _restore_staging_path(path, lock_token)
+    staging_path = restore_staging_path(path, lock_token)
     _clear_path(staging_path)
     _checkout_with_fallback(path, cache_path, checkout_modes, staging_path=staging_path)
     return True

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/storage/restore.py

@@ -237,12 +237,18 @@ def _verify_content_hash(content: bytes, expected_hash: str) -> bool:
     return actual_hash == expected_hash
 
 
-def _write_fallback_content(dest_path: pathlib.Path, content: bytes, force: bool) -> None:
-    """Replace a cache-miss target while holding the same lock as cache restores."""
-    with cache.restore_lock(dest_path, config.get_state_dir()):
-        if (force or dest_path.is_symlink()) and (dest_path.exists() or dest_path.is_symlink()):
-            cache.remove_output(dest_path)
-        dest_path.write_bytes(content)
+def _write_fallback_content(dest_path: pathlib.Path, content: bytes) -> None:
+    """Atomically replace a cache-miss target under the cache restore lock."""
+
+    def write_content(fd: int) -> None:
+        with os.fdopen(fd, "wb") as file:
+            file.write(content)
+
+    with cache.restore_lock(dest_path, config.get_state_dir()) as lock_token:
+        staging_path = (
+            cache.restore_staging_path(dest_path, lock_token) if lock_token is not None else None
+        )
+        cache.atomic_write_file(dest_path, write_content, staging_path=staging_path)
 
 
 def restore_file(
@@ -299,7 +305,7 @@ def restore_file(
     content = git.read_file_from_revision(rel_path, rev)
     if content is not None:
         try:
-            _write_fallback_content(dest_path, content, force)
+            _write_fallback_content(dest_path, content)
             return RestoreResult(
                 status=RestoreStatus.RESTORED,
                 path=path_str,
@@ -328,7 +334,7 @@ def restore_file(
                     ),
                 )
             try:
-                _write_fallback_content(dest_path, remote_content, force)
+                _write_fallback_content(dest_path, remote_content)
             except OSError as e:
                 return RestoreResult(
                     status=RestoreStatus.ERROR,

{pivot-0.3.0 → pivot-0.3.2}/src/pivot/storage/store.py

@@ -165,6 +165,11 @@ class WorkspaceStore:
         self._cache_dir = cache_dir.resolve() if cache_dir is not None else None
         self._checkout_modes = checkout_modes
 
+    @property
+    def cache_dir(self) -> pathlib.Path | None:
+        """The cache object root (``<cache.dir>/files``), or None when this store has no cache."""
+        return self._cache_dir
+
     def _resolve_input_path(self, ref: types.ArtifactRef) -> pathlib.Path:
         name = ref.identity.producer
         if name in self._input_bindings:
@@ -216,11 +221,21 @@ class WorkspaceStore:
         path: pathlib.Path,
         *,
         state_dir: pathlib.Path | None = None,
+        warn_left_out_symlinks: bool = True,
     ) -> types.HashInfo:
-        """Hash a written output once and, unless it is a metric, place it in the cache."""
+        """Hash a written output once and, unless it is a metric, place it in the cache.
+
+        ``warn_left_out_symlinks`` logs the warning naming the symlinks a directory output
+        leaves out; a quiet command turns it off.
+        """
         output_hash: types.HashInfo
         if path.is_dir():
-            tree_hash, manifest = cache.hash_directory(path, None)
+            tree_hash, manifest = cache.hash_directory(
+                path,
+                None,
+                cache_dir=self._cache_dir,
+                warn_left_out_symlinks=warn_left_out_symlinks,
+            )
             output_hash = types.DirHash(hash=tree_hash, manifest=manifest)
         else:
             file_hash, _ = cache.hash_file(path, None)
@@ -261,7 +276,7 @@ class WorkspaceStore:
     ) -> types.HashInfo:
         path = self._resolve_path(ref)
         if ref.tag == types.ArtifactTag.DIRECTORY or path.is_dir():
-            tree_hash, manifest = cache.hash_directory(path, state_db)
+            tree_hash, manifest = cache.hash_directory(path, state_db, cache_dir=self._cache_dir)
             return types.DirHash(hash=tree_hash, manifest=manifest)
         file_hash, _ = cache.hash_file(path, state_db)
         return types.FileHash(hash=file_hash)