pivot 0.3.4__tar.gz → 0.3.6__tar.gz

{pivot-0.3.4 → pivot-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pivot
-Version: 0.3.4
+Version: 0.3.6
 Summary: High-performance Python pipeline tool with automatic code change detection
 Author: Sami Jawhar
 Author-email: Sami Jawhar <sami@metr.org>

{pivot-0.3.4 → pivot-0.3.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pivot"
-version = "0.3.4"
+version = "0.3.6"
 description = "High-performance Python pipeline tool with automatic code change detection"
 requires-python = ">=3.13,<3.14"
 readme = "README.md"

{pivot-0.3.4 → pivot-0.3.6}/src/pivot/cli/track.py

@@ -161,10 +161,6 @@ def _track_single_path(
         total_size = sum(e["size"] for e in manifest)
         num_files = len(manifest)
 
-        # copy_to_cache is idempotent and already checks existence itself; letting it see
-        # every entry (not just ones missing from the cache) is what lets it republish an
-        # object `pivot track --force` finds cached non-executable by an older pivot or a
-        # cold `pivot pull`, when this entry's bytes should be executable.
         for entry in manifest:
             file_path = path / entry["relpath"]
             file_cache_path = cache.get_cache_path(cache_dir, entry["hash"])

{pivot-0.3.4 → pivot-0.3.6}/src/pivot/exceptions.py

@@ -221,6 +221,8 @@ class CacheModeCollisionError(CacheError):
     This happens when two files with identical content disagree on their executable bit (the
     cache key is content-only) or when a cache object was populated by a path that does not
     know a file's intended executable bit (e.g. `pivot pull`, which downloads by hash alone).
+    Pivot never changes a published cache object, so only a private copy, or the object deleted
+    and published again from bytes that carry the right bit, can give the file its bit.
     """
 
     @override
@@ -228,10 +230,10 @@ class CacheModeCollisionError(CacheError):
         return (
             "Add CheckoutMode.COPY to checkout_modes (cache: checkout_mode: [..., copy] in "
             ".pivot/config.yaml) so checkout can fall back to a private, correctly-permissioned "
-            "copy instead of sharing the cache object's own permissions; or, if you have the "
-            "source bytes in hand, re-running what produced them (`pivot repro --force "
-            "<stage>`, `pivot track --force <path>`) republishes the cache object with the "
-            "right bit"
+            "copy instead of sharing the cache object's own permissions; or delete the cache "
+            "object the error names, then re-run what produced the file (`pivot repro --force "
+            "<stage>`, `pivot track --force <path>`) to publish it with the right bit; until it "
+            "is published again, any workspace file symlinked to it is broken"
         )
 
 

{pivot-0.3.4 → pivot-0.3.6}/src/pivot/executor/worker.py

@@ -251,7 +251,7 @@ def execute_stage(
                 lock_data = production_lock.read()
 
                 with state.StateDB(state_dir, readonly=True) as state_db:
-                    outputs_missing_from_cache = False
+                    outputs_unusable: str | None = None
                     if lock_data is not None and not stage_info["force"]:
                         out_paths = [types.identity_key(out.identity) for out in stage_outs]
                         if can_skip_via_generation(
@@ -272,15 +272,16 @@ def execute_stage(
                                 stage_outs,
                             )
                             restored_entries = list[tuple[str, int, int, int, str]]()
-                            restored = restore_outputs_to_hashes(
+                            restoration = restore_outputs_to_hashes(
                                 store,
                                 stage_outs,
                                 lock_data["output_hashes"],
                                 state_db,
                                 state_dir=project_state_dir,
+                                report_changes=True,
                                 file_hash_entries=restored_entries,
                             )
-                            if restored:
+                            if restoration.available:
                                 return _make_result(
                                     StageStatus.CACHED,
                                     "unchanged (generation)",
@@ -288,7 +289,7 @@ def execute_stage(
                                     input_hash=input_hash,
                                     file_hash_entries=restored_entries,
                                 )
-                            outputs_missing_from_cache = True
+                            outputs_unusable = _outputs_unusable_reason(restoration)
 
                     dep_hashes, missing, unreadable, file_hash_entries = hash_dependencies(
                         stage_info["deps"], store, state_db
@@ -316,9 +317,9 @@ def execute_stage(
                         dep_hashes,
                     )
 
-                    if outputs_missing_from_cache and skip_reason is not None:
+                    if outputs_unusable is not None and skip_reason is not None:
                         skip_reason = None
-                        run_reason = "outputs missing from cache"
+                        run_reason = outputs_unusable
 
                     # Override skip decision if force flag is set
                     if stage_info["force"] and skip_reason is not None:
@@ -327,15 +328,16 @@ def execute_stage(
 
                     if skip_reason is not None and lock_data is not None:
                         restored_entries = list[tuple[str, int, int, int, str]]()
-                        restored = restore_outputs_to_hashes(
+                        restoration = restore_outputs_to_hashes(
                             store,
                             stage_outs,
                             lock_data["output_hashes"],
                             state_db,
                             state_dir=project_state_dir,
+                            report_changes=True,
                             file_hash_entries=restored_entries,
                         )
-                        if restored:
+                        if restoration.available:
                             return _make_result(
                                 StageStatus.CACHED,
                                 skip_reason,
@@ -343,7 +345,7 @@ def execute_stage(
                                 input_hash=input_hash,
                                 file_hash_entries=restored_entries,
                             )
-                        run_reason = "outputs missing from cache"
+                        run_reason = _outputs_unusable_reason(restoration)
 
                     # Check run cache for previously executed configuration (skip if forcing)
                     if run_reason and not stage_info["force"] and run_cache_allowed:
@@ -494,13 +496,49 @@ class OutputRestoreResult(NamedTuple):
     """Whether output restoration succeeded and wrote workspace bytes.
 
     ``unavailable`` lists outputs that neither match their locked hash nor can be restored
-    from the cache. A check (``restore=False``) lists every such output; a restore stops at
-    the first, since the stage must run anyway.
+    from the cache. ``empty_dirs_unrecorded`` lists outputs that hold every file their locked
+    hash records plus empty directories it does not, as a hash pivot wrote before 0.3.4 cannot
+    record them (``unrecorded_empty_dirs_only``): restoring such an output would delete the
+    directories, so the stage runs instead. A check (``restore=False``) lists every such output;
+    a restore stops at the first, since the stage must run anyway.
     """
 
     available: bool
     outputs_restored: bool
     unavailable: list[types.ArtifactRef]
+    empty_dirs_unrecorded: list[types.ArtifactRef]
+
+
+def unrecorded_empty_dirs_only(expected: HashInfo, actual: HashInfo) -> bool:
+    """Whether directory output ``actual`` is ``expected``'s files plus empty directories that
+    ``expected`` cannot record.
+
+    Pivot before 0.3.4 hashed a directory by its files alone and recorded none of its empty
+    directories. Restoring an output to such a hash deletes the empty directories its stage
+    made, and the stages that read the output then hash the stripped tree to the old value and
+    stay cached, though a fresh build would hold the directories.
+    """
+    return (
+        is_dir_hash(expected)
+        and is_dir_hash(actual)
+        and "empty_dirs" not in expected
+        and "empty_dirs" in actual
+        and actual["manifest"] == expected["manifest"]
+    )
+
+
+def empty_dirs_unrecorded_reason(refs: list[types.ArtifactRef]) -> str:
+    """Say why a stage whose lock is current must run: ``refs`` are its outputs whose lock does
+    not record the empty directories they hold."""
+    keys = ", ".join(types.identity_key(ref.identity) for ref in refs)
+    return f"Output holds empty directories its lock does not record: {keys}"
+
+
+def _outputs_unusable_reason(restoration: OutputRestoreResult) -> str:
+    """The reason a stage whose lock is current runs because its outputs could not be used."""
+    if restoration.empty_dirs_unrecorded:
+        return empty_dirs_unrecorded_reason(restoration.empty_dirs_unrecorded)
+    return "outputs missing from cache"
 
 
 @overload
@@ -548,6 +586,7 @@ def restore_outputs_to_hashes(
     restored from the cache, so the next check finds it without reading it.
     """
     unavailable = list[types.ArtifactRef]()
+    empty_dirs_unrecorded = list[types.ArtifactRef]()
     outputs_restored = False
     if not isinstance(store, store_mod.WorkspaceStore):
         unavailable = [ref for ref in outs if not store.exists(ref)]
@@ -562,6 +601,13 @@ def restore_outputs_to_hashes(
                     actual_hash = None
                 if actual_hash == expected_hash:
                     continue
+                if actual_hash is not None and unrecorded_empty_dirs_only(
+                    expected_hash, actual_hash
+                ):
+                    empty_dirs_unrecorded.append(ref)
+                    if restore:
+                        break
+                    continue
                 if store.cache_contains(ref, expected_hash):
                     if not restore:
                         continue
@@ -575,13 +621,15 @@ def restore_outputs_to_hashes(
             unavailable.append(ref)
             if restore:
                 break
+    available = not unavailable and not empty_dirs_unrecorded
     if report_changes:
         return OutputRestoreResult(
-            available=not unavailable,
+            available=available,
             outputs_restored=outputs_restored,
             unavailable=unavailable,
+            empty_dirs_unrecorded=empty_dirs_unrecorded,
         )
-    return not unavailable
+    return available
 
 
 def _check_skip_or_run(

{pivot-0.3.4 → pivot-0.3.6}/src/pivot/explain.py

@@ -261,8 +261,9 @@ def get_stage_explanation(
             force=force,
         )
         unavailable = list[types.ArtifactRef]()
+        empty_dirs_unrecorded = list[types.ArtifactRef]()
         if not allow_missing:
-            unavailable = worker.restore_outputs_to_hashes(
+            restoration = worker.restore_outputs_to_hashes(
                 store,
                 outs,
                 lock_data["output_hashes"],
@@ -270,13 +271,19 @@ def get_stage_explanation(
                 state_dir=state_dir,
                 restore=False,
                 report_changes=True,
-            ).unavailable
-        if not decision["changed"] and unavailable:
+            )
+            unavailable = restoration.unavailable
+            empty_dirs_unrecorded = restoration.empty_dirs_unrecorded
+        if not decision["changed"] and (unavailable or empty_dirs_unrecorded):
             return StageExplanation(
                 stage_name=stage_name,
                 will_run=True,
                 is_forced=force,
-                reason=_unavailable_outputs_reason(store, unavailable),
+                reason=(
+                    _unavailable_outputs_reason(store, unavailable)
+                    if unavailable
+                    else worker.empty_dirs_unrecorded_reason(empty_dirs_unrecorded)
+                ),
                 code_changes=[],
                 param_changes=[],
                 dep_changes=[],

{pivot-0.3.4 → pivot-0.3.6}/src/pivot/remote/storage.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import asyncio
 import contextlib
-import errno
 import logging
 import os
 import re
@@ -13,6 +12,7 @@ import xxhash
 
 from pivot import config, exceptions, metrics
 from pivot.remote import config as remote_config
+from pivot.storage import cache
 from pivot.types import TransferResult
 
 if TYPE_CHECKING:
@@ -38,9 +38,6 @@ BULK_EXISTS_LIST_THRESHOLD = 50
 
 _HEX_PATTERN = re.compile(r"^[a-f0-9]{16}$")
 
-# link() errors on filesystems without hard links (vfat, exFAT, many SMB and FUSE mounts).
-_NO_HARDLINK_ERRNO = frozenset({errno.EPERM, errno.ENOTSUP, errno.EOPNOTSUPP})
-
 # Cached S3 config to avoid re-parsing config on every call.
 # Safe without locking - asyncio is single-threaded, worst case race just
 # creates duplicate config (harmless).
@@ -230,66 +227,61 @@ async def _atomic_download(
     authenticate remote content.
 
     Args:
-        overwrite: Replace an existing file at local_path. When False, an existing file is
-            kept: a content-addressed cache entry another process may be hardlinking.
+        overwrite: Replace an existing file at local_path. When False, local_path is a cache
+            object: an existing one is kept (another process may be hardlinking it), and the
+            download is staged so a killed one is swept (``cache.staged_cache_file``).
         readonly: If True, set file permissions to 0o444 (for cache files).
     """
-    local_path.parent.mkdir(parents=True, exist_ok=True)
-    fd, tmp_path = tempfile.mkstemp(dir=local_path.parent, prefix=".pivot_download_")
-    move_succeeded = False
-    try:
-        try:
-            response = await s3.get_object(Bucket=bucket, Key=key)
-        except Exception as exc:
-            error_str = str(exc)
-            if "NoSuchKey" in error_str or "404" in error_str:
-                raise exceptions.RemoteError(
-                    f"Remote cache file not found: {bucket}/{key}"
-                ) from exc
-            if "AccessDenied" in error_str or "403" in error_str:
-                raise exceptions.RemoteError(
-                    f"Access denied to {bucket}/{key}. Check AWS credentials."
-                ) from exc
-            raise exceptions.RemoteError(f"Failed to download from S3: {exc}") from exc
-        actual_hash = await _stream_download_to_fd(response, fd)
-        os.close(fd)
-        fd = -1  # Mark as closed
-        if actual_hash != expected_hash:
-            raise exceptions.RemoteError(
-                f"Hash mismatch: expected {expected_hash}, got {actual_hash} "
-                + f"for {bucket}/{key}. Delete {bucket}/{key} from the remote, then run "
-                + "`pivot push` on a machine whose cache has a good copy and has not run "
-                + "`pivot push`, `pivot pull`, `pivot fetch`, or `pivot status --remote` "
-                + "against this remote for this object. Pivot records hashes those commands "
-                + "upload, download, or find, then skips recorded hashes. If every machine "
-                + "with a good copy has recorded it, delete `.pivot/state.lmdb/` before "
-                + "pushing; this also clears run history and local caches."
-            )
+    with contextlib.ExitStack() as stack:
+        if overwrite:
+            local_path.parent.mkdir(parents=True, exist_ok=True)
+            fd, name = tempfile.mkstemp(dir=local_path.parent, prefix=".pivot_download_")
+            os.close(fd)
+            tmp_path = local_path.with_name(os.path.basename(name))
+            stack.callback(tmp_path.unlink, missing_ok=True)
+        else:
+            tmp_path = stack.enter_context(cache.staged_cache_file(local_path))
+        await _download_verified(s3, bucket, key, tmp_path, expected_hash)
         if readonly:
             os.chmod(tmp_path, 0o444)
         if overwrite:
             os.replace(tmp_path, local_path)
-            move_succeeded = True
         else:
-            # link() refuses an existing name, so a concurrent reader never sees the
-            # entry disappear. The temp file is removed below either way.
-            try:
-                os.link(tmp_path, local_path)
-            except FileExistsError:
-                pass
-            except OSError as exc:
-                if exc.errno not in _NO_HARDLINK_ERRNO:
-                    raise
-                # No hard links here, so nothing can be hardlinking the entry either:
-                # rename unless another process already promoted it.
-                if not os.path.exists(local_path):
-                    os.replace(tmp_path, local_path)
-                    move_succeeded = True
+            _ = cache.publish_cache_object(tmp_path, local_path)
+
+
+async def _download_verified(
+    s3: S3Client, bucket: str, key: str, dest: Path, expected_hash: str
+) -> None:
+    """Stream bucket/key into the existing file ``dest`` and check its xxh64 digest."""
+    try:
+        response = await s3.get_object(Bucket=bucket, Key=key)
+    except Exception as exc:
+        error_str = str(exc)
+        if "NoSuchKey" in error_str or "404" in error_str:
+            raise exceptions.RemoteError(f"Remote cache file not found: {bucket}/{key}") from exc
+        if "AccessDenied" in error_str or "403" in error_str:
+            raise exceptions.RemoteError(
+                f"Access denied to {bucket}/{key}. Check AWS credentials."
+            ) from exc
+        raise exceptions.RemoteError(f"Failed to download from S3: {exc}") from exc
+    fd = os.open(dest, os.O_WRONLY | os.O_TRUNC)
+    try:
+        actual_hash = await _stream_download_to_fd(response, fd)
     finally:
-        if fd >= 0:
+        with contextlib.suppress(OSError):
             os.close(fd)
-        if not move_succeeded and os.path.exists(tmp_path):
-            os.unlink(tmp_path)
+    if actual_hash != expected_hash:
+        raise exceptions.RemoteError(
+            f"Hash mismatch: expected {expected_hash}, got {actual_hash} "
+            + f"for {bucket}/{key}. Delete {bucket}/{key} from the remote, then run "
+            + "`pivot push` on a machine whose cache has a good copy and has not run "
+            + "`pivot push`, `pivot pull`, `pivot fetch`, or `pivot status --remote` "
+            + "against this remote for this object. Pivot records hashes those commands "
+            + "upload, download, or find, then skips recorded hashes. If every machine "
+            + "with a good copy has recorded it, delete `.pivot/state.lmdb/` before "
+            + "pushing; this also clears run history and local caches."
+        )
 
 
 async def _stream_upload(

{pivot-0.3.4 → pivot-0.3.6}/src/pivot/storage/AGENTS.md

@@ -66,6 +66,48 @@ When multiple `pivot` processes share the same StateDB:
   temporary directory and swap it. Both run under the target's restore lock, and each
   first removes the token-tagged entries a killed restore of the same target left.
 
+## Publishing Cache Objects
+
+- Every writer of a file object publishes it with `cache.publish_cache_object`, which links it
+  into place and keeps an object already there. A published object never changes, mode
+  included: a hardlink checkout looks the name up, then links the inode it found, and once a
+  rename takes that inode's last name Linux fails the link with ENOENT, though the name never
+  went missing; a symlink checkout reads its executable bit through the name. A directory
+  entry whose executable bit the object does not carry is checked out as a copy.
+- On a filesystem without hard links, `publish_cache_object` renames, and only when no object is
+  there yet. Nothing there can hardlink an object, but the check and the rename are two steps:
+  two writers that both find the name free both rename, the later object wins, and a symlink
+  checkout reads the later object's mode.
+- Every temporary entry in a prefix directory (a file staged for an object, a download, a tree
+  being copied) is made through `cache.staged_cache_file` or `_staging_in`, named
+  `.pivot_tmp_*`, while the process holds a shared `flock` on the directory's
+  `.pivot_cache.lock`. A process that gets the exclusive lock first (it never waits for it), at
+  most once per `_CACHE_SWEEP_INTERVAL` (the lock's mtime records the last sweep), removes every
+  `.pivot_tmp_*` entry: none has a live owner. An entry it cannot remove is logged and left.
+  Never judge a temporary entry abandoned by host and pid: a process in another PID namespace,
+  or on another host sharing the cache, looks dead.
+- Sharing a cache across hosts needs `flock` that is coherent across them (NFSv4, or NFSv3 with
+  NLM). With host-local locks (NFS `nolock` or `local_lock=flock`/`all`, Lustre `localflock`)
+  the locks here exclude nothing across hosts: one host's sweep can remove another host's live
+  temporary entry, and two hosts can repair one tree at once. A publisher checks its tree before
+  marking it, which narrows the window for marking a copy that lost files without closing it: a
+  sweep already deleting through the copy's directory descriptors keeps deleting after the
+  rename. Where `flock()` is refused (Lustre without `flock`: ENOSYS; NFS when its lock manager
+  does not answer: ENOLCK) staging goes on without the shared lock, and this process neither
+  sweeps nor repairs a tree. Only where every host sharing the cache is refused does nothing
+  sweep: an NFSv3 client whose own `rpc.statd` is down gets ENOLCK while other clients' locks
+  work, and their sweeps can remove its live temporary entries, with the effects above.
+- A lock file in the cache is created writable by whoever can write its directory, since NFS
+  takes an exclusive lock only on a file opened for writing; one this user cannot write is
+  opened read-only. Lock files are never deleted.
+- A whole-directory SYMLINK tree is built under a temporary name beside its cache name, renamed
+  into place, checked, then marked (`<tree>.verified`), because a restore symlinks to a tree as
+  soon as its name exists. A restore or commit checks an unmarked tree (exact files, sizes,
+  executable bits and directories) under the tree's exclusive `<tree>.lock`, repairs it in place
+  from the files' own cache objects, then marks it. A tree it cannot repair is not used: a
+  restore goes file by file, a commit fails. `<tree>.layout-checked`, the mark pivot 0.3.4 also
+  gave partial trees, is not trusted.
+
 ## Path Storage
 
 All paths stored in the database must be **relative** to ensure portability across machines and correct cache behavior.