pivot 0.2.4__tar.gz → 0.3.0__tar.gz

{pivot-0.2.4 → pivot-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pivot
-Version: 0.2.4
+Version: 0.3.0
 Summary: High-performance Python pipeline tool with automatic code change detection
 Author: Sami Jawhar
 Author-email: Sami Jawhar <sami@metr.org>

{pivot-0.2.4 → pivot-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pivot"
-version = "0.2.4"
+version = "0.3.0"
 description = "High-performance Python pipeline tool with automatic code change detection"
 requires-python = ">=3.13,<3.14"
 readme = "README.md"

{pivot-0.2.4 → pivot-0.3.0}/src/pivot/cli/checkout.py

@@ -116,7 +116,6 @@ def _restore_path_locked(
     lock_token: str | None,
 ) -> tuple[RestoreResult, str]:
     """Body of _restore_path_sync; the caller holds the restore lock for path."""
-    keep_ignored = False
     if path.exists():
         match behavior:
             case CheckoutBehavior.ERROR:
@@ -127,10 +126,9 @@ def _restore_path_locked(
                     + "Use --force to overwrite or --only-missing to skip existing files."
                 )
             case CheckoutBehavior.SKIP_EXISTING:
-                # Existing file: skip. Existing real directory: fill in its missing files, or
-                # swap in the target when every differing file is pivot's own cached output,
-                # keeping the entries the directory hash skips. Never overwrite content the
-                # cache lacks (a local change).
+                # Existing real directories can fill missing files in place, or swap to the
+                # target when every differing or extra file is pivot's own cached output.
+                # Never overwrite content the cache lacks (a local change).
                 if not is_dir_hash(output_hash) or path.is_symlink() or not path.is_dir():
                     return ("skipped", path.name)
                 # Reuse hashes `pivot status` and `pivot track` recorded; never create the DB.
@@ -147,13 +145,10 @@ def _restore_path_locked(
                         f"'{path.name}' has local changes that are not in the cache: "
                         + f"{', '.join(outcome['local_changes'])}. Use --force to overwrite."
                     )
-                # An interrupted restore may have staged kept entries beside the directory;
-                # the restore below puts them back, so take it even when nothing else differs.
-                if not outcome["replace"] and cache.staged_kept_entries(path) is None:
+                if not outcome["replace"]:
                     if outcome["unavailable"]:
                         return ("missing", path.name)
                     return ("restored" if outcome["restored"] else "skipped", path.name)
-                keep_ignored = True
             case CheckoutBehavior.FORCE:
                 cache.remove_output(path)
             case _:  # pyright: ignore[reportUnnecessaryComparison] - defensive for future enum values
@@ -166,7 +161,6 @@ def _restore_path_locked(
         checkout_modes=checkout_modes,
         state_dir=state_dir,
         lock_token=lock_token,
-        keep_ignored=keep_ignored,
     )
     if not success:
         return ("missing", path.name)

{pivot-0.2.4 → pivot-0.3.0}/src/pivot/fingerprint.py

@@ -11,6 +11,7 @@ import logging
 import marshal
 import os
 import pathlib
+import re
 import sys
 import textwrap
 import types
@@ -60,9 +61,12 @@ _PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
 # `ext:`), not `mod:`, and follows `settings.Config.SCALE` and `pkg.sub.helper`.
 # Version 25 records call-site artifact wiring (parameter bindings, collection kind/order,
 # and project-relative input paths) in the code manifest.
-_CACHE_SCHEMA_VERSION = 25
+# Version 26 records tuples and frozensets of scalars and compiled regexes by value.
+_CACHE_SCHEMA_VERSION = 26
 _REPR_SIZE_LIMIT = 10_000
 
+_SCALAR_TYPES = (bool, int, float, str, bytes, type(None))
+
 _SITE_PACKAGE_PATHS = ("site-packages", "dist-packages")
 
 # Type alias for AST hash cache entry tuple.
@@ -563,6 +567,24 @@ class _ValueKind(enum.Enum):
     SCALAR = "scalar"
 
 
+def _immutable_value_text(value: Any) -> str | None:
+    """Return the manifest text for a safely trackable immutable value, if any.
+
+    Scalars keep their existing repr. Exact tuples and frozensets are tracked only when every
+    member is an exact scalar type, one level deep. Compiled regexes record their full pattern
+    and flags because their repr truncates long patterns.
+    """
+    if isinstance(value, _SCALAR_TYPES):
+        return repr(value)
+    if type(value) in (tuple, frozenset) and all(
+        type(item) in _SCALAR_TYPES for item in cast("tuple[Any, ...] | frozenset[Any]", value)
+    ):
+        return _serialize_value_for_hash(value)
+    if isinstance(value, re.Pattern):
+        return _serialize_value_for_hash(value)
+    return None
+
+
 def _value_kind(value: Any) -> _ValueKind:
     """Classify a value a walked function reads; the walk and a cache hit share this."""
     # functools.partial fails is_user_code() (module is functools/stdlib).
@@ -576,7 +598,7 @@ def _value_kind(value: Any) -> _ValueKind:
         return _ValueKind.MODULE
     if isinstance(value, logging.Logger):
         return _ValueKind.LOGGER
-    if isinstance(value, (bool, int, float, str, bytes, type(None))):
+    if _immutable_value_text(value) is not None:
         return _ValueKind.SCALAR
     return _ValueKind.REFUSED
 
@@ -598,7 +620,7 @@ def _value_token(value: Any) -> str:
     if kind is _ValueKind.MODULE:
         return f"module:{value.__name__}"
     if kind is _ValueKind.SCALAR:
-        return f"{type(value).__name__}:{value!r}"
+        return f"{type(value).__name__}:{_immutable_value_text(value)}"
     if kind is _ValueKind.EXTERNAL:
         return f"external:{_library_name(value)}"
     return kind.value
@@ -1335,9 +1357,9 @@ def _check_non_scalar_capture(
         subject += f" as the default of parameter '{parameter}'"
     message = (
         f"Stage '{stage_name}': {subject} (type: {type(value).__name__}).\n"
-        + "Pivot tracks only module-level bool, int, float, str, bytes, and None values. "
-        + "Other values may change without a reliable fingerprint, which can cause silent wrong "
-        + "outputs.\n"
+        + "Pivot tracks module-level bool, int, float, str, bytes, and None values, tuples and "
+        + "frozensets of them, and compiled regexes. Other values may change without a reliable "
+        + "fingerprint, which can cause silent wrong outputs.\n"
         + "Fix: move this value into the stage's StageParams, or read it from a pipeline input.\n"
         + "To suppress: set core.unsafe_fingerprinting=true or PIVOT_UNSAFE_FINGERPRINTING=1"
     )
@@ -1548,7 +1570,9 @@ def _process_captured_value(
             if chain is not None:
                 _process_module_dependency(chain, value, func, manifest, visited)
         case _ValueKind.SCALAR:
-            manifest[f"const:{binding_ref}"] = repr(value)
+            text = _immutable_value_text(value)
+            assert text is not None
+            manifest[f"const:{binding_ref}"] = text
         case _ValueKind.REFUSED:
             _check_non_scalar_capture(name, value, stage_name, parameter=parameter)
         case _ValueKind.EXTERNAL:
@@ -1681,7 +1705,7 @@ def _process_default_values(
         binding_ref = f"{func_ref}.{parameter}"
         step = f"d:{parameter}"
         _record_read(func, step)
-        if isinstance(value, (bool, int, float, str, bytes, type(None))):
+        if isinstance(value, _SCALAR_TYPES):
             manifest[f"default:{binding_ref}"] = repr(value)
             continue
         if isinstance(value, stage_def.StageParams):
@@ -1695,6 +1719,10 @@ def _process_default_values(
         expression = expressions.get(parameter)
         if expression is not None and _is_literal(expression):
             continue
+        text = _immutable_value_text(value)
+        if text is not None:
+            manifest[f"default:{binding_ref}"] = text
+            continue
         source = ast.unparse(expression) if expression is not None else parameter
         with _walking(step):
             _process_captured_value(
@@ -2140,12 +2168,16 @@ def _canonicalize_value_for_hash(value: Any, active: tuple[int, ...] = ()) -> An
     canonical({field: value})]`, field by field, so a secret inside one reaches that leaf
     rather than the generated repr's mask. An object whose class keeps `object.__repr__`
     (which holds the address) and that has a `__dict__` becomes
-    `[module.qualname, canonical(vars(obj))]`.
-    Other leaves keep their `repr`, tagged with their type. A class is a leaf, even a pydantic
-    model class: only an instance is dumped.
+    `[module.qualname, canonical(vars(obj))]`. A compiled regex becomes
+    `["re.Pattern", canonical(pattern), flags]`, so its full pattern is recorded. Other leaves
+    keep their `repr`, tagged with their type. A class is a leaf, even a pydantic model class:
+    only an instance is dumped.
     """
     if value is None or isinstance(value, (bool, int, float, str)):
         return value
+    if isinstance(value, re.Pattern):
+        pattern = cast("re.Pattern[Any]", value)
+        return ["re.Pattern", _canonicalize_value_for_hash(pattern.pattern), pattern.flags]
     if id(value) in active:
         return ["cycle", len(active) - active.index(id(value))]
     if hasattr(value, "model_dump") and not isinstance(value, type):

{pivot-0.2.4 → pivot-0.3.0}/src/pivot/storage/cache.py

@@ -36,7 +36,6 @@ XXHASH64_HEX_LENGTH = 16  # xxhash64 produces 64-bit hash = 16 hex characters
 _RESTORE_TEMP_PREFIX = ".pivot_restore_"
 _BACKUP_TEMP_PREFIX = ".pivot_backup_"
 _RESTORE_LOCK_PREFIX = ".pivot_restore_lock_"
-_KEEP_STAGING_PREFIX = ".pivot_keep_"
 _RESTORE_TEMP_MAX_AGE = 3600  # 1 hour
 
 
@@ -46,36 +45,6 @@ def _restore_target_token(target_path: pathlib.Path) -> str:
     return hashlib.sha256(str(logical_path).encode()).hexdigest()[:32]
 
 
-def _keep_staging_path(path: pathlib.Path) -> pathlib.Path:
-    """Sibling that holds a directory's kept entries while it is replaced."""
-    return path.parent / f"{_KEEP_STAGING_PREFIX}{_restore_target_token(path)}"
-
-
-def _move_kept_entries(source: pathlib.Path, destination: pathlib.Path) -> None:
-    """Move every entry under source into destination, merging directories, then drop source."""
-    with os.scandir(source) as entries:
-        for entry in entries:
-            target = destination / entry.name
-            if not os.path.lexists(target):
-                destination.mkdir(parents=True, exist_ok=True)
-                pathlib.Path(entry.path).rename(target)
-            elif (
-                entry.is_dir(follow_symlinks=False) and target.is_dir() and not target.is_symlink()
-            ):
-                _move_kept_entries(pathlib.Path(entry.path), target)
-            else:
-                raise exceptions.CacheRestoreError(
-                    f"Cannot put back {entry.path}: {target}. Move the entry by hand, then retry."
-                )
-    source.rmdir()
-
-
-def staged_kept_entries(path: pathlib.Path) -> pathlib.Path | None:
-    """Return the staging directory an interrupted keep-ignored restore left, if any."""
-    keep_dir = _keep_staging_path(path)
-    return keep_dir if keep_dir.exists() else None
-
-
 def _restore_staging_path(path: pathlib.Path, token: str | None = None) -> pathlib.Path:
     """Return the deterministic staging path for a file checkout."""
     return path.parent / f"{_RESTORE_TEMP_PREFIX}{token or _restore_target_token(path)}.file"
@@ -190,39 +159,12 @@ def hash_file(
     return file_hash, file_stat
 
 
-# Hardcoded ignore patterns for hot path - O(1) lookups only.
-# For full .pivotignore support, see pivot.ignore module.
-_IGNORE_DIRS: frozenset[str] = frozenset(
-    {
-        "__pycache__",
-        ".venv",
-        "venv",
-        ".git",
-        ".hg",
-        ".svn",
-        ".idea",
-        ".vscode",
-        "node_modules",
-        ".pivot",
-        "dist",
-        "build",
-    }
-)
-_IGNORE_SUFFIXES: tuple[str, ...] = (".pyc", ".pyo", ".swp", ".swo")
-
-
-def _should_skip_entry(entry: os.DirEntry[str]) -> bool:
-    """Fast ignore check for hot path. O(1) lookups only."""
-    name = entry.name
-    if entry.is_dir(follow_symlinks=False):
-        return name in _IGNORE_DIRS
-    # File checks: suffix and special patterns
-    return name.endswith(_IGNORE_SUFFIXES) or name.endswith("~") or name.startswith(".#")
-
-
 def _scandir_recursive(path: pathlib.Path) -> Generator[os.DirEntry[str]]:
-    """Yield all files recursively using os.scandir() for efficiency.
+    """Yield every file under `path` recursively, using os.scandir() for efficiency.
 
+    A declared directory artifact is hashed and cached whole: no name is ignored, so a change
+    to any file in a directory input reruns its consumers, and every file in a directory output
+    survives cache and restore. Ignore rules belong to code fingerprinting and watch mode.
     DirEntry objects cache stat results, avoiding redundant syscalls.
     Symlinks are skipped to prevent loops.
     PermissionError propagates to caller - partial hashes would be incorrect.
@@ -234,8 +176,6 @@ def _scandir_recursive(path: pathlib.Path) -> Generator[os.DirEntry[str]]:
         for entry in entries:
             if entry.is_symlink():
                 continue
-            if _should_skip_entry(entry):
-                continue
             if entry.is_file():
                 yield entry
             elif entry.is_dir():
@@ -773,7 +713,6 @@ def restore_from_cache(
     state_dir: pathlib.Path | None = None,
     *,
     lock_token: str | None = None,
-    keep_ignored: bool = False,
 ) -> bool:
     """Restore file or directory from cache. Returns True if successful.
 
@@ -785,8 +724,6 @@ def restore_from_cache(
         checkout_modes: Ordered list of link modes to try with fallback on failure.
         state_dir: State directory for restore locks and crash cleanup.
         lock_token: Target token yielded by an already-held restore_lock for path.
-        keep_ignored: When replacing an existing directory, preserve the entries the directory
-            hash skips in the restored tree.
     """
     effective_modes = _resolve_checkout_modes(checkout_mode, checkout_modes)
     if lock_token is not None and state_dir is None:
@@ -800,7 +737,6 @@ def restore_from_cache(
             effective_modes,
             state_dir,
             lock_token=lock_token,
-            keep_ignored=keep_ignored,
         )
     return _restore_file_from_cache(
         path, output_hash, cache_dir, effective_modes, state_dir, lock_token=lock_token
@@ -855,9 +791,8 @@ def _restore_directory_from_cache(
     state_dir: pathlib.Path | None = None,
     *,
     lock_token: str | None = None,
-    keep_ignored: bool = False,
 ) -> bool:
-    """Restore a directory atomically, optionally carrying ignored local entries forward."""
+    """Restore a directory atomically."""
     cache_dir_path = get_cache_path(cache_dir, output_hash["hash"])
     symlink_fast_path = checkout_modes[0] == CheckoutMode.SYMLINK and cache_dir_path.exists()
     if not symlink_fast_path:
@@ -877,24 +812,12 @@ def _restore_directory_from_cache(
                 checkout_modes,
                 state_dir,
                 lock_token=held_token,
-                keep_ignored=keep_ignored,
             )
 
     target_token = lock_token or _restore_target_token(path)
     if lock_token is not None:
         _cleanup_fresh_restore_orphans(path, target_token)
     restore_prefix = f"{_RESTORE_TEMP_PREFIX}{target_token}."
-    keep_dir = _keep_staging_path(path)
-    keep = list[str]()
-    if (keep_ignored or keep_dir.exists()) and path.is_dir() and not path.is_symlink():
-        if keep_dir.exists():
-            _move_kept_entries(keep_dir, path)
-        _, keep = _scan_entries(path, {entry["relpath"] for entry in output_hash["manifest"]})
-    if symlink_fast_path and keep:
-        symlink_fast_path = False
-        for entry in output_hash["manifest"]:
-            if not get_cache_path(cache_dir, entry["hash"]).exists():
-                return False
 
     if symlink_fast_path:
         temp_symlink = path.parent / f"{restore_prefix}symlink_{secrets.token_hex(4)}"
@@ -948,14 +871,6 @@ def _restore_directory_from_cache(
             except OSError as e:
                 logger.debug(f"chmod failed (may be NFS with root_squash): {e}")
 
-        for relpath in keep:
-            source = path / relpath
-            if not os.path.lexists(source):
-                continue
-            staged = keep_dir / relpath
-            staged.parent.mkdir(parents=True, exist_ok=True)
-            source.rename(staged)
-
         try:
             if path.exists() or path.is_symlink():
                 path.rename(backup_path)
@@ -964,8 +879,6 @@ def _restore_directory_from_cache(
 
         temp_path.replace(path)
         swap_completed = True
-        if keep_dir.exists():
-            _move_kept_entries(keep_dir, path)
 
         try:
             _clear_path(backup_path)
@@ -979,14 +892,6 @@ def _restore_directory_from_cache(
             except OSError as e:
                 logger.debug(f"Failed to restore backup: {e}")
                 backup_restore_failed = True
-        if not swap_completed and keep_dir.exists() and path.is_dir() and not path.is_symlink():
-            try:
-                _move_kept_entries(keep_dir, path)
-            except (OSError, exceptions.CacheRestoreError) as e:
-                logger.warning(
-                    f"Entries kept from {path} remain in {keep_dir} ({e}); "
-                    + "the next restore of it puts them back"
-                )
         raise
     finally:
         if temp_path.exists():
@@ -1017,39 +922,25 @@ def _is_cached(cache_dir: pathlib.Path, content_hash: str | None) -> bool:
     )
 
 
-def _scan_entries(path: pathlib.Path, expected_paths: set[str]) -> tuple[list[str], list[str]]:
-    """Return (files and symlinks, entries the directory hash skips) under path, as relpaths.
+def _scan_entries(path: pathlib.Path) -> list[str]:
+    """Return every file and symlink under ``path`` as relative paths.
 
-    Symlinked directories are not followed. A skipped name that is, or holds, a manifest
-    path is part of the output and is scanned like any other entry.
+    Symlinked directories are not followed.
     """
-    expected_dirs = {
-        str(parent)
-        for relpath in expected_paths
-        for parent in pathlib.PurePosixPath(relpath).parents
-        if str(parent) != "."
-    }
     files = list[str]()
-    ignored = list[str]()
 
     def _walk(directory: pathlib.Path) -> None:
         with os.scandir(directory) as entries:
             for entry in entries:
                 entry_path = pathlib.Path(entry.path)
                 relpath = str(entry_path.relative_to(path))
-                if (
-                    _should_skip_entry(entry)
-                    and relpath not in expected_paths
-                    and relpath not in expected_dirs
-                ):
-                    ignored.append(relpath)
-                elif entry.is_symlink() or entry.is_file():
+                if entry.is_symlink() or entry.is_file():
                     files.append(relpath)
                 elif entry.is_dir():
                     _walk(entry_path)
 
     _walk(path)
-    return files, ignored
+    return files
 
 
 def _is_same_file(file_stat: os.stat_result, cache_path: pathlib.Path) -> bool:
@@ -1069,15 +960,12 @@ def restore_missing_in_directory(
     checkout_modes: list[CheckoutMode],
     state_db: state_mod.StateDB | None = None,
 ) -> MissingRestore:
-    """Restore a directory's missing target files without overwriting local changes.
+    """Restore missing target files without overwriting local changes.
 
-    When every existing file matches the target, the missing files are restored in place
-    and nothing else is touched. When some file differs from the target or is not in it,
-    only a whole-directory swap brings the target. It is safe only if every file it
-    removes is in the cache (pivot's own earlier output): then nothing is written and
-    ``replace`` asks the caller to swap with ``restore_from_cache(..., keep_ignored=True)``,
-    which carries over the entries the directory hash skips (build/, *.swp, ...). Otherwise
-    nothing is written, and the files the cache lacks (local edits, extra files) are returned.
+    When every existing file matches the target, missing files are restored in place. When a
+    file differs from the target or an extra file exists, only a whole-directory swap can
+    restore the target. The swap is safe only when every file it removes is in the cache;
+    otherwise the local changes are returned without writing anything.
 
     A retained file is read only when nothing else proves its content: a symlink to the
     target's cache path, a hardlink to the target's cache blob, or a matching ``state_db``
@@ -1128,7 +1016,7 @@ def restore_missing_in_directory(
             )
         missing.append((entry, file_path, file_cache_path))
 
-    files, _ = _scan_entries(path, expected_paths)
+    files = _scan_entries(path)
     for relpath in files:
         if relpath in expected_paths:
             continue