pivot 0.2.0__tar.gz → 0.2.2__tar.gz

{pivot-0.2.0 → pivot-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pivot
-Version: 0.2.0
+Version: 0.2.2
 Summary: High-performance Python pipeline tool with automatic code change detection
 Author: Sami Jawhar
 Author-email: Sami Jawhar <sami@metr.org>

{pivot-0.2.0 → pivot-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pivot"
-version = "0.2.0"
+version = "0.2.2"
 description = "High-performance Python pipeline tool with automatic code change detection"
 requires-python = ">=3.13,<3.14"
 readme = "README.md"

{pivot-0.2.0 → pivot-0.2.2}/src/pivot/cli/checkout.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import enum
 import pathlib
 from typing import TYPE_CHECKING, Literal
@@ -12,7 +13,7 @@ from pivot.cli import completion
 from pivot.cli import decorators as cli_decorators
 from pivot.cli import helpers as cli_helpers
 from pivot.cli import targets as cli_targets
-from pivot.storage import cache, lock, track
+from pivot.storage import cache, lock, state, track
 from pivot.types import HashInfo, is_dir_hash
 
 if TYPE_CHECKING:
@@ -96,6 +97,7 @@ def _restore_path_sync(
             "already exists" without --force). Cache misses return ("missing", name)
             instead of raising.
     """
+    keep_ignored = False
     if path.exists():
         match behavior:
             case CheckoutBehavior.ERROR:
@@ -106,12 +108,33 @@ def _restore_path_sync(
                     + "Use --force to overwrite or --only-missing to skip existing files."
                 )
             case CheckoutBehavior.SKIP_EXISTING:
-                # For directories with manifests, don't skip - files inside may be missing.
-                # Let restore_from_cache() handle it (does full directory restoration).
-                # DirHash has "manifest" key, FileHash does not.
-                is_directory = is_dir_hash(output_hash)
-                if not is_directory:
+                # Existing file: skip. Existing real directory: fill in its missing files, or
+                # swap in the target when every differing file is pivot's own cached output,
+                # keeping the entries the directory hash skips. Never overwrite content the
+                # cache lacks (a local change).
+                if not is_dir_hash(output_hash) or path.is_symlink() or not path.is_dir():
                     return ("skipped", path.name)
+                # Reuse hashes `pivot status` and `pivot track` recorded; never create the DB.
+                with (
+                    state.StateDB(state_dir, readonly=True)
+                    if state_dir is not None and (state_dir / "state.lmdb").exists()
+                    else contextlib.nullcontext(None)
+                ) as state_db:
+                    outcome = cache.restore_missing_in_directory(
+                        path, output_hash, cache_dir, checkout_modes, state_db
+                    )
+                if outcome["local_changes"]:
+                    raise click.ClickException(
+                        f"'{path.name}' has local changes that are not in the cache: "
+                        + f"{', '.join(outcome['local_changes'])}. Use --force to overwrite."
+                    )
+                # An interrupted restore may have staged kept entries beside the directory;
+                # the restore below puts them back, so take it even when nothing else differs.
+                if not outcome["replace"] and cache.staged_kept_entries(path) is None:
+                    if outcome["unavailable"]:
+                        return ("missing", path.name)
+                    return ("restored" if outcome["restored"] else "skipped", path.name)
+                keep_ignored = True
             case CheckoutBehavior.FORCE:
                 cache.remove_output(path)
             case _:  # pyright: ignore[reportUnnecessaryComparison] - defensive for future enum values
@@ -123,6 +146,7 @@ def _restore_path_sync(
         cache_dir,
         checkout_modes=checkout_modes,
         state_dir=state_dir,
+        keep_ignored=keep_ignored,
     )
     if not success:
         return ("missing", path.name)

{pivot-0.2.0 → pivot-0.2.2}/src/pivot/cli/verify.py

@@ -190,6 +190,29 @@ def _create_remote_if_needed(allow_missing: bool) -> remote_mod.S3Remote | None:
     return remote
 
 
+def _status_from_explanations(
+    explanations: list[types.StageExplanation],
+) -> list[PipelineStatusInfo]:
+    """Convert rich explanations to verification status, retaining changed dep identities."""
+    statuses = list[PipelineStatusInfo]()
+    for explanation in explanations:
+        reason = explanation["reason"]
+        if reason == "Input dependencies changed" and explanation["dep_changes"]:
+            changed = ", ".join(
+                types.identity_key(change["identity"]) for change in explanation["dep_changes"]
+            )
+            reason = f"{reason}: {changed}"
+        statuses.append(
+            PipelineStatusInfo(
+                name=explanation["stage_name"],
+                status=(PipelineStatus.STALE if explanation["will_run"] else PipelineStatus.CACHED),
+                reason=reason,
+                upstream_stale=explanation["upstream_stale"],
+            )
+        )
+    return statuses
+
+
 def _verify_stages(
     pipeline_status: list[PipelineStatusInfo],
     cache_dir: pathlib.Path,
@@ -345,14 +368,16 @@ def verify(
 
     cache_dir = config.get_cache_dir()
 
-    # Get pipeline status (uses default state directory internally)
-    pipeline_status, _ = status_mod.get_pipeline_status(
+    # Get rich explanations so lock-only dependency mismatches identify the
+    # producer artifact in verification output.
+    explanations = status_mod.get_pipeline_explanations(
         stages_list,
         single_stage=False,
         all_stages=all_stages,
         pipeline=cli_helpers.get_pipeline(),
         allow_missing=allow_missing,
     )
+    pipeline_status = _status_from_explanations(explanations)
 
     if not pipeline_status:
         raise click.ClickException("No stages to verify.")

{pivot-0.2.0 → pivot-0.2.2}/src/pivot/engine/sinks.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import dataclasses
 import sys
 import time
 from typing import TYPE_CHECKING
@@ -10,8 +11,8 @@ import rich.live
 import rich.markup
 import rich.text
 
-from pivot.engine.types import StageCompleted
-from pivot.types import DisplayCategory, categorize_stage_result
+from pivot.engine.types import LogLine, StageCompleted
+from pivot.types import LOSSY_OUTPUT_PHRASE, DisplayCategory, categorize_stage_result
 
 if TYPE_CHECKING:
     from pivot.engine.types import OutputEvent
@@ -84,6 +85,20 @@ def _categorize(event: StageCompleted) -> DisplayCategory:
     return categorize_stage_result(event["status"])
 
 
+@dataclasses.dataclass
+class _StageWarnings:
+    """A stage's ``WARNING:`` lines for the report.
+
+    Pivot's own output-write warnings are always kept and print first. The stage's own
+    warnings are counted per distinct line (in first-seen order); the report shows the
+    first ``_MAX_LOG_LINES_PER_STAGE`` distinct lines and counts the rest.
+    """
+
+    stage: dict[str, int] = dataclasses.field(default_factory=dict[str, int])
+    lossy_outputs: list[str] = dataclasses.field(default_factory=list[str])
+    dropped: int = 0
+
+
 def _print_completions(
     console: rich.console.Console,
     events: list[StageCompleted],
@@ -91,13 +106,19 @@ def _print_completions(
     total: int,
     max_name_width: int,
     stage_logs: dict[str, list[str]] | None = None,
+    stage_warnings: dict[str, _StageWarnings] | None = None,
 ) -> None:
-    """Print buffered completions sorted by index with skip collapsing."""
+    """Print buffered completions sorted by index with skip collapsing.
+
+    A failed stage shows its captured output; any other stage shows the ``WARNING:``
+    lines in ``stage_warnings`` (pass None when those already streamed).
+    """
     if not events:
         return
 
     sorted_events = sorted(events, key=lambda e: e["index"])
     logs = stage_logs or {}
+    warnings_by_stage = stage_warnings or {}
 
     i = 0
     while i < len(sorted_events):
@@ -152,12 +173,53 @@ def _print_completions(
                 if event["reason"]:
                     for detail in _format_error_detail(event["reason"], total=total):
                         console.print(detail)
+            elif event["stage"] in warnings_by_stage:
+                buffered = warnings_by_stage[event["stage"]]
+                stage_lines = [
+                    line if count == 1 else f"{line} (×{count})"
+                    for line, count in buffered.stage.items()
+                ]
+                for warning_line in [*buffered.lossy_outputs, *stage_lines]:
+                    escaped = rich.markup.escape(warning_line)
+                    console.print(f"          [yellow]{escaped}[/yellow]")
+                if buffered.dropped:
+                    console.print(
+                        f"          [yellow]… {buffered.dropped} more warnings "
+                        + "(run with --show-output)[/yellow]"
+                    )
             i += 1
 
 
 _MAX_LOG_LINES_PER_STAGE = 50
 
 
+def _buffer_log_line(
+    event: LogLine,
+    stage_logs: dict[str, list[str]],
+    stage_warnings: dict[str, _StageWarnings],
+) -> None:
+    """Keep a stage's output for a failure report and its warnings for every report.
+
+    Warnings have their own buffer: output writes log them after the stage's own output,
+    which can already fill the log buffer.
+    """
+    line = event["line"]
+    buf = stage_logs.setdefault(event["stage"], [])
+    if len(buf) < _MAX_LOG_LINES_PER_STAGE:
+        buf.append(line)
+    if not line.startswith("WARNING:"):
+        return
+    buffered = stage_warnings.setdefault(event["stage"], _StageWarnings())
+    if LOSSY_OUTPUT_PHRASE in line and line.startswith("WARNING: Stage '"):
+        buffered.lossy_outputs.append(line)
+    elif line in buffered.stage:
+        buffered.stage[line] += 1
+    elif len(buffered.stage) < _MAX_LOG_LINES_PER_STAGE:
+        buffered.stage[line] = 1
+    else:
+        buffered.dropped += 1
+
+
 class StaticConsoleSink:
     """Pipe/CI sink: buffers completions, prints a sorted report when each run ends."""
 
@@ -166,6 +228,7 @@ class StaticConsoleSink:
     _max_name_width: int
     _completion_buffer: list[StageCompleted]
     _stage_logs: dict[str, list[str]]
+    _stage_warnings: dict[str, _StageWarnings]
     _total: int
 
     def __init__(self, *, console: rich.console.Console, show_output: bool = False) -> None:
@@ -174,6 +237,7 @@ class StaticConsoleSink:
         self._max_name_width = 0
         self._completion_buffer = list[StageCompleted]()
         self._stage_logs = dict[str, list[str]]()
+        self._stage_warnings = dict[str, _StageWarnings]()
         self._total = 0
 
     async def handle(self, event: OutputEvent) -> None:
@@ -187,11 +251,9 @@ class StaticConsoleSink:
                 self._max_name_width = max(self._max_name_width, len(event["stage"]))
                 self._completion_buffer.append(event)
             case "log_line":
-                # Always buffer for failed-stage output; stream if --show-output
+                # Buffer for the report; stream if --show-output
                 stage = event["stage"]
-                buf = self._stage_logs.setdefault(stage, [])
-                if len(buf) < _MAX_LOG_LINES_PER_STAGE:
-                    buf.append(event["line"])
+                _buffer_log_line(event, self._stage_logs, self._stage_warnings)
                 if self._show_output:
                     line = rich.markup.escape(event["line"])
                     if event["is_stderr"]:
@@ -218,10 +280,12 @@ class StaticConsoleSink:
             total=self._total,
             max_name_width=self._max_name_width,
             stage_logs=self._stage_logs,
+            stage_warnings=None if self._show_output else self._stage_warnings,
         )
         self._max_name_width = 0
         self._completion_buffer.clear()
         self._stage_logs.clear()
+        self._stage_warnings.clear()
         self._total = 0
 
     async def close(self) -> None:
@@ -254,6 +318,7 @@ class LiveConsoleSink:
     _max_name_width: int
     _completion_buffer: list[StageCompleted]
     _stage_logs: dict[str, list[str]]
+    _stage_warnings: dict[str, _StageWarnings]
     _total: int
     _live: rich.live.Live | None
 
@@ -268,6 +333,7 @@ class LiveConsoleSink:
         self._max_name_width = 0
         self._completion_buffer = list[StageCompleted]()
         self._stage_logs = dict[str, list[str]]()
+        self._stage_warnings = dict[str, _StageWarnings]()
         self._total = 0
         self._live = None
 
@@ -372,9 +438,7 @@ class LiveConsoleSink:
                 self._update_live()
             case "log_line":
                 stage = event["stage"]
-                buf = self._stage_logs.setdefault(stage, [])
-                if len(buf) < _MAX_LOG_LINES_PER_STAGE:
-                    buf.append(event["line"])
+                _buffer_log_line(event, self._stage_logs, self._stage_warnings)
                 if self._show_output:
                     line = rich.markup.escape(event["line"])
                     if event["is_stderr"]:
@@ -405,6 +469,7 @@ class LiveConsoleSink:
             total=self._total,
             max_name_width=self._max_name_width,
             stage_logs=self._stage_logs,
+            stage_warnings=None if self._show_output else self._stage_warnings,
         )
         self._running.clear()
         self._completed_count = 0
@@ -412,6 +477,7 @@ class LiveConsoleSink:
         self._max_name_width = 0
         self._completion_buffer.clear()
         self._stage_logs.clear()
+        self._stage_warnings.clear()
         self._total = 0
 
     async def close(self) -> None:

{pivot-0.2.0 → pivot-0.2.2}/src/pivot/executor/worker.py

@@ -6,6 +6,7 @@ Must be module-level and picklable.
 
 from __future__ import annotations
 
+import codecs
 import collections
 import collections.abc as collections_abc
 import contextlib
@@ -17,6 +18,7 @@ import random
 import sys
 import threading
 import traceback
+import warnings
 from typing import IO, TYPE_CHECKING, Any, Literal, NamedTuple, TypedDict, cast, overload, override
 
 import pydantic
@@ -818,6 +820,42 @@ def _write_output_with_store(
         _reject_path_inside_project(ref, value, project_root, dependency_paths)
     output_path = store.prepare_output(ref)
     writer = cast("Any", ref.format)
+
+    output_label = (
+        f"its {type(writer).__name__} output"
+        if ref.identity.key is None
+        else f"output {ref.identity.key!r}"
+    )
+
+    def save_output(output_value: Any, destination: pathlib.Path) -> None:
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", loaders.LossyOutputWarning)
+            try:
+                writer.save(output_value, destination)
+            except TypeError as e:
+                if isinstance(writer, loaders.YAML):
+                    raise TypeError(
+                        f"Stage '{ref.identity.producer}' cannot write {output_label}: {e}"
+                    ) from e
+                raise
+        for caught_warning in caught:
+            if issubclass(caught_warning.category, loaders.LossyOutputWarning):
+                logger.warning(
+                    "Stage '%s' wrote %s %s: %s",
+                    ref.identity.producer,
+                    output_label,
+                    types.LOSSY_OUTPUT_PHRASE,
+                    caught_warning.message,
+                )
+            else:
+                warnings.warn_explicit(
+                    caught_warning.message,
+                    caught_warning.category,
+                    caught_warning.filename,
+                    caught_warning.lineno,
+                    source=caught_warning.source,
+                )
+
     if ref.tag is types.ArtifactTag.DIRECTORY:
         if not isinstance(value, dict):
             raise RuntimeError(
@@ -844,11 +882,11 @@ def _write_output_with_store(
             members.append((output_path / normalized, item_value))
         for full_path, item_value in members:
             full_path.parent.mkdir(parents=True, exist_ok=True)
-            writer.save(item_value, full_path)
+            save_output(item_value, full_path)
         return output_path
 
     output_path.parent.mkdir(parents=True, exist_ok=True)
-    writer.save(value, output_path)
+    save_output(value, output_path)
     return output_path
 
 
@@ -1095,14 +1133,19 @@ class _QueueWriter:
 
     def _pipe_reader(self) -> None:
         assert self._read_fd is not None  # Guaranteed by _ensure_pipe() before thread starts
+        # One incremental decoder per pipe: a multi-byte character split across two reads
+        # is held until its remaining bytes arrive instead of becoming U+FFFD.
+        decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
         try:
             while True:
                 data = os.read(self._read_fd, 8192)
                 if not data:
                     break
-                self.write(data.decode("utf-8", errors="replace"))
+                self.write(decoder.decode(data))
         except OSError:
             pass
+        # Flush a truncated trailing character so it is reported rather than dropped.
+        self.write(decoder.decode(b"", final=True))
 
     def _ensure_pipe(self) -> None:
         if self._read_fd is not None:

{pivot-0.2.0 → pivot-0.2.2}/src/pivot/import_artifact.py

@@ -1,8 +1,12 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import functools
 import logging
+import os
+import pathlib
+import tempfile
 from typing import TYPE_CHECKING, TypedDict, cast
 
 import aiohttp
@@ -16,7 +20,6 @@ from pivot.storage import cache, track
 from pivot.storage import lock as storage_lock
 
 if TYPE_CHECKING:
-    import pathlib
     from collections.abc import Callable
 
     from pivot.types import DirManifestEntry, OutEntry, StorageLockData
@@ -39,6 +42,31 @@ class ImportResult(TypedDict):
     downloaded: bool
 
 
+async def _download_verified(remote_url: str, file_hash: str, data_path: pathlib.Path) -> int:
+    """Download ``file_hash`` to ``data_path`` only if its content matches; return its size.
+
+    The file is downloaded to a temp file in the destination directory and moved over
+    ``data_path`` after its hash is checked, so a corrupt remote copy leaves any existing
+    file (the last good import) untouched.
+    """
+    fd, tmp_name = tempfile.mkstemp(
+        dir=data_path.parent, prefix=f".{data_path.name}.", suffix=".tmp"
+    )
+    os.close(fd)
+    tmp_path = pathlib.Path(tmp_name)
+    try:
+        await remote_storage.S3Remote(remote_url).download_file(file_hash, tmp_path)
+        actual_hash, _ = cache.hash_file(tmp_path)
+        if actual_hash != file_hash:
+            raise exceptions.RemoteError(f"Hash mismatch: expected {file_hash}, got {actual_hash}")
+        size = tmp_path.stat().st_size
+        os.replace(tmp_path, data_path)
+    finally:
+        with contextlib.suppress(FileNotFoundError):
+            tmp_path.unlink()
+    return size
+
+
 def _maybe_token(repo_url: str, token: str | None) -> str | None:
     if token is not None:
         return token
@@ -111,7 +139,13 @@ async def read_remote_config(
     remotes = {str(k): str(v) for k, v in remotes_data.items()}
 
     default_remote = config_data.get("default_remote")
-    if not isinstance(default_remote, str):
+    if default_remote is not None and not isinstance(default_remote, str):
+        # Warn-first: the source config is a saved input outside this repo.
+        logger.warning(
+            "Remote config default_remote in %s must be a remote name, got %r; ignoring it",
+            repo_url,
+            default_remote,
+        )
         default_remote = None
     if default_remote is None:
         if len(remotes) == 1:
@@ -370,16 +404,8 @@ async def import_artifact(
     downloaded = False
     file_size = resolved["size"]
     if not no_download:
-        remote = remote_storage.S3Remote(resolved["remote_url"])
-        await remote.download_file(resolved["hash"], data_path)
+        file_size = await _download_verified(resolved["remote_url"], resolved["hash"], data_path)
         downloaded = True
-        actual_hash, _ = cache.hash_file(data_path)
-        if actual_hash != resolved["hash"]:
-            data_path.unlink(missing_ok=True)
-            raise exceptions.RemoteError(
-                f"Downloaded file hash mismatch: expected {resolved['hash']}, got {actual_hash}"
-            )
-        file_size = data_path.stat().st_size
 
     source = track.ImportSource(
         repo=repo_url,
@@ -456,16 +482,8 @@ async def update_import(pvt_path: pathlib.Path, *, new_rev: str | None = None) -
 
     file_size = resolved["size"]
     if need_download:
-        remote = remote_storage.S3Remote(resolved["remote_url"])
-        await remote.download_file(resolved["hash"], data_path)
-        actual_hash, _ = cache.hash_file(data_path)
-        if actual_hash != resolved["hash"]:
-            data_path.unlink(missing_ok=True)
-            raise exceptions.RemoteError(
-                f"Hash mismatch: expected {resolved['hash']}, got {actual_hash}"
-            )
+        file_size = await _download_verified(resolved["remote_url"], resolved["hash"], data_path)
         downloaded = True
-        file_size = data_path.stat().st_size
     elif data_path.exists():
         file_size = data_path.stat().st_size