shrinkray 25.12.29.0__py3-none-any.whl → 26.2.4.0__py3-none-any.whl

shrinkray/state.py

@@ -17,6 +17,13 @@ import humanize
 import trio
 from attrs import define
 
+from shrinkray.cli import InputType
+from shrinkray.formatting import default_reformat_data, determine_formatter_command
+from shrinkray.history import (
+    HistoryManager,
+    deserialize_directory,
+    serialize_directory,
+)
 from shrinkray.passes.clangdelta import ClangDelta
 from shrinkray.problem import (
     BasicReductionProblem,
@@ -24,6 +31,7 @@ from shrinkray.problem import (
     ReductionProblem,
     sort_key_for_initial,
 )
+from shrinkray.process import interrupt_wait_and_kill
 from shrinkray.reducer import DirectoryShrinkRay, Reducer, ShrinkRay
 from shrinkray.work import Volume, WorkContext
 
@@ -191,29 +199,113 @@ class ShrinkRayState[TestCase](ABC):
 
     first_call_time: float | None = None
 
-    # Lazy imports to break circular dependencies:
-    # - shrinkray.process imports from shrinkray.work which imports from here
-    # - shrinkray.cli imports from here for state configuration
-    # These are cached after first import for performance.
-    _interrupt_wait_and_kill: Any = None
-    _InputType: Any = None  # InputType enum from shrinkray.cli
-
     # Stores the output from the last debug run
     _last_debug_output: str = ""
 
-    # Optional output manager for capturing test output (TUI mode)
+    # Stores the output from the most recently completed test (for history recording)
+    # This is read immediately after the test's output file is closed to avoid
+    # race conditions with other parallel tests
+    _last_test_output: bytes | None = None
+
+    # Optional output manager for capturing test output (TUI mode or history)
     output_manager: OutputCaptureManager | None = None
 
+    # History recording (enabled by default)
+    history_enabled: bool = True
+    history_base_dir: str | None = None  # Base directory for .shrinkray folder
+    history_manager: HistoryManager | None = None
+
+    # Also-interesting exit code (None = disabled)
+    # When a test returns this code, it's recorded but not used for reduction
+    also_interesting_code: int | None = None
+
+    # Set of test cases to exclude from interestingness (for restart-from-point)
+    # These are byte-identical matches of previously reduced values
+    excluded_test_cases: set[bytes] | None = None
+
+    # Temp directory for output capture (when not using TUI's output manager)
+    _output_tempdir: TemporaryDirectory | None = None
+
+    # Stores output from successful tests, keyed by test case bytes
+    # This avoids race conditions when multiple tests run in parallel
+    _successful_outputs: dict[bytes, bytes] = {}
+
     def __attrs_post_init__(self):
         self.is_interesting_limiter = trio.CapacityLimiter(max(self.parallelism, 1))
+        self._successful_outputs = {}  # Initialize mutable default
         self.setup_formatter()
+        self._setup_history()
 
     @abstractmethod
     def setup_formatter(self): ...
 
+    @property
+    def is_directory_mode(self) -> bool:
+        """Whether this state manages directory test cases."""
+        return False
+
+    def _setup_history(self) -> None:
+        """Set up history recording if enabled or also-interesting is configured."""
+        # Create history manager if either:
+        # 1. Full history is enabled, or
+        # 2. also_interesting_code is set (records only also-interesting cases)
+        if not self.history_enabled and self.also_interesting_code is None:
+            return
+
+        # Create history manager (record_reductions=False if only also-interesting)
+        self.history_manager = HistoryManager.create(
+            self.test,
+            self.filename,
+            record_reductions=self.history_enabled,
+            is_directory=self.is_directory_mode,
+            base_dir=self.history_base_dir,
+        )
+
+        # Ensure we have an output manager for capturing test output
+        if self.output_manager is None:
+            self._output_tempdir = TemporaryDirectory()
+            self.output_manager = OutputCaptureManager(
+                output_dir=self._output_tempdir.name
+            )
+
+    def _get_last_captured_output(self) -> bytes | None:
+        """Get the output from the most recently completed test.
+
+        Returns the output content if available, None otherwise.
+        This returns the output that was captured immediately when the test
+        completed, avoiding race conditions with other parallel tests.
+        """
+        return self._last_test_output
+
+    def _check_also_interesting(self, exit_code: int, test_case: TestCase) -> None:
+        """Check if exit code matches also-interesting and record if so.
+
+        Args:
+            exit_code: The exit code from the test
+            test_case: The test case that was tested
+        """
+        if (
+            self.also_interesting_code is not None
+            and exit_code == self.also_interesting_code
+            and self.history_manager is not None
+        ):
+            test_case_bytes = self._get_test_case_bytes(test_case)
+            output = self._get_last_captured_output()
+            self.history_manager.record_also_interesting(test_case_bytes, output)
+
     @abstractmethod
     def new_reducer(self, problem: ReductionProblem[TestCase]) -> Reducer[TestCase]: ...
 
+    @abstractmethod
+    def _get_initial_bytes(self) -> bytes:
+        """Get the initial test case as bytes for history recording."""
+        ...
+
+    @abstractmethod
+    def _get_test_case_bytes(self, test_case: TestCase) -> bytes:
+        """Convert a test case to bytes for history recording."""
+        ...
+
     @abstractmethod
     async def write_test_case_to_file_impl(self, working: str, test_case: TestCase): ...
 
@@ -223,19 +315,9 @@ class ShrinkRayState[TestCase](ABC):
     async def run_script_on_file(
         self, working: str, cwd: str, debug: bool = False
     ) -> int:
-        # Lazy import to avoid circular dependency
-        if self._interrupt_wait_and_kill is None:
-            from shrinkray.process import interrupt_wait_and_kill
-
-            self._interrupt_wait_and_kill = interrupt_wait_and_kill
-        if self._InputType is None:
-            from shrinkray.cli import InputType
-
-            self._InputType = InputType
-
         if not os.path.exists(working):
             raise ValueError(f"No such file {working}")
-        if self.input_type.enabled(self._InputType.arg):
+        if self.input_type.enabled(InputType.arg):
             command = self.test + [working]
         else:
             command = self.test
@@ -246,9 +328,7 @@ class ShrinkRayState[TestCase](ABC):
             "cwd": cwd,
             "check": False,
         }
-        if self.input_type.enabled(self._InputType.stdin) and not os.path.isdir(
-            working
-        ):
+        if self.input_type.enabled(InputType.stdin) and not os.path.isdir(working):
             with open(working, "rb") as i:
                 kwargs["stdin"] = i.read()
         else:
@@ -291,6 +371,7 @@ class ShrinkRayState[TestCase](ABC):
         # Determine output handling
         test_id: int | None = None
         output_file_handle = None
+        output_path: str | None = None
         exit_code: int | None = None  # Track for output manager
 
         if self.output_manager is not None:
@@ -337,7 +418,7 @@ class ShrinkRayState[TestCase](ABC):
 
                     if sp.returncode is None:
                         # Process didn't terminate before timeout - kill it
-                        await self._interrupt_wait_and_kill(sp)
+                        await interrupt_wait_and_kill(sp)
 
                     # Check for timeout violation (only when timeout is explicitly set)
                     if (
@@ -364,21 +445,25 @@ class ShrinkRayState[TestCase](ABC):
 
                 return result
         finally:
-            # Clean up output file handle and mark test as completed
+            # Clean up output file handle and capture output immediately
             if output_file_handle is not None:
                 output_file_handle.close()
+                # Read the output file NOW, before any other test can interfere
+                # This avoids race conditions where get_current_output() returns
+                # a different test's partial output
+                # output_path must be set since it's assigned with output_file_handle
+                assert output_path is not None
+                try:
+                    with open(output_path, "rb") as f:
+                        self._last_test_output = f.read()
+                except OSError:
+                    self._last_test_output = None
             if test_id is not None and self.output_manager is not None:
                 self.output_manager.mark_completed(test_id, exit_code or 0)
 
     async def run_for_exit_code(self, test_case: TestCase, debug: bool = False) -> int:
-        # Lazy import
-        if self._InputType is None:
-            from shrinkray.cli import InputType
-
-            self._InputType = InputType
-
         if self.in_place:
-            if self.input_type == self._InputType.basename:
+            if self.input_type == InputType.basename:
                 working = self.filename
                 await self.write_test_case_to_file(working, test_case)
 
@@ -430,7 +515,7 @@ class ShrinkRayState[TestCase](ABC):
     @property
     def reducer(self):
         try:
-            return self.__reducer
+            return self._cached_reducer
         except AttributeError:
             pass
 
@@ -458,8 +543,20 @@ class ShrinkRayState[TestCase](ABC):
             async with write_lock:
                 await self.write_test_case_to_file(self.filename, test_case)
 
-        self.__reducer = self.new_reducer(problem)
-        return self.__reducer
+        # Initialize history and register callback if enabled
+        if self.history_manager is not None:
+            self._initialize_history_manager()
+
+            @problem.on_reduce
+            async def record_history(test_case: TestCase):
+                test_case_bytes = self._get_test_case_bytes(test_case)
+                # Use output captured at is_interesting time to avoid race conditions
+                output = self._successful_outputs.pop(test_case_bytes, None)
+                assert self.history_manager is not None
+                self.history_manager.record_reduction(test_case_bytes, output)
+
+        self._cached_reducer = self.new_reducer(problem)
+        return self._cached_reducer
 
     @property
     def extra_problem_kwargs(self):
@@ -470,10 +567,65 @@ class ShrinkRayState[TestCase](ABC):
         return self.reducer.target
 
     async def is_interesting(self, test_case: TestCase) -> bool:
+        # Check exclusion set first (for restart-from-point feature)
+        if self.excluded_test_cases is not None:
+            test_case_bytes = self._get_test_case_bytes(test_case)
+            if test_case_bytes in self.excluded_test_cases:
+                return False
+
         if self.first_call_time is None:
             self.first_call_time = time.time()
         async with self.is_interesting_limiter:
-            return await self.run_for_exit_code(test_case) == 0
+            exit_code = await self.run_for_exit_code(test_case)
+            self._check_also_interesting(exit_code, test_case)
+            if exit_code == 0:
+                # Capture output now while still in the limiter to avoid race conditions
+                # where another test starts and overwrites the "current" output
+                test_case_bytes = self._get_test_case_bytes(test_case)
+                output = self._get_last_captured_output()
+                if output is not None:
+                    self._successful_outputs[test_case_bytes] = output
+                return True
+            return False
+
+    def reset_for_restart(self, new_initial: bytes, excluded: set[bytes]) -> None:
+        """Reset state for restart from a history point.
+
+        This clears the cached reducer so it will be recreated with the new
+        initial value, and sets the exclusion set to reject previously
+        reduced values.
+
+        Args:
+            new_initial: The new initial test case content
+            excluded: Set of test cases to reject as uninteresting
+        """
+        self.excluded_test_cases = excluded
+        # Clear cached reducer so it will be recreated on next access
+        try:
+            del self._cached_reducer
+        except AttributeError:
+            pass
+        # Clear stored successful outputs (no longer relevant after restart)
+        self._successful_outputs.clear()
+        # Reset initial_exit_code - the new initial is known to be interesting
+        # (it came from history) so its exit code was 0
+        self.initial_exit_code = 0
+        # Update initial (implementation depends on subclass)
+        self._set_initial_for_restart(new_initial)
+
+    @abstractmethod
+    def _set_initial_for_restart(self, content: bytes) -> None:
+        """Set the initial test case for restart. Subclasses implement."""
+        ...
+
+    def _initialize_history_manager(self) -> None:
+        """Initialize the history manager. Subclasses can override for different modes."""
+        assert self.history_manager is not None
+        self.history_manager.initialize(
+            self._get_initial_bytes(),
+            self.test,
+            self.filename,
+        )
 
     @property
     def parallel_tasks_running(self) -> int:
@@ -622,12 +774,16 @@ class ShrinkRayStateSingleFile(ShrinkRayState[bytes]):
     def new_reducer(self, problem: ReductionProblem[bytes]) -> Reducer[bytes]:
         return ShrinkRay(problem, clang_delta=self.clang_delta_executable)
 
-    def setup_formatter(self):
-        from shrinkray.formatting import (
-            default_reformat_data,
-            determine_formatter_command,
-        )
+    def _get_initial_bytes(self) -> bytes:
+        return self.initial
+
+    def _get_test_case_bytes(self, test_case: bytes) -> bytes:
+        return test_case
 
+    def _set_initial_for_restart(self, content: bytes) -> None:
+        self.initial = content
+
+    def setup_formatter(self):
         if self.formatter.lower() == "none":
 
             async def format_data(test_case: bytes) -> bytes | None:
@@ -678,8 +834,24 @@ class ShrinkRayStateSingleFile(ShrinkRayState[bytes]):
             await o.write(test_case)
 
     async def is_interesting(self, test_case: bytes) -> bool:
+        # Check exclusion set first (for restart-from-point feature)
+        if (
+            self.excluded_test_cases is not None
+            and test_case in self.excluded_test_cases
+        ):
+            return False
+
         async with self.is_interesting_limiter:
-            return await self.run_for_exit_code(test_case) == 0
+            exit_code = await self.run_for_exit_code(test_case)
+            self._check_also_interesting(exit_code, test_case)
+            if exit_code == 0:
+                # Capture output now while still in the limiter to avoid race conditions
+                # where another test starts and overwrites the "current" output
+                output = self._get_last_captured_output()
+                if output is not None:
+                    self._successful_outputs[test_case] = output
+                return True
+            return False
 
     async def print_exit_message(self, problem):
         formatting_increase = 0
@@ -727,6 +899,11 @@ class ShrinkRayStateSingleFile(ShrinkRayState[bytes]):
 class ShrinkRayDirectoryState(ShrinkRayState[dict[str, bytes]]):
     def setup_formatter(self): ...
 
+    @property
+    def is_directory_mode(self) -> bool:
+        """Whether this state manages directory test cases."""
+        return True
+
     @property
     def extra_problem_kwargs(self) -> dict[str, Any]:
         return {
@@ -740,6 +917,35 @@ class ShrinkRayDirectoryState(ShrinkRayState[dict[str, bytes]]):
             target=problem, clang_delta=self.clang_delta_executable
         )
 
+    def _get_initial_bytes(self) -> bytes:
+        # Serialize directory content for history recording
+        return self._serialize_directory(self.initial)
+
+    def _get_test_case_bytes(self, test_case: dict[str, bytes]) -> bytes:
+        # Serialize directory content for comparison/exclusion
+        return self._serialize_directory(test_case)
+
+    def _set_initial_for_restart(self, content: bytes) -> None:
+        # Deserialize and update initial directory content
+        self.initial = self._deserialize_directory(content)
+
+    def _initialize_history_manager(self) -> None:
+        """Initialize the history manager in directory mode."""
+        assert self.history_manager is not None
+        self.history_manager.initialize_directory(
+            self.initial,
+            self.test,
+            self.filename,
+        )
+
+    @staticmethod
+    def _serialize_directory(content: dict[str, bytes]) -> bytes:
+        return serialize_directory(content)
+
+    @staticmethod
+    def _deserialize_directory(data: bytes) -> dict[str, bytes]:
+        return deserialize_directory(data)
+
     async def write_test_case_to_file_impl(
         self, working: str, test_case: dict[str, bytes]
     ):

shrinkray/subprocess/client.py

@@ -1,11 +1,13 @@
 """Client for communicating with the reducer subprocess."""
 
 import asyncio
+import os
 import sys
+import tempfile
 import traceback
 import uuid
 from collections.abc import AsyncGenerator
-from typing import Any
+from typing import IO, Any
 
 from shrinkray.subprocess.protocol import (
     ProgressUpdate,
@@ -27,18 +29,28 @@ class SubprocessClient:
         self._completed = False
         self._error_message: str | None = None
         self._debug_mode = debug_mode
+        self._stderr_log_file: IO[str] | None = None
+        self._stderr_log_path: str | None = None
 
     async def start(self) -> None:
         """Launch the subprocess."""
-        # In debug mode, inherit stderr so interestingness test output
-        # goes directly to the parent process's stderr
+        # Log subprocess stderr to a temp file for debugging.
+        # This captures bootstrap errors before history is set up.
+        # Once the worker starts with history enabled, it redirects its own
+        # stderr to the per-run history directory.
+        fd, self._stderr_log_path = tempfile.mkstemp(
+            prefix="shrinkray-stderr-",
+            suffix=".log",
+        )
+        self._stderr_log_file = os.fdopen(fd, "w", encoding="utf-8")
+
         self._process = await asyncio.create_subprocess_exec(
             sys.executable,
             "-m",
             "shrinkray.subprocess.worker",
             stdin=asyncio.subprocess.PIPE,
             stdout=asyncio.subprocess.PIPE,
-            stderr=sys.stderr,
+            stderr=self._stderr_log_file,
         )
         self._reader_task = asyncio.create_task(self._read_output())
 
@@ -137,6 +149,8 @@ class SubprocessClient:
         clang_delta: str = "",
         trivial_is_error: bool = True,
         skip_validation: bool = False,
+        history_enabled: bool = True,
+        also_interesting_code: int | None = None,
     ) -> Response:
         """Start the reduction process."""
         params: dict[str, Any] = {
@@ -151,6 +165,8 @@ class SubprocessClient:
             "clang_delta": clang_delta,
             "trivial_is_error": trivial_is_error,
             "skip_validation": skip_validation,
+            "history_enabled": history_enabled,
+            "also_interesting_code": also_interesting_code,
         }
         if parallelism is not None:
             params["parallelism"] = parallelism
@@ -203,6 +219,27 @@ class SubprocessClient:
             traceback.print_exc()
             return Response(id="", error="Failed to skip pass")
 
+    async def restart_from(self, reduction_number: int) -> Response:
+        """Restart reduction from a specific history point.
+
+        This moves all reductions after the specified point to also-interesting,
+        resets the current test case to that point, and continues reduction
+        from there, rejecting previously reduced values.
+
+        Args:
+            reduction_number: The reduction entry number to restart from
+                (e.g., 3 for reduction 0003)
+        """
+        if self._completed:
+            return Response(id="", error="Reduction already completed")
+        try:
+            return await self.send_command(
+                "restart_from", {"reduction_number": reduction_number}
+            )
+        except Exception:
+            traceback.print_exc()
+            return Response(id="", error="Failed to send restart command")
+
     async def get_progress_updates(self) -> AsyncGenerator[ProgressUpdate, None]:
         """Yield progress updates as they arrive."""
         while not self._completed:
@@ -248,6 +285,18 @@ class SubprocessClient:
                 except ProcessLookupError:
                     pass  # Process already exited
 
+        # Close and remove the stderr log file
+        if self._stderr_log_file is not None:
+            try:
+                self._stderr_log_file.close()
+            except Exception:
+                pass
+        if self._stderr_log_path is not None:
+            try:
+                os.unlink(self._stderr_log_path)
+            except Exception:
+                pass
+
     async def __aenter__(self) -> "SubprocessClient":
         await self.start()
         return self

shrinkray/subprocess/protocol.py

@@ -64,6 +64,10 @@ class ProgressUpdate:
     # New size history entries since last update: list of (runtime_seconds, size)
     # Client should accumulate these over time
     new_size_history: list[tuple[float, int]] = field(default_factory=list)
+    # History directory path (for browsing reductions/also-interesting)
+    history_dir: str | None = None
+    # Target file basename (for reading history files)
+    target_basename: str = ""
 
 
 @dataclass
@@ -127,6 +131,8 @@ def serialize(msg: Request | Response | ProgressUpdate) -> str:
                 "active_test_id": msg.active_test_id,
                 "last_test_return_code": msg.last_test_return_code,
                 "new_size_history": msg.new_size_history,
+                "history_dir": msg.history_dir,
+                "target_basename": msg.target_basename,
             },
         }
     else:
@@ -178,6 +184,8 @@ def deserialize(line: str) -> Request | Response | ProgressUpdate:
             active_test_id=d.get("active_test_id"),
             last_test_return_code=d.get("last_test_return_code"),
             new_size_history=[tuple(x) for x in d.get("new_size_history", [])],
+            history_dir=d.get("history_dir"),
+            target_basename=d.get("target_basename", ""),
         )
 
     # Check for response (has "result" or "error" field)