prefactor-core 0.2.5__tar.gz → 0.2.6__tar.gz

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: prefactor-core
-Version: 0.2.5
+Version: 0.2.6
 Summary: Core Prefactor SDK with async queue-based operations
 Author-email: Prefactor Pty Ltd <josh@prefactor.tech>
 License: MIT
 Requires-Python: <4.0.0,>=3.11.0
-Requires-Dist: prefactor-http>=0.1.3
+Requires-Dist: prefactor-http>=0.1.4
 Requires-Dist: pydantic>=2.0.0
 Description-Content-Type: text/markdown
 

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/examples/agent_e2e.py

@@ -31,6 +31,8 @@ deployment-scoped token, leave it unset and the API derives the agent and
 environment from the token.
 """
 
+from __future__ import annotations
+
 import asyncio
 import os
 
@@ -186,6 +188,8 @@ async def simulate_retrieval(query: str) -> list[str]:
 
 async def main() -> None:
     agent_id = os.environ.get("PREFACTOR_AGENT_ID")
+    if agent_id is not None:
+        agent_id = agent_id.strip() or None
 
     config = PrefactorCoreConfig(
         http_config=HttpClientConfig(

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/pyproject.toml

@@ -9,7 +9,7 @@ authors = [
 ]
 requires-python = ">=3.11.0, <4.0.0"
 dependencies = [
-    "prefactor-http>=0.1.3",
+    "prefactor-http>=0.1.4",
     "pydantic>=2.0.0",
 ]
 

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/src/prefactor_core/__init__.py

@@ -16,6 +16,7 @@ from .exceptions import (
     OperationError,
     PrefactorCoreError,
     PrefactorTelemetryFailureError,
+    PrefactorTerminatedError,
     SpanNotFoundError,
 )
 from .managers.agent_instance import AgentInstanceHandle
@@ -43,6 +44,7 @@ __all__ = [
     "InstanceNotFoundError",
     "SpanNotFoundError",
     "PrefactorTelemetryFailureError",
+    "PrefactorTerminatedError",
     # Models
     "AgentInstance",
     "Span",

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/src/prefactor_core/_version.py

@@ -3,5 +3,5 @@
 from __future__ import annotations
 
 PACKAGE_NAME = "prefactor-core"
-__version__ = "0.2.5"
+__version__ = "0.2.6"
 PACKAGE_VERSION = __version__

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/src/prefactor_core/client.py

@@ -7,13 +7,18 @@ through an async queue-based architecture.
 
 from __future__ import annotations
 
+import asyncio
 import logging
 import time
 from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Any
 
 from prefactor_http.client import PrefactorHttpClient
-from prefactor_http.exceptions import is_permanent_http_error, is_transient_http_error
+from prefactor_http.exceptions import (
+    PrefactorApiError,
+    is_permanent_http_error,
+    is_transient_http_error,
+)
 
 from ._version import PACKAGE_NAME as CORE_PACKAGE_NAME
 from ._version import PACKAGE_VERSION as CORE_PACKAGE_VERSION
@@ -26,6 +31,7 @@ from .exceptions import (
 )
 from .managers.agent_instance import AgentInstanceManager
 from .managers.span import SpanManager
+from .monitoring.termination_monitor import TerminationMonitor
 from .operations import Operation, OperationType
 from .queue.base import Queue
 from .queue.executor import TaskExecutor
@@ -86,6 +92,9 @@ class PrefactorCoreClient:
         self._initialized = False
         self._telemetry_failure: PrefactorTelemetryFailureError | None = None
         self._telemetry_failure_observed = False
+        self._termination_monitor: TerminationMonitor | None = None
+        self._sync_task: asyncio.Task | None = None
+        self._current_instance_id: str | None = None
 
     def _build_http_sdk_header(self) -> str:
         """Build the effective SDK header for HTTP requests."""
@@ -155,6 +164,13 @@ class PrefactorCoreClient:
 
         self._initialized = True
 
+        self._termination_monitor = TerminationMonitor(
+            fetch_instance=self._fetch_instance_for_poll,
+        )
+        self._sync_task = asyncio.create_task(
+            self._run_sync_loop(), name="prefactor-termination-sync"
+        )
+
     async def close(self) -> None:
         """Close the client and cleanup resources.
 
@@ -164,6 +180,24 @@ class PrefactorCoreClient:
         if not self._initialized:
             return
 
+        if self._sync_task is not None:
+            if not self._sync_task.done():
+                self._sync_task.cancel()
+            try:
+                await self._sync_task
+            except asyncio.CancelledError:
+                pass
+            except Exception:
+                logger.exception(
+                    "Termination sync loop exited with error during close()"
+                )
+            finally:
+                self._sync_task = None
+
+        if self._termination_monitor is not None:
+            self._termination_monitor.destroy()
+            self._termination_monitor = None
+
         # Stop executor
         if self._executor:
             await self._executor.stop()
@@ -272,12 +306,23 @@ class PrefactorCoreClient:
                 )
 
             elif operation.type == OperationType.FINISH_AGENT_INSTANCE:
-                await self._http.agent_instances.finish(
-                    agent_instance_id=operation.payload["instance_id"],
-                    status=operation.payload.get("status", "complete"),
-                    timestamp=operation.timestamp,
-                    idempotency_key=operation.payload.get("idempotency_key"),
-                )
+                try:
+                    await self._http.agent_instances.finish(
+                        agent_instance_id=operation.payload["instance_id"],
+                        status=operation.payload.get("status", "complete"),
+                        timestamp=operation.timestamp,
+                        idempotency_key=operation.payload.get("idempotency_key"),
+                    )
+                except PrefactorApiError as finish_err:
+                    if finish_err.status_code == 409:
+                        logger.debug(
+                            "[prefactor:http] Agent instance %s already in"
+                            " terminal state; skipping finish.",
+                            operation.payload["instance_id"],
+                        )
+                        return
+                    raise
+
             elif operation.type == OperationType.CREATE_SPAN:
                 await self._http.agent_spans.create(
                     agent_instance_id=operation.payload["instance_id"],
@@ -286,6 +331,7 @@ class PrefactorCoreClient:
                     id=operation.payload.get("span_id"),
                     parent_span_id=operation.payload.get("parent_span_id"),
                     payload=operation.payload.get("payload"),
+                    control_signal_callback=self._on_control_signal,
                 )
 
             elif operation.type == OperationType.FINISH_SPAN:
@@ -295,6 +341,7 @@ class PrefactorCoreClient:
                     result_payload=operation.payload.get("result_payload"),
                     timestamp=operation.timestamp,
                     idempotency_key=operation.payload.get("idempotency_key"),
+                    control_signal_callback=self._on_control_signal,
                 )
 
         except Exception as e:
@@ -307,6 +354,32 @@ class PrefactorCoreClient:
             )
             raise
 
+    async def _fetch_instance_for_poll(self, instance_id: str):
+        if self._http is None:
+            return None
+        return await self._http.agent_instances.get(instance_id)
+
+    async def _run_sync_loop(self) -> None:
+        while True:
+            await asyncio.sleep(1)
+            if self._termination_monitor is None:
+                continue
+            try:
+                self._termination_monitor.sync(self._current_instance_id)
+            except Exception:
+                logger.exception("Termination sync iteration failed")
+
+    def _on_control_signal(self, reason: str | None) -> None:
+        if self._termination_monitor is not None:
+            self._termination_monitor.detect_termination(reason)
+
+    def _set_current_instance(self, instance_id: str | None) -> None:
+        self._current_instance_id = instance_id
+
+    def _clear_current_instance(self, instance_id: str) -> None:
+        if self._current_instance_id == instance_id:
+            self._current_instance_id = None
+
     @property
     def instance_manager(self) -> AgentInstanceManager | None:
         """Public accessor for the agent instance manager."""
@@ -381,6 +454,8 @@ class PrefactorCoreClient:
             environment_id=environment_id,
         )
 
+        self._set_current_instance(instance_id)
+
         return AgentInstanceHandle(
             instance_id=instance_id,
             client=self,

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/src/prefactor_core/exceptions.py

@@ -58,6 +58,23 @@ class PrefactorTelemetryFailureError(PrefactorCoreError):
         self.dropped_operations = dropped_operations
 
 
+class PrefactorTerminatedError(PrefactorCoreError):
+    """Raised when the agent instance has been terminated by p2.
+
+    Args:
+        reason: Optional reason reported by p2 for the termination.
+    """
+
+    def __init__(self, reason: str | None = None) -> None:
+        msg = (
+            f"Agent instance terminated by p2: {reason}"
+            if reason
+            else "Agent instance terminated by p2"
+        )
+        super().__init__(msg)
+        self.reason = reason
+
+
 __all__ = [
     "PrefactorCoreError",
     "ClientNotInitializedError",
@@ -66,4 +83,5 @@ __all__ = [
     "InstanceNotFoundError",
     "SpanNotFoundError",
     "PrefactorTelemetryFailureError",
+    "PrefactorTerminatedError",
 ]

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/src/prefactor_core/managers/agent_instance.py

@@ -218,12 +218,18 @@ class AgentInstanceHandle:
     async def finish(self, status: "FinishStatus" = "complete") -> None:
         """Mark the instance as finished.
 
-        This queues a finish operation for the instance.
+        Resets the termination monitor (fence + new event) before enqueueing
+        the HTTP finish so stale span responses from the dying run cannot
+        trigger termination on the next run.
 
         Args:
             status: Terminal status for the instance — one of ``"complete"``,
                 ``"failed"``, or ``"cancelled"``. Defaults to ``"complete"``.
         """
+        monitor = getattr(self._client, "_termination_monitor", None)
+        if monitor is not None:
+            monitor.reset()
+        self._client._clear_current_instance(self._instance_id)
         manager = self._client.instance_manager
         assert manager is not None
         await manager.finish_with_idempotency_key(

prefactor_core-0.2.6/src/prefactor_core/monitoring/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from prefactor_core.monitoring.termination_monitor import TerminationMonitor
+
+__all__ = ["TerminationMonitor"]

prefactor_core-0.2.6/src/prefactor_core/monitoring/termination_monitor.py

@@ -0,0 +1,151 @@
+"""TerminationMonitor — detects agent instance termination via two paths:
+1. Fast: control signal in span API responses (pushed via callback)
+2. Slow: polling the instance status endpoint every `poll_interval` seconds
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Callable
+
+logger = logging.getLogger(__name__)
+
+
+class TerminationMonitor:
+    """Monitors an agent instance for termination.
+
+    Thread-safety note: `detect_termination` and `get_termination_event().is_set()`
+    are safe to call from sync worker threads — they only read/write a bool under
+    the GIL (asyncio.Event._value is a plain bool).
+    """
+
+    def __init__(
+        self,
+        fetch_instance: Callable,
+        poll_interval: float = 30.0,
+    ) -> None:
+        self._fetch_instance = fetch_instance
+        self._poll_interval = poll_interval
+
+        self._event = asyncio.Event()
+        self._termination_reason: str | None = None
+        self._callbacks: list[Callable[[], None]] = []
+
+        self._current_instance_id: str | None = None
+        self._poll_task: asyncio.Task | None = None
+        self._destroyed = False
+
+        # Fence blocks stale span callbacks from a previous run after reset()
+        self._fenced = False
+        # Generation increments on each reset so stale polls self-discard
+        self._generation = 0
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    @property
+    def termination_reason(self) -> str | None:
+        return self._termination_reason
+
+    def get_termination_event(self) -> asyncio.Event:
+        return self._event
+
+    def detect_termination(self, reason: str | None) -> None:
+        """Signal that the instance has been terminated.
+
+        No-op if already terminated, destroyed, or fenced (post-reset stale call).
+        """
+        if self._destroyed or self._event.is_set() or self._fenced:
+            return
+        self._termination_reason = reason
+        self._event.set()
+        self._stop_poll()
+        for cb in list(self._callbacks):
+            try:
+                cb()
+            except Exception:
+                logger.exception("Termination callback failed: %r", cb)
+
+    def sync(self, instance_id: str | None) -> None:
+        """Update the tracked instance ID and start/stop the fallback poll.
+
+        Idempotent: calling with the same non-None ID when a poll is already
+        running does not restart it (preserving the sleep interval).
+        """
+        if instance_id is not None:
+            self._fenced = False
+        if instance_id == self._current_instance_id:
+            # Same ID — no change needed; poll (if any) keeps running
+            return
+        self._current_instance_id = instance_id
+        self._stop_poll()
+        if instance_id is not None and not self._event.is_set() and not self._destroyed:
+            self._start_poll(instance_id, self._generation)
+
+    def reset(self) -> None:
+        """Prepare monitor for the next agent run.
+
+        - Creates a fresh (unset) event
+        - Clears the termination reason
+        - Cancels any in-flight poll
+        - Sets fence so stale callbacks from the dying run are ignored
+        - Increments generation so stale polls self-discard
+        """
+        self._stop_poll()
+        self._event = asyncio.Event()
+        self._termination_reason = None
+        self._current_instance_id = None
+        self._generation += 1
+        self._fenced = True
+
+    def subscribe(self, callback: Callable[[], None]) -> Callable[[], None]:
+        """Register a callback invoked on termination. Returns an unsubscribe fn."""
+        self._callbacks.append(callback)
+
+        def unsubscribe() -> None:
+            try:
+                self._callbacks.remove(callback)
+            except ValueError:
+                pass
+
+        return unsubscribe
+
+    def destroy(self) -> None:
+        """Permanently shut down the monitor (no further events will fire)."""
+        self._destroyed = True
+        self._stop_poll()
+        self._callbacks.clear()
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _start_poll(self, instance_id: str, generation: int) -> None:
+        self._poll_task = asyncio.get_event_loop().create_task(
+            self._poll(instance_id, generation)
+        )
+
+    def _stop_poll(self) -> None:
+        if self._poll_task is not None and not self._poll_task.done():
+            self._poll_task.cancel()
+        self._poll_task = None
+
+    async def _poll(self, instance_id: str, generation: int) -> None:
+        while True:
+            await asyncio.sleep(self._poll_interval)
+            # Guard: generation changed means reset() was called
+            if self._generation != generation:
+                return
+            try:
+                instance = await self._fetch_instance(instance_id)
+            except Exception:
+                logger.debug("Termination poll error (transient)", exc_info=True)
+                continue
+            # Guard again after await
+            if self._generation != generation:
+                return
+            if instance.status == "terminated":
+                self.detect_termination(instance.termination_reason)
+                return

prefactor_core-0.2.6/tests/monitoring/test_termination_monitor.py

@@ -0,0 +1,259 @@
+"""Tests for TerminationMonitor — 19 tests covering primary path, fallback poll,
+reset(), and callback lifecycle."""
+
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+from prefactor_core.monitoring.termination_monitor import TerminationMonitor
+
+
+def _make_monitor(fetch_instance=None) -> TerminationMonitor:
+    if fetch_instance is None:
+        fetch_instance = AsyncMock(
+            return_value=MagicMock(status="active", termination_reason=None)
+        )
+    return TerminationMonitor(fetch_instance=fetch_instance)
+
+
+def _terminated_instance(reason: str | None = "test reason"):
+    inst = MagicMock()
+    inst.status = "terminated"
+    inst.termination_reason = reason
+    return inst
+
+
+# ---------------------------------------------------------------------------
+# Primary path (5 tests)
+# ---------------------------------------------------------------------------
+
+
+class TestPrimaryPath:
+    async def test_detect_termination_sets_event(self):
+        monitor = _make_monitor()
+        assert not monitor.get_termination_event().is_set()
+        monitor.detect_termination("reason")
+        assert monitor.get_termination_event().is_set()
+
+    async def test_reason_propagates(self):
+        monitor = _make_monitor()
+        monitor.detect_termination("my reason")
+        assert monitor.termination_reason == "my reason"
+
+    async def test_null_reason_accepted(self):
+        monitor = _make_monitor()
+        monitor.detect_termination(None)
+        assert monitor.get_termination_event().is_set()
+        assert monitor.termination_reason is None
+
+    async def test_second_detect_termination_is_idempotent(self):
+        callback = MagicMock()
+        monitor = _make_monitor()
+        monitor.subscribe(callback)
+        monitor.detect_termination("first")
+        monitor.detect_termination("second")
+        callback.assert_called_once()
+        assert monitor.termination_reason == "first"
+
+    async def test_detect_termination_noop_after_destroy(self):
+        monitor = _make_monitor()
+        monitor.destroy()
+        monitor.detect_termination("reason")
+        assert not monitor.get_termination_event().is_set()
+
+
+# ---------------------------------------------------------------------------
+# Fallback poll (5 tests)
+# ---------------------------------------------------------------------------
+
+
+class TestFallbackPoll:
+    async def test_poll_starts_when_instance_id_arrives(self):
+        monitor = _make_monitor()
+        monitor.sync("inst-1")
+        await asyncio.sleep(0)  # yield to let task start
+        assert monitor._poll_task is not None
+        assert not monitor._poll_task.done()
+        monitor.destroy()
+
+    async def test_poll_stops_when_sync_called_with_none(self):
+        monitor = _make_monitor()
+        monitor.sync("inst-1")
+        await asyncio.sleep(0)
+        poll_task = monitor._poll_task
+        assert poll_task is not None
+        monitor.sync(None)
+        await asyncio.sleep(0)
+        assert poll_task.cancelled() or poll_task.done()
+        monitor.destroy()
+
+    async def test_no_poll_without_instance_id(self):
+        monitor = _make_monitor()
+        monitor.sync(None)
+        await asyncio.sleep(0)
+        assert monitor._poll_task is None
+        monitor.destroy()
+
+    async def test_poll_stops_after_termination_detected(self):
+        monitor = _make_monitor()
+        monitor.sync("inst-1")
+        await asyncio.sleep(0)
+        poll_task = monitor._poll_task
+        assert poll_task is not None
+        monitor.detect_termination("reason")
+        await asyncio.sleep(0)
+        assert poll_task.cancelled() or poll_task.done()
+        monitor.destroy()
+
+    async def test_poll_survives_transient_http_errors(self):
+        fetch = AsyncMock(side_effect=Exception("network error"))
+        monitor = TerminationMonitor(fetch_instance=fetch, poll_interval=0.05)
+        monitor.sync("inst-1")
+        await asyncio.sleep(0.2)  # let poll fire a couple times
+        # monitor should not be terminated — error was swallowed
+        assert not monitor.get_termination_event().is_set()
+        monitor.destroy()
+
+
+# ---------------------------------------------------------------------------
+# reset() (7 tests)
+# ---------------------------------------------------------------------------
+
+
+class TestReset:
+    async def test_reset_creates_fresh_event(self):
+        monitor = _make_monitor()
+        monitor.detect_termination("reason")
+        old_event = monitor.get_termination_event()
+        assert old_event.is_set()
+        monitor.reset()
+        new_event = monitor.get_termination_event()
+        assert not new_event.is_set()
+        assert new_event is not old_event
+
+    async def test_reset_allows_new_termination_after_sync_with_new_id(self):
+        monitor = _make_monitor()
+        monitor.detect_termination("run 1")
+        monitor.reset()
+        # fenced — detect_termination should be blocked
+        monitor.detect_termination("stale")
+        assert not monitor.get_termination_event().is_set()
+        # sync with new id lifts fence
+        monitor.sync("inst-2")
+        monitor.detect_termination("run 2")
+        assert monitor.get_termination_event().is_set()
+        assert monitor.termination_reason == "run 2"
+        monitor.destroy()
+
+    async def test_get_termination_event_returns_new_event_after_reset(self):
+        monitor = _make_monitor()
+        getter = monitor.get_termination_event
+        monitor.detect_termination("reason")
+        monitor.reset()
+        # getter returns new (unset) event
+        assert not getter().is_set()
+
+    async def test_reset_cancels_poll_task(self):
+        monitor = _make_monitor()
+        monitor.sync("inst-1")
+        await asyncio.sleep(0)
+        poll_task = monitor._poll_task
+        assert poll_task is not None
+        monitor.reset()
+        await asyncio.sleep(0)
+        assert poll_task.cancelled() or poll_task.done()
+
+    async def test_reset_preserves_callbacks(self):
+        callback = MagicMock()
+        monitor = _make_monitor()
+        monitor.subscribe(callback)
+        monitor.reset()
+        # sync to lift fence, then detect
+        monitor.sync("inst-2")
+        monitor.detect_termination("after reset")
+        callback.assert_called_once()
+
+    async def test_fence_blocks_detect_termination_until_sync_with_new_id(self):
+        monitor = _make_monitor()
+        monitor.detect_termination("run 1")
+        monitor.reset()  # fenced = True
+
+        # stale span response fires during queue drain
+        monitor.detect_termination("stale span response")
+        assert not monitor.get_termination_event().is_set()
+
+        # sync with None doesn't lift fence
+        monitor.sync(None)
+        monitor.detect_termination("still stale")
+        assert not monitor.get_termination_event().is_set()
+
+        # sync with new instance id lifts fence
+        monitor.sync("instance-2")
+        monitor.detect_termination("run 2")
+        assert monitor.get_termination_event().is_set()
+        assert monitor.termination_reason == "run 2"
+        monitor.destroy()
+
+    async def test_stale_poll_response_discarded_after_reset(self):
+        """Poll fires for old instance after reset — generation check discards it."""
+        slow_fetch = AsyncMock(return_value=_terminated_instance("old run"))
+        monitor = TerminationMonitor(fetch_instance=slow_fetch, poll_interval=0.05)
+        monitor.sync("inst-1")
+        await asyncio.sleep(0)  # poll task started
+
+        # reset before poll completes
+        monitor.reset()
+        await asyncio.sleep(0.2)  # let poll fire with old generation
+
+        # monitor should NOT be terminated
+        assert not monitor.get_termination_event().is_set()
+        monitor.destroy()
+
+
+# ---------------------------------------------------------------------------
+# Callback lifecycle (2 tests)
+# ---------------------------------------------------------------------------
+
+
+class TestCallbackLifecycle:
+    async def test_unsubscribe_removes_callback(self):
+        callback = MagicMock()
+        monitor = _make_monitor()
+        unsubscribe = monitor.subscribe(callback)
+        unsubscribe()
+        monitor.detect_termination("reason")
+        callback.assert_not_called()
+
+    async def test_callbacks_fire_in_registration_order(self):
+        order = []
+        monitor = _make_monitor()
+        monitor.subscribe(lambda: order.append(1))
+        monitor.subscribe(lambda: order.append(2))
+        monitor.subscribe(lambda: order.append(3))
+        monitor.detect_termination("reason")
+        assert order == [1, 2, 3]
+
+    async def test_bad_callback_does_not_block_later_callbacks(self):
+        order = []
+        monitor = _make_monitor()
+
+        def failing_callback():
+            order.append("bad")
+            raise RuntimeError("callback failed")
+
+        monitor.subscribe(failing_callback)
+        monitor.subscribe(lambda: order.append("good"))
+
+        monitor.detect_termination("reason")
+
+        assert order == ["bad", "good"]
+        assert monitor.get_termination_event().is_set()
+
+    async def test_destroy_clears_callbacks(self):
+        monitor = _make_monitor()
+        monitor.subscribe(MagicMock())
+
+        monitor.destroy()
+
+        assert monitor._callbacks == []

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/tests/test_agent_instance_finish_status.py

@@ -102,3 +102,70 @@ async def test_finish_forwards_explicit_status(status):
 
     assert len(stub_http.agent_instances.finish_calls) == 1
     assert stub_http.agent_instances.finish_calls[0]["status"] == status
+
+
+class TestFinishAgentInstance409Handling:
+    async def test_409_on_finish_treated_as_success(self):
+        """FINISH_AGENT_INSTANCE with 409 response should not raise."""
+        from datetime import datetime, timezone
+        from unittest.mock import AsyncMock, MagicMock
+
+        from prefactor_core.operations import Operation, OperationType
+        from prefactor_http.exceptions import PrefactorApiError
+
+        mock_http = MagicMock()
+        mock_http.agent_instances = MagicMock()
+        mock_http.agent_instances.finish = AsyncMock(
+            side_effect=PrefactorApiError("already terminated", "conflict", 409)
+        )
+        mock_http.agent_spans = MagicMock()
+        mock_http.agent_spans.create = AsyncMock()
+        mock_http.agent_spans.finish = AsyncMock()
+
+        config = PrefactorCoreConfig(
+            http_config=HttpClientConfig(api_url="http://fake", api_token="tok")
+        )
+        client = PrefactorCoreClient(config)
+        client._http = mock_http
+        client._initialized = True
+
+        op = Operation(
+            type=OperationType.FINISH_AGENT_INSTANCE,
+            payload={
+                "instance_id": "inst-1",
+                "idempotency_key": "key-1",
+                "status": "complete",
+            },
+            timestamp=datetime.now(timezone.utc),
+        )
+        # Should not raise
+        await client._process_operation(op)
+
+
+class TestAgentInstanceHandleFinishResetsMonitor:
+    async def test_finish_resets_termination_monitor(self):
+        """handle.finish() should reset the termination monitor before enqueueing."""
+        from unittest.mock import AsyncMock, MagicMock
+
+        from prefactor_core.managers.agent_instance import AgentInstanceHandle
+        from prefactor_core.monitoring.termination_monitor import TerminationMonitor
+
+        fetch = AsyncMock(
+            return_value=MagicMock(status="active", termination_reason=None)
+        )
+        monitor = TerminationMonitor(fetch_instance=fetch)
+        monitor.detect_termination("run 1")
+        assert monitor.get_termination_event().is_set()
+
+        mock_client = MagicMock()
+        mock_client._termination_monitor = monitor
+        mock_client.instance_manager = MagicMock()
+        mock_client.instance_manager.finish_with_idempotency_key = AsyncMock()
+
+        handle = AgentInstanceHandle(instance_id="inst-1", client=mock_client)
+        await handle.finish()
+
+        # Monitor should be reset (new unset event)
+        assert not monitor.get_termination_event().is_set()
+        # finish_with_idempotency_key should still be called
+        mock_client.instance_manager.finish_with_idempotency_key.assert_called_once()

{prefactor_core-0.2.5 → prefactor_core-0.2.6}/tests/test_failure_handling.py

@@ -3,8 +3,9 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 from types import SimpleNamespace
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import aiohttp
 import pytest
@@ -103,6 +104,60 @@ async def _wait_until(
     raise AssertionError("Timed out waiting for expected condition")
 
 
+@pytest.mark.asyncio
+async def test_termination_sync_loop_survives_iteration_failures(caplog):
+    """A failed sync iteration should be logged without stopping future syncs."""
+    client = PrefactorCoreClient(_make_client_config())
+    monitor = Mock()
+    calls = 0
+
+    def sync(_instance_id):
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            raise RuntimeError("sync failed")
+
+    monitor.sync.side_effect = sync
+    client._termination_monitor = monitor
+    original_sleep = asyncio.sleep
+
+    async def fast_sleep(_delay):
+        await original_sleep(0)
+
+    with (
+        caplog.at_level(logging.ERROR, logger="prefactor_core.client"),
+        patch("prefactor_core.client.asyncio.sleep", side_effect=fast_sleep),
+    ):
+        task = asyncio.create_task(client._run_sync_loop())
+        try:
+            await _wait_until(lambda: monitor.sync.call_count >= 2)
+        finally:
+            task.cancel()
+            with pytest.raises(asyncio.CancelledError):
+                await task
+
+    assert "Termination sync iteration failed" in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_close_observes_completed_sync_task_exceptions(caplog):
+    """close() should await already-failed sync tasks and clear the task handle."""
+    client = PrefactorCoreClient(_make_client_config())
+    client._initialized = True
+
+    async def fail_sync_loop():
+        raise RuntimeError("sync task failed")
+
+    client._sync_task = asyncio.create_task(fail_sync_loop())
+    await asyncio.sleep(0)
+
+    with caplog.at_level(logging.ERROR, logger="prefactor_core.client"):
+        await client.close()
+
+    assert client._sync_task is None
+    assert "Termination sync loop exited with error during close()" in caplog.text
+
+
 @pytest.mark.asyncio
 async def test_permanent_worker_failure_latches_and_rejects_future_operations():
     """Permanent failures should latch and reject later queued operations."""