memwal 0.1.11.dev10__tar.gz → 0.1.11.dev11__tar.gz

{memwal-0.1.11.dev10 → memwal-0.1.11.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: memwal
-Version: 0.1.11.dev10
+Version: 0.1.11.dev11
 Summary: Python SDK for Walrus Memory — Privacy-first AI memory with Ed25519 signing
 Project-URL: Homepage, https://memory.walrus.xyz
 Project-URL: Documentation, https://memory.walrus.xyz

{memwal-0.1.11.dev10 → memwal-0.1.11.dev11}/memwal/__init__.py

@@ -128,4 +128,4 @@ __all__ = [
     "RecallManualResult",
 ]
 
-__version__ = "0.1.11.dev10"
+__version__ = "0.1.11.dev11"

{memwal-0.1.11.dev10 → memwal-0.1.11.dev11}/memwal/middleware.py

@@ -140,10 +140,67 @@ class _PendingSaves:
     run — this keeps a strong reference until each one is done.
     """
 
-    def __init__(self) -> None:
+    def __init__(
+        self,
+        on_save_error: Optional[Callable[[BaseException], None]] = None,
+    ) -> None:
         self._tasks: "set[asyncio.Task[Any]]" = set()
         self._threads: List[threading.Thread] = []
+        self._failures: List[tuple[str, str]] = []
+        self._dropped = 0
+        self._on_save_error = on_save_error
         self._lock = threading.Lock()
+        self._callback_threads: set[int] = set()
+
+    def note_error(self, err: BaseException) -> None:
+        """Remember a background analyze failure so flush does not report success.
+
+        The ledger is updated before the callback. A flush from that callback
+        returns without draining or clearing the failure. A flush from any
+        other thread still waits and raises. The ledger keeps only a type and
+        a one-line message, so a client that never flushes does not pin the
+        request traceback.
+        """
+        type_name = type(err).__name__
+        message = str(err).splitlines()[0][:200] if str(err) else ""
+        with self._lock:
+            if len(self._failures) < 8:
+                self._failures.append((type_name, message))
+            else:
+                self._dropped += 1
+        if self._on_save_error is None:
+            return
+        ident = threading.get_ident()
+        with self._lock:
+            self._callback_threads.add(ident)
+        try:
+            self._on_save_error(err)
+        except Exception:
+            logger.warning("Walrus Memory on_save_error raised", exc_info=True)
+        finally:
+            with self._lock:
+                self._callback_threads.discard(ident)
+
+    def _flush_called_from_callback(self) -> bool:
+        with self._lock:
+            return threading.get_ident() in self._callback_threads
+
+    def _note_result(self, result: Any) -> None:
+        if isinstance(result, BaseException):
+            self.note_error(result)
+
+    def _raise_if_failed(self) -> None:
+        with self._lock:
+            failures, self._failures = self._failures, []
+            dropped, self._dropped = self._dropped, 0
+        total = len(failures) + dropped
+        if total == 0:
+            return
+        type_name, message = failures[0] if failures else ("Error", "")
+        detail = f"{type_name}: {message}" if message else type_name
+        raise RuntimeError(
+            f"Walrus Memory auto-save failed ({total} call(s)): {detail}"
+        )
 
     def track_task(self, task: "asyncio.Task[Any]") -> None:
         self._tasks.add(task)
@@ -169,25 +226,44 @@ class _PendingSaves:
         thread.start()
 
     async def flush(self) -> None:
-        """Await every pending task, then join every pending thread."""
+        """Await every pending task, then join every pending thread.
+
+        Raises when any background auto-save failed. The chat call itself
+        still returned; this is the signal for a caller that waited. A call
+        made from ``on_save_error`` returns before draining, joining, or
+        clearing the failure being recorded.
+        """
+        if self._flush_called_from_callback():
+            return
         if self._tasks:
-            await asyncio.gather(*list(self._tasks), return_exceptions=True)
+            tasks = list(self._tasks)
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            for result in results:
+                self._note_result(result)
         self._join_threads()
+        self._raise_if_failed()
 
     def flush_sync(self) -> None:
-        """Join every pending thread. A pending Task belongs to the loop
-        that created it (e.g. an earlier `await llm.ainvoke(...)` call) and
-        cannot be awaited from a different one -- if any are still pending
-        here, the caller mixed sync and async entry points on the same
-        wrapped client, and this cleanup path can't safely drain them. Log
-        and continue draining threads rather than raise out of what's
-        usually shutdown code."""
+        """Join every pending thread and raise when a background auto-save failed.
+
+        A pending Task belongs to the loop that created it. When this thread
+        has no running loop, those tasks are awaited here. When it does, they
+        are logged and left for :meth:`flush`. A call made from
+        ``on_save_error`` returns before draining, joining, or clearing the
+        failure being recorded.
+        """
+        if self._flush_called_from_callback():
+            return
         if self._tasks:
-            async def _drain(tasks: "list[asyncio.Task[Any]]") -> None:
-                await asyncio.gather(*tasks, return_exceptions=True)
+            tasks = list(self._tasks)
+
+            async def _drain(pending_tasks: "list[asyncio.Task[Any]]") -> None:
+                results = await asyncio.gather(*pending_tasks, return_exceptions=True)
+                for result in results:
+                    self._note_result(result)
 
             try:
-                asyncio.run(_drain(list(self._tasks)))
+                asyncio.run(_drain(tasks))
             except (RuntimeError, ValueError):
                 logger.warning(
                     "Walrus Memory flush_sync() could not await %d pending "
@@ -197,10 +273,14 @@ class _PendingSaves:
                     len(self._tasks),
                 )
         self._join_threads()
+        self._raise_if_failed()
 
     def _join_threads(self) -> None:
+        current = threading.current_thread()
         with self._lock:
             threads, self._threads = self._threads, []
+            self._threads.extend(thread for thread in threads if thread is current)
+            threads = [thread for thread in threads if thread is not current]
         for thread in threads:
             thread.join()
 
@@ -291,6 +371,7 @@ def with_memwal_langchain(
     min_relevance: float = 0.3,
     debug: bool = False,
     env: Optional[str] = None,
+    on_save_error: Optional[Callable[[BaseException], None]] = None,
 ) -> "BaseChatModel":
     """Wrap a LangChain ``BaseChatModel`` with Walrus Memory management.
 
@@ -313,6 +394,9 @@ def with_memwal_langchain(
         debug: Enable debug logging.
         env: Optional relayer preset (``"prod"`` / ``"dev"`` / ``"staging"`` /
             ``"local"``). Same precedence as :meth:`MemWal.create`.
+        on_save_error: Called with the original exception when a background
+            auto-save fails. ``flush`` / ``flush_sync`` from this callback does
+            not clear that failure.
 
     Returns:
         A wrapped ``BaseChatModel`` that automatically uses Walrus Memory.
@@ -335,7 +419,7 @@ def with_memwal_langchain(
     )
 
     log = logger.debug if not debug else logger.warning
-    pending = _PendingSaves()
+    pending = _PendingSaves(on_save_error)
 
     original_agenerate = llm._agenerate
     original_generate = llm._generate
@@ -408,6 +492,7 @@ def with_memwal_langchain(
                 await memwal.analyze(user_text, namespace)
             except Exception as e:
                 log(f"[Walrus Memory] Auto-save failed: {e}")
+                pending.note_error(e)
 
     async def patched_agenerate(
         messages: List[List[BaseMessage]], *args: Any, **kwargs: Any
@@ -470,6 +555,7 @@ def with_memwal_openai(
     min_relevance: float = 0.3,
     debug: bool = False,
     env: Optional[str] = None,
+    on_save_error: Optional[Callable[[BaseException], None]] = None,
 ) -> Any:
     """Wrap an OpenAI client with Walrus Memory management.
 
@@ -494,6 +580,9 @@ def with_memwal_openai(
         debug: Enable debug logging.
         env: Optional relayer preset (``"prod"`` / ``"dev"`` / ``"staging"`` /
             ``"local"``). Same precedence as :meth:`MemWal.create`.
+        on_save_error: Called with the original exception when a background
+            auto-save fails. ``flush`` / ``flush_sync`` from this callback does
+            not clear that failure.
 
     Returns:
         The same client, with ``chat.completions.create`` wrapped to use Walrus Memory.
@@ -507,7 +596,7 @@ def with_memwal_openai(
     )
 
     log = logger.debug if not debug else logger.warning
-    pending = _PendingSaves()
+    pending = _PendingSaves(on_save_error)
 
     is_async = hasattr(client, "_async_client") or type(client).__name__ == "AsyncOpenAI"
 
@@ -572,6 +661,7 @@ def _wrap_async_openai(
                     await memwal.analyze(user_text, namespace)
                 except Exception as e:
                     log(f"[Walrus Memory] Auto-save failed: {e}")
+                    pending.note_error(e)
 
             _fire_and_forget(_analyze(), pending)
 
@@ -633,6 +723,7 @@ def _wrap_sync_openai(
                     _run_memwal(lambda: memwal.analyze(user_text, namespace))
                 except Exception as e:
                     log(f"[Walrus Memory] Auto-save failed: {e}")
+                    pending.note_error(e)
 
             pending.spawn_thread(_analyze)
 

{memwal-0.1.11.dev10 → memwal-0.1.11.dev11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "memwal"
-version = "0.1.11.dev10"
+version = "0.1.11.dev11"
 description = "Python SDK for Walrus Memory — Privacy-first AI memory with Ed25519 signing"
 readme = "README.md"
 license = "MIT"

{memwal-0.1.11.dev10 → memwal-0.1.11.dev11}/tests/test_middleware.py

@@ -1000,3 +1000,157 @@ class TestPendingSaves:
 
         time.sleep(0.05)
         assert len(pending._threads) == 0
+
+    def test_flush_raises_when_a_background_save_failed(self) -> None:
+        seen: list[BaseException] = []
+        pending = _PendingSaves(seen.append)
+        pending.note_error(RuntimeError("analyze down"))
+
+        async def go() -> None:
+            await pending.flush()
+
+        try:
+            asyncio.run(go())
+        except RuntimeError as err:
+            assert "auto-save failed" in str(err)
+        else:
+            raise AssertionError("flush did not raise")
+        assert len(seen) == 1
+
+    def test_flush_sync_raises_when_a_background_save_failed(self) -> None:
+        pending = _PendingSaves()
+        pending.note_error(RuntimeError("analyze down"))
+        try:
+            pending.flush_sync()
+        except RuntimeError as err:
+            assert "auto-save failed" in str(err)
+        else:
+            raise AssertionError("flush_sync did not raise")
+
+    def test_flush_sees_a_failing_task_without_note_error(self) -> None:
+        pending = _PendingSaves()
+
+        async def fail() -> None:
+            raise RuntimeError("analyze down")
+
+        async def go() -> None:
+            task = asyncio.create_task(fail())
+            pending.track_task(task)
+            await pending.flush()
+
+        try:
+            asyncio.run(go())
+        except RuntimeError as err:
+            assert "analyze down" in str(err)
+        else:
+            raise AssertionError("flush did not raise")
+
+    def test_callback_flush_does_not_swallow_the_failure(self) -> None:
+        pending = _PendingSaves()
+
+        def on_error(_err: BaseException) -> None:
+            pending.flush_sync()
+
+        pending._on_save_error = on_error
+        pending.note_error(RuntimeError("analyze down"))
+
+        async def go() -> None:
+            await pending.flush()
+
+        try:
+            asyncio.run(go())
+        except RuntimeError as err:
+            assert "analyze down" in str(err)
+        else:
+            raise AssertionError("flush did not raise")
+
+    def test_worker_callback_flush_sync_keeps_the_failure(self) -> None:
+        pending = _PendingSaves()
+        callback_error: list[str] = []
+        saw_self = []
+
+        def on_error(_err: BaseException) -> None:
+            try:
+                pending.flush_sync()
+            except Exception as err:
+                callback_error.append(repr(err))
+            with pending._lock:
+                saw_self.append(threading.current_thread() in pending._threads)
+
+        pending._on_save_error = on_error
+        done = threading.Event()
+
+        def fail() -> None:
+            pending.note_error(RuntimeError("analyze down"))
+            done.set()
+
+        pending.spawn_thread(fail)
+        assert done.wait(2)
+        assert callback_error == []
+        assert saw_self == [True]
+        try:
+            pending.flush_sync()
+        except RuntimeError as err:
+            assert "analyze down" in str(err)
+        else:
+            raise AssertionError("flush_sync did not raise")
+
+    def test_two_worker_callbacks_do_not_join_each_other(self) -> None:
+        """Overlapping on_save_error flushes must not join the other worker.
+
+        Each callback returns before drain, join, and raise. The later caller
+        flush still reports both failures.
+        """
+        pending = _PendingSaves()
+        entered = threading.Barrier(2)
+        callback_error: list[str] = []
+        done = threading.Barrier(3)
+
+        def on_error(_err: BaseException) -> None:
+            entered.wait(2)
+            try:
+                pending.flush_sync()
+            except Exception as err:
+                callback_error.append(repr(err))
+
+        pending._on_save_error = on_error
+
+        def fail(message: str):
+            def run() -> None:
+                pending.note_error(RuntimeError(message))
+                done.wait(2)
+            return run
+
+        pending.spawn_thread(fail("analyze down"))
+        pending.spawn_thread(fail("analyze also down"))
+        done.wait(2)
+        assert callback_error == []
+        try:
+            pending.flush_sync()
+        except RuntimeError as err:
+            assert "2 call" in str(err)
+            assert "analyze down" in str(err)
+        else:
+            raise AssertionError("flush_sync did not raise")
+
+    def test_flush_from_another_thread_still_raises_during_callback(self) -> None:
+        """The callback thread bails out. Any other flush still reports the failure."""
+        import time
+
+        pending = _PendingSaves()
+        started = threading.Event()
+
+        def on_error(_err: BaseException) -> None:
+            started.set()
+            time.sleep(0.2)
+
+        pending._on_save_error = on_error
+        pending.spawn_thread(lambda: pending.note_error(RuntimeError("analyze down")))
+        assert started.wait(2)
+        try:
+            pending.flush_sync()
+        except RuntimeError as err:
+            assert "analyze down" in str(err)
+        else:
+            raise AssertionError("flush_sync did not raise")
+