geny-executor 2.64.2__tar.gz → 2.64.3__tar.gz

{geny_executor-2.64.2 → geny_executor-2.64.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geny-executor
-Version: 2.64.2
+Version: 2.64.3
 Summary: Harness-engineered agent pipeline library with 21-stage dual-abstraction architecture, built on the Anthropic API
 Project-URL: Homepage, https://github.com/CocoRoF/geny-executor
 Project-URL: Repository, https://github.com/CocoRoF/geny-executor

{geny_executor-2.64.2 → geny_executor-2.64.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "geny-executor"
-version = "2.64.2"
+version = "2.64.3"
 description = "Harness-engineered agent pipeline library with 21-stage dual-abstraction architecture, built on the Anthropic API"
 readme = "README.md"
 license = "Apache-2.0"

geny_executor-2.64.3/src/geny_executor/memory/_offload.py

@@ -0,0 +1,40 @@
+"""Dedicated worker pool for the memory subsystem's internal offloads.
+
+Why not ``asyncio.to_thread``
+-----------------------------
+``LoopAgnosticLock`` parks CONTENDED acquirers in the loop's default
+thread pool (``to_thread(lock.acquire)``) so the loop stays free. If a
+lock/gate HOLDER then runs its own work via ``to_thread`` too, holder and
+waiters compete for the same bounded pool — and on small machines
+(GitHub CI runners: 2 vCPUs → default pool ≈ 6 workers) a burst of
+waiters can fill every slot while the holder's work item sits queued
+behind them forever: a starvation deadlock. Observed live: the 2.64.3
+release pipeline hung twice in ``Verify tests pass`` on exactly this
+shape (10 concurrent deletes → 9 pooled waiters + 1 queued build),
+while 32-core dev machines sailed through.
+
+Running the subsystem's own offloads on this small dedicated pool makes
+holder progress independent of waiter pressure: the holder finishes,
+releases, and the pooled waiters drain. Two workers are plenty — these
+are serialized-by-lock jobs (vault scan, sidecar build), not a fan-out.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from typing import Any, Callable, TypeVar
+
+T = TypeVar("T")
+
+_MEM_EXECUTOR = ThreadPoolExecutor(max_workers=2, thread_name_prefix="mem-offload")
+
+
+async def run_offloaded(fn: Callable[..., T], *args: Any, **kwargs: Any) -> T:
+    """Run ``fn`` on the memory subsystem's dedicated worker pool."""
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(_MEM_EXECUTOR, partial(fn, *args, **kwargs))
+
+
+__all__ = ["run_offloaded"]

{geny_executor-2.64.2 → geny_executor-2.64.3}/src/geny_executor/memory/providers/file/index_store.py

@@ -12,6 +12,7 @@ the format that `_FilesystemNotesStore` writes.
 
 from __future__ import annotations
 
+import asyncio
 import json
 import os
 import tempfile
@@ -21,6 +22,7 @@ from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
 from geny_executor.memory._locks import LoopAgnosticLock
+from geny_executor.memory._offload import run_offloaded
 from geny_executor.memory.provider import NoteGraph, NoteMeta, NoteOutline, NoteSummary
 from geny_executor.memory.providers.file.graph_edges import derive_graph_edges
 from geny_executor.memory.providers.file.layout import DirectoryLayout
@@ -78,6 +80,15 @@ class _FileIndexStore:
         # keyed by a cheap (filename, updated_at) signature of the vault.
         self._edges_sig: Optional[tuple] = None
         self._edges_cache: Optional[List[Dict[str, Any]]] = None
+        # Sidecar-refresh coalescing (2.64.3). A burst of writes/deletes
+        # (e.g. an observation prune sweep) used to run one full-vault
+        # payload build PER CHANGE, inline on the event loop — a 6k-note
+        # vault blocked the loop for tens of seconds and got the host
+        # watchdog-restarted mid-sweep. Changes now just mark their
+        # category dirty; whoever holds the gate services ALL accumulated
+        # marks with ONE snapshot + one worker-thread compute/write.
+        self._refresh_dirty: Set[str] = set()
+        self._refresh_gate = LoopAgnosticLock()
 
     def set_category_descriptions(self, descriptions: Dict[str, str]) -> None:
         """Late-set or replace the canonical description map. Called
@@ -100,9 +111,14 @@ class _FileIndexStore:
         the disk view consistent.
         """
         async with self._lock:
-            payload = await self._compute()
-        self._write_hierarchical_sidecars(payload, category=None)
-        return payload
+            notes = await self._notes.all()
+
+        def _work() -> Dict[str, Any]:
+            payload = self._payload_from_notes(notes)
+            self._write_hierarchical_sidecars(payload, category=None)
+            return payload
+
+        return await run_offloaded(_work)
 
     async def refresh_for_category(self, category: Optional[str]) -> None:
         """Incrementally refresh the on-disk sidecars affected by a
@@ -113,10 +129,36 @@ class _FileIndexStore:
 
         Other category shards are left untouched. Called after every
         ``NotesStore.write`` / ``update`` / ``delete``.
+
+        COALESCED + OFF-LOOP (2.64.3): concurrent callers mark their
+        category dirty and serialize on a gate — the gate holder services
+        every accumulated mark with ONE payload build, executed in a
+        worker thread so a large vault never stalls the event loop. A
+        burst of N deletes therefore costs ~2 builds, not N. Sidecars
+        are best-effort by contract; a torn read of a concurrently
+        mutated note is repaired by the next refresh.
         """
-        async with self._lock:
-            payload = await self._compute()
-        self._write_hierarchical_sidecars(payload, category=category)
+        self._refresh_dirty.add(category or "__all__")
+        async with self._refresh_gate:
+            if not self._refresh_dirty:
+                return  # a sibling gate-holder already serviced our mark
+            pending = set(self._refresh_dirty)
+            self._refresh_dirty.clear()
+            async with self._lock:
+                notes = await self._notes.all()
+
+            def _work() -> None:
+                payload = self._payload_from_notes(notes)
+                if "__all__" in pending:
+                    self._write_hierarchical_sidecars(payload, category=None)
+                    return
+                for cat in pending:
+                    self._write_hierarchical_sidecars(payload, category=cat)
+
+            # Dedicated pool: lock WAITERS park in the default to_thread pool
+            # (LoopAgnosticLock) — running the holder's build there too can
+            # starve-deadlock small pools (see memory/_offload.py).
+            await run_offloaded(_work)
 
     async def tag_counts(self) -> Dict[str, int]:
         payload = await self._cached_or_compute()
@@ -410,10 +452,16 @@ class _FileIndexStore:
         cache, so ``_compute`` is fast even on large vaults.
         """
         async with self._lock:
-            return await self._compute()
+            notes = await self._notes.all()
+        return await run_offloaded(self._payload_from_notes, notes)
 
     async def _compute(self) -> Dict[str, Any]:
         notes = await self._notes.all()
+        return self._payload_from_notes(notes)
+
+    def _payload_from_notes(self, notes) -> Dict[str, Any]:
+        """Pure payload build from a notes snapshot — sync by design so
+        heavy vaults can run it in a worker thread (2.64.3)."""
         files: Dict[str, Dict[str, Any]] = {}
         tag_map: Dict[str, List[str]] = {}
         link_graph: Dict[str, List[str]] = {}

{geny_executor-2.64.2 → geny_executor-2.64.3}/src/geny_executor/memory/providers/file/notes_store.py

@@ -19,6 +19,7 @@ from pathlib import Path
 from typing import Any, Awaitable, Callable, Dict, Iterable, List, Optional, Set, Tuple
 
 from geny_executor.memory._locks import LoopAgnosticLock
+from geny_executor.memory._offload import run_offloaded
 from geny_executor.memory.provider import (
     Importance,
     MemoryHooks,
@@ -466,7 +467,10 @@ class _FilesystemNotesStore(NotesHandle):
         # BLOCKS the host's event loop — health checks stop answering and the
         # process gets watchdog-killed mid-load. Run it in a worker thread;
         # callers already hold self._lock, so the swap below is single-flight.
-        cache = await asyncio.to_thread(self._scan_disk)
+        # Dedicated pool — the scan holder must never queue behind its own
+        # lock-waiters in the default to_thread pool (starvation deadlock on
+        # small machines; see memory/_offload.py).
+        cache = await run_offloaded(self._scan_disk)
         self._cache = cache
         self._explicit_links.clear()
         self._refresh_backlinks()

{geny_executor-2.64.2 → geny_executor-2.64.3}/src/geny_executor/memory/providers/file/vector_store.py

@@ -25,6 +25,8 @@ backend.
 
 from __future__ import annotations
 
+import hashlib
+
 import json
 import logging
 import math
@@ -133,10 +135,27 @@ class _FileVectorStore:
             return 0
         async with self._lock:
             await self._ensure_loaded()
-            texts = [text for _, text in items]
+            # Idempotent backfill (2.64.3): index_batch is the session-resume
+            # warm-up path — it used to re-embed EVERY note on every resume
+            # (6k-note vault ≈ minutes of embedding HTTP + real money, every
+            # 30-min idle-evict cycle). Rows persist a content sha, so only
+            # new/changed notes reach the embedder.
+            existing_sha = {
+                row.get("filename"): row.get("sha")
+                for row in self._rows
+                if row.get("sha")
+            }
+            todo = [
+                (ref, text)
+                for ref, text in items
+                if existing_sha.get(ref.filename) != _text_sha(text)
+            ]
+            if not todo:
+                return 0
+            texts = [text for _, text in todo]
             vectors = await self._embed_guarded(texts)
             added = 0
-            for (ref, text), vec in zip(items, vectors):
+            for (ref, text), vec in zip(todo, vectors):
                 self._validate_dim(vec)
                 removed = self._remove_by_filename(ref.filename)
                 self._vectors.append(vec)
@@ -423,6 +442,10 @@ class _FileVectorStore:
 # ── helpers ──────────────────────────────────────────────────────────
 
 
+def _text_sha(text: str) -> str:
+    return hashlib.sha1((text or "").encode("utf-8")).hexdigest()
+
+
 def _row_for(ref: NoteRef, text: str) -> Dict[str, Any]:
     return {
         "filename": ref.filename,
@@ -430,6 +453,10 @@ def _row_for(ref: NoteRef, text: str) -> Dict[str, Any]:
         "category": ref.category,
         "backend": ref.backend,
         "preview": (text or "")[:400],
+        # Content fingerprint — lets index_batch (session-resume backfill)
+        # skip re-embedding unchanged notes (2.64.3). Rows persisted before
+        # this field simply lack it and re-embed once, then carry it.
+        "sha": _text_sha(text),
     }
 
 

{geny_executor-2.64.2 → geny_executor-2.64.3}/src/geny_executor/memory/vector/qdrant_store.py

@@ -172,8 +172,53 @@ class QdrantVectorStore:
         return await self.index_document(ref, [DocumentChunk(text=text)])
 
     async def index_batch(self, items: Sequence[Tuple[NoteRef, str]]) -> int:
+        """Bulk (re)index — the session-resume warm-up path.
+
+        Idempotent since 2.64.3: points already carry ``content_sha1``, so
+        one payload-only scroll of the collection lets unchanged notes skip
+        the embedder entirely. Re-embedding the whole vault on every resume
+        (6k notes ≈ minutes of embedding HTTP, repeated every idle-evict
+        cycle) was pure waste. Scroll failure falls back to full indexing —
+        correctness never depends on the skip.
+        """
+        if not items:
+            return 0
+        existing: Dict[str, str] = {}
+        try:
+            await self._ensure_collection()
+            qdrant = self._get_qdrant()
+            offset = None
+            while True:
+                batch, offset = await qdrant.scroll(
+                    collection_name=self._collection,
+                    limit=1024,
+                    offset=offset,
+                    with_payload=["filename", "content_sha1", "chunk_index", "chunk_count"],
+                    with_vectors=False,
+                )
+                for point in batch or []:
+                    payload = getattr(point, "payload", None) or {}
+                    # Only single-chunk chunk-0 points represent a plain note
+                    # body — multi-chunk documents always re-index.
+                    if payload.get("chunk_index") == 0 and payload.get("chunk_count") == 1:
+                        fname = payload.get("filename")
+                        sha = payload.get("content_sha1")
+                        if fname and sha:
+                            existing[str(fname)] = str(sha)
+                if not offset:
+                    break
+        except Exception:  # noqa: BLE001 — skip-map is an optimization only
+            logger.debug("qdrant: sha scroll failed — full re-index", exc_info=True)
+            existing = {}
+
         total = 0
         for ref, text in items:
+            # Compare the sha of the UNMODIFIED text — index_document stores
+            # sha1(chunk.text) verbatim (strip() is only its emptiness test).
+            if (text or "").strip() and existing.get(ref.filename) == hashlib.sha1(
+                (text or "").encode("utf-8")
+            ).hexdigest():
+                continue
             total += await self.index(ref, text)
         return total
 

geny_executor-2.64.3/tests/unit/test_index_sidecar_coalescing.py

@@ -0,0 +1,162 @@
+"""Sidecar refresh must be coalesced and off-loop; batch re-index must skip
+unchanged notes.
+
+Field incident (2026-08-03, prod): an observation prune sweep deleted notes
+one-by-one and every delete ran a full-vault sidecar payload build inline on
+the event loop — a 6k-note vault blocked the loop for tens of seconds, the
+host was watchdog-restarted mid-sweep, and every resume re-embedded the whole
+vault (minutes of embedding HTTP per 30-min idle-evict cycle). These tests
+lock in the fixes.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import tempfile
+import time
+from pathlib import Path
+
+import pytest
+
+from geny_executor.memory.embedding import LocalHashEmbeddingClient
+from geny_executor.memory.provider import NoteDraft, Scope
+from geny_executor.memory.providers.file.provider import FileMemoryProvider
+
+
+def _provider(td: str) -> FileMemoryProvider:
+    return FileMemoryProvider(root=Path(td), scope=Scope.SESSION)
+
+
+@pytest.mark.asyncio
+async def test_burst_of_deletes_coalesces_payload_builds(monkeypatch):
+    with tempfile.TemporaryDirectory() as td:
+        p = _provider(td)
+        await p.initialize()
+        for i in range(12):
+            await p.notes().write(
+                NoteDraft(title=f"n{i}", body=f"b{i}", category="observations", filename=f"n{i}.md")
+            )
+
+        index = p.index()
+        builds = {"n": 0}
+        real_build = index._payload_from_notes
+
+        def counting_build(notes):
+            builds["n"] += 1
+            time.sleep(0.02)  # make the build observable
+            return real_build(notes)
+
+        monkeypatch.setattr(index, "_payload_from_notes", counting_build)
+
+        # Concurrent burst — the prune-sweep shape.
+        await asyncio.gather(*(p.notes().delete(f"n{i}.md") for i in range(10)))
+
+        # Coalescing: a 10-delete burst must not run 10 builds. (≤ half is
+        # the contract; typically 2-3 depending on gate timing.)
+        assert builds["n"] <= 5, f"sidecar builds not coalesced: {builds['n']} for 10 deletes"
+
+
+@pytest.mark.asyncio
+async def test_sidecar_refresh_keeps_loop_responsive(monkeypatch):
+    with tempfile.TemporaryDirectory() as td:
+        p = _provider(td)
+        await p.initialize()
+        for i in range(5):
+            await p.notes().write(
+                NoteDraft(title=f"m{i}", body=f"b{i}", category="topics", filename=f"m{i}.md")
+            )
+        index = p.index()
+        real_build = index._payload_from_notes
+
+        def slow_build(notes):
+            time.sleep(0.3)  # heavy vault stand-in — MUST run off-loop
+            return real_build(notes)
+
+        monkeypatch.setattr(index, "_payload_from_notes", slow_build)
+
+        beats = 0
+
+        async def heartbeat():
+            nonlocal beats
+            while True:
+                beats += 1
+                await asyncio.sleep(0.01)
+
+        hb = asyncio.create_task(heartbeat())
+        try:
+            await p.notes().delete("m0.md")  # triggers refresh (slow build)
+        finally:
+            hb.cancel()
+        assert beats >= 10, f"event loop starved during sidecar refresh (beats={beats})"
+
+
+@pytest.mark.asyncio
+async def test_index_batch_skips_unchanged_notes():
+    from geny_executor.memory.provider import NoteRef
+
+    with tempfile.TemporaryDirectory() as td:
+        p = FileMemoryProvider(
+            root=Path(td), scope=Scope.SESSION,
+            embedding_client=LocalHashEmbeddingClient(),
+        )
+        await p.initialize()
+        vec = p.vector()
+        assert vec is not None
+
+        embeds = {"n": 0}
+        real_embed = vec._embed_guarded
+
+        async def counting_embed(texts):
+            embeds["n"] += len(texts)
+            return await real_embed(texts)
+
+        vec._embed_guarded = counting_embed  # type: ignore[method-assign]
+
+        items = [
+            (NoteRef(filename=f"k{i}.md", scope=Scope.SESSION, backend="filesystem"), f"body {i}")
+            for i in range(6)
+        ]
+        await vec.index_batch(items)
+        first = embeds["n"]
+        assert first == 6
+
+        # Same content again — the resume-warm-up shape: zero embeds.
+        await vec.index_batch(items)
+        assert embeds["n"] == first, "unchanged notes were re-embedded"
+
+        # One changed note — exactly one embed.
+        items[3] = (items[3][0], "body 3 CHANGED")
+        await vec.index_batch(items)
+        assert embeds["n"] == first + 1
+
+
+@pytest.mark.asyncio
+async def test_burst_survives_tiny_default_executor():
+    """CI-shape regression: on a 2-vCPU runner the default to_thread pool is
+    ~6 workers; a delete burst filled every slot with LOCK WAITERS while the
+    gate holder's sidecar build sat queued behind them — starvation deadlock
+    (both 2.64.3 release pipelines hung exactly here). Memory offloads now
+    run on a dedicated pool, so the holder always makes progress no matter
+    how many waiters the default pool holds."""
+    import concurrent.futures
+
+    loop = asyncio.get_running_loop()
+    prev = getattr(loop, "_default_executor", None)
+    tiny = concurrent.futures.ThreadPoolExecutor(max_workers=2)
+    loop.set_default_executor(tiny)
+    try:
+        with tempfile.TemporaryDirectory() as td:
+            p = _provider(td)
+            await p.initialize()
+            for i in range(12):
+                await p.notes().write(
+                    NoteDraft(title=f"t{i}", body=f"b{i}", category="observations", filename=f"t{i}.md")
+                )
+            await asyncio.wait_for(
+                asyncio.gather(*(p.notes().delete(f"t{i}.md") for i in range(12))),
+                timeout=30,
+            )
+    finally:
+        if prev is not None:
+            loop.set_default_executor(prev)
+        tiny.shutdown(wait=False)