memorytalk 0.8.0__tar.gz → 0.8.2__tar.gz

{memorytalk-0.8.0 → memorytalk-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memorytalk
-Version: 0.8.0
+Version: 0.8.2
 Summary: Persistent cross-session memory for AI agents — Talk-Card architecture with forum-dynamics sinking/floating (v3)
 License-Expression: Apache-2.0
 Requires-Python: >=3.10

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/api/__init__.py

@@ -32,6 +32,7 @@ from memorytalk.service import (
     RecallService, ReviewService,
 )
 from memorytalk.service.backfill import IndexBackfill
+from memorytalk.service.index_buffer import IndexWriteBuffer
 from memorytalk.service.search import SearchService
 from memorytalk.service.sync import SyncWatcher
 
@@ -71,9 +72,22 @@ def create_app(config: Config | None = None) -> FastAPI:
         app.state.vectors = vectors
         app.state.embedder = embedder
         app.state.events = events
+        # IndexWriteBuffer aggregates LanceDB inserts across sessions so
+        # one ``table.add()`` carries many embedder batches' worth of
+        # rows. Without it the ingest path creates one fragment per
+        # embedder batch (10 with DashScope) → vector search eventually
+        # EMFILEs on fd ceiling. See service/index_buffer.py and
+        # docs/issue #4 §4.3.
+        app.state.index_buffer = IndexWriteBuffer(
+            vectors=vectors, db=db,
+            flush_rows=config.settings.index.lance_flush_rows,
+            flush_interval_seconds=config.settings.index.lance_flush_interval_seconds,
+        )
+        app.state.index_buffer.start()
         app.state.read = ReadService(db=db, events=events)
         app.state.ingest = IngestService(
             db=db, vectors=vectors, embedder=embedder, events=events,
+            index_buffer=app.state.index_buffer,
         )
         app.state.sync_checkpoints = sync_checkpoints
         app.state.sync = SyncWatcher(
@@ -108,8 +122,15 @@ def create_app(config: Config | None = None) -> FastAPI:
         # lifespan shutdown.
         app.state.backfill = IndexBackfill(
             db=db, vectors=vectors, embedder=embedder,
+            index_buffer=app.state.index_buffer,
         )
         app.state.backfill.start()
+        # Guaranteed one-shot compaction on every boot — grinds down the
+        # append-only fragment pile (cause of EMFILE in vector search)
+        # so a restart always makes progress. Side path off the re-embed
+        # loop: gated only on vectors, runs in the background, never
+        # blocks startup. See IndexBackfill.trigger_startup_compaction.
+        app.state.backfill.trigger_startup_compaction()
 
         yield
 
@@ -123,6 +144,12 @@ def create_app(config: Config | None = None) -> FastAPI:
             await app.state.backfill.stop()
         except Exception:
             pass
+        # Drain in-flight LanceDB writes before tearing down the DB
+        # — otherwise pending vectors are lost on shutdown.
+        try:
+            await app.state.index_buffer.stop()
+        except Exception:
+            pass
         await db.close()
         await sync_checkpoints.close()
 

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/api/search.py

@@ -21,7 +21,8 @@ async def post_search(payload: SearchRequest, request: Request) -> SearchRespons
             query=payload.query or "",
             where=payload.where,
             top_k=payload.top_k,
-            show_all=payload.show_all,
+            recall_mode=payload.recall_mode,
+            recall_session_id=payload.recall_session_id,
         )
     except DSLError as e:
         raise HTTPException(status_code=400, detail=str(e))

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/api/sync.py

@@ -12,6 +12,50 @@ from fastapi import APIRouter, Query, Request
 router = APIRouter()
 
 
+def _gather_lance_health(state) -> dict:
+    """Collect LanceDB-layer observability for ``index.lance``.
+
+    Pulls from three sources: the IndexWriteBuffer (write pipeline),
+    IndexBackfill (compaction cadence), and LanceStore (EMFILE
+    recovery count). All fields default to safe zeros / None when the
+    corresponding component is absent so a partially-disabled boot
+    still returns a well-shaped response.
+    """
+    buf = getattr(state, "index_buffer", None)
+    backfill = getattr(state, "backfill", None)
+    vectors = getattr(state, "vectors", None)
+
+    soft = hard = None
+    try:
+        import resource
+        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    except (ImportError, OSError):
+        # Windows / sandboxed envs — leave None so the field's
+        # absence is the signal.
+        pass
+
+    return {
+        "pending_vector_rows": (buf.pending_rows if buf is not None else 0),
+        "last_flush_at": (buf.last_flush_at_iso if buf is not None else None),
+        "last_flush_error": (buf.last_flush_error if buf is not None else None),
+        "flush_count_since_boot": (buf.flush_count if buf is not None else 0),
+        "last_compaction_at": (
+            backfill.last_compact_at_iso if backfill is not None else None
+        ),
+        "last_compaction_error": (
+            backfill.last_compact_error if backfill is not None else None
+        ),
+        "emfile_recoveries_since_boot": (
+            vectors.emfile_recoveries if vectors is not None else 0
+        ),
+        "last_emfile_at": (
+            vectors.last_emfile_at_iso if vectors is not None else None
+        ),
+        "fd_soft_limit": soft,
+        "fd_hard_limit": hard,
+    }
+
+
 @router.get("/sync/status")
 async def get_sync_status(request: Request, limit: int = Query(5, ge=0, le=20)):
     config = request.app.state.config
@@ -31,6 +75,7 @@ async def get_sync_status(request: Request, limit: int = Query(5, ge=0, le=20)):
     index["last_index_error"] = (
         backfill.last_error if backfill is not None else None
     )
+    index["lance"] = _gather_lance_health(request.app.state)
 
     if not config.settings.sync.enabled:
         return {"status": "disabled", "index": index}

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/cli/__init__.py

@@ -15,7 +15,7 @@ import click
 @click.option(
     "--no-pager", "no_pager", is_flag=True, default=False,
     help="Disable the scrollable pager (only applies to commands that "
-         "opt-in — currently just `read`). Equivalent to NO_PAGER=1.",
+         "opt-in — currently `read` and `search`). Equivalent to NO_PAGER=1.",
 )
 def main(no_pager: bool) -> None:
     """memory.talk v3."""

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/cli/_format.py

@@ -354,15 +354,12 @@ def fmt_recall(payload: dict) -> str:
 def fmt_search(payload: dict) -> str:
     query = payload.get("query") or ""
     count = payload.get("count", 0)
-    hidden = int(payload.get("hidden_count") or 0)
     sid = payload.get("search_id", "")
     header = f"`search_id={sid}` · {count} results"
-    if hidden:
-        header += f" · {hidden} hidden"
     parts: list[str] = [f"# search: {query}" if query else "# search",
                         "", header, ""]
 
-    if count == 0 and hidden == 0:
+    if count == 0:
         return "\n".join(parts) + "\n"
 
     for entry in payload.get("results") or []:
@@ -374,13 +371,6 @@ def fmt_search(payload: dict) -> str:
             parts.append(_fmt_search_session(entry))
         parts.append("")
 
-    if hidden:
-        parts.append(
-            f"_({hidden} weak result{'s' if hidden != 1 else ''} hidden "
-            "by strong-floor filter — pass `--all` to see)_"
-        )
-        parts.append("")
-
     return "\n".join(parts).rstrip() + "\n"
 
 

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/cli/_render.py

@@ -12,7 +12,7 @@ Errors:
 - Markdown mode → ``**error:** <msg>`` to stderr, exit 1
 - JSON mode    → ``{"error": ...}`` to stdout, exit 1
 
-Pager (opt-in per command, currently only ``read``):
+Pager (opt-in per command — currently ``read`` and ``search``):
 
 - ``emit_md_paged`` wraps rich rendering in a less-style pager when
   both stdin and stdout are TTYs. Subprocess / pipe / ``--json`` paths

memorytalk-0.8.2/memorytalk/cli/search.py

@@ -0,0 +1,76 @@
+"""CLI: search <query> [--where DSL] [--top-k N] [--json]."""
+from __future__ import annotations
+import sys
+
+import click
+
+from memorytalk.cli._format import fmt_error, fmt_search
+from memorytalk.cli._http import ApiError, api, extract_error_message
+from memorytalk.cli._render import (
+    emit_json, emit_json_err, emit_md_err, emit_md_paged,
+)
+from memorytalk.config import Config
+
+
+@click.command("search")
+@click.argument("query", required=False, default="")
+@click.option("--where", "-w", "where", type=str, default=None,
+              help="DSL filter (see docs/cli/v3/search.md#DSL)")
+@click.option("--top-k", "top_k", type=int, default=None,
+              help="Total result cap (default = settings.search.default_top_k)")
+@click.option("--recall", "recall_mode", is_flag=True, default=False,
+              help="Debug lens: rank like `recall` (cards-only, raw RRF, "
+                   "no ranking_formula). Combine with --session to also "
+                   "preview that session's recall_log dedup. Read-only — "
+                   "does NOT bump recall_count or write recall_log.")
+@click.option("--session", "session_id", type=str, default=None,
+              help="Session id for recall-mode dedup (only meaningful "
+                   "with --recall).")
+@click.option("--json", "json_out", is_flag=True, default=False, help="Emit JSON")
+def search(
+    query: str, where: str | None, top_k: int | None,
+    recall_mode: bool, session_id: str | None, json_out: bool,
+) -> None:
+    """Hybrid FTS + vector search across cards and sessions."""
+    cfg = Config()
+    body: dict = {"query": query or ""}
+    if where:
+        body["where"] = where
+    if top_k is not None:
+        body["top_k"] = top_k
+    if recall_mode:
+        body["recall_mode"] = True
+    if session_id:
+        if not recall_mode:
+            # --session without --recall is a probable mistake; flag it
+            # rather than silently dropping the field.
+            emit_md_err(fmt_error(
+                "--session only takes effect with --recall (it scopes "
+                "the recall-log dedup preview)"
+            ))
+            sys.exit(1)
+        body["recall_session_id"] = session_id
+    try:
+        result = api("POST", "/v3/search", cfg, json_body=body)
+    except ApiError as e:
+        if json_out:
+            emit_json_err(e.payload)
+        else:
+            emit_md_err(fmt_error(extract_error_message(e.payload)))
+        sys.exit(1)
+    except Exception as e:
+        if json_out:
+            emit_json_err(str(e))
+        else:
+            emit_md_err(fmt_error(f"cannot reach server: {e}"))
+        sys.exit(1)
+
+    if json_out:
+        emit_json(result)
+    else:
+        # Long result blocks (cards + per-session hit fences + ctx
+        # windows) routinely exceed a terminal page; route through the
+        # same less-style pager that ``read`` uses. Subprocess / pipe /
+        # ``--no-pager`` / ``--json`` fall back to plain output — see
+        # emit_md_paged docstring.
+        emit_md_paged(fmt_search(result))

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/config.py

@@ -16,10 +16,17 @@ from pathlib import Path
 from pydantic import BaseModel, ConfigDict
 
 
-_DEFAULT_RANKING_FORMULA = (
-    "relevance + 0.1 * (review_up - review_down) "
-    "+ 0.02 * log(read_count + 1) - 0.005 * age_days"
-)
+_DEFAULT_RANKING_FORMULA = "relevance"
+# ↑ As of 0.8.x: explicit search defaults to pure relevance (raw RRF
+# score from LanceDB hybrid recall). Earlier defaults mixed in forum-
+# dynamics signals (review_up - review_down + log(read_count+1) - age),
+# which made identifier-style queries like `vvp-ai` unreliable — the
+# strongest text match could rank below weakly-matched high-read
+# cards. The forum-stats counters are still maintained on every card
+# and remain queryable via ``--where 'DSL'`` (filter); the *ranking*
+# is just left as relevance. Users who want forum dynamics back can
+# set ``settings.search.ranking_formula`` to a richer expression like
+# the old default. See docs/cli/v3/search.md.
 
 
 class ConfigValidationError(RuntimeError):
@@ -101,6 +108,26 @@ class ExploreConfig(BaseModel):
     auto_default_limit: int = 5
 
 
+class IndexConfig(BaseModel):
+    """Vector index write tuning (0.8.x — issue #4 §4.3 fix).
+
+    Decouples LanceDB ``table.add()`` batch size from the embedder's
+    per-request cap. Embedding still batches small (API limit); these
+    knobs control how the embedded rows aggregate before they hit
+    LanceDB, which directly drives fragment count and downstream fd
+    pressure on search.
+    """
+    # Row count that triggers a synchronous flush. 500 is a balance
+    # between fragment-count savings (50× fewer fragments than the
+    # naive embedder-batch-sized writes at DashScope's 10-cap) and
+    # search-visibility latency for newly-ingested rounds.
+    lance_flush_rows: int = 500
+    # Wall-clock interval for the background flusher — catches the
+    # last partial batch when ingest is bursty then idle. 0 disables
+    # the background tick (tests use this).
+    lance_flush_interval_seconds: float = 30.0
+
+
 class Settings(BaseModel):
     server: ServerConfig = ServerConfig()
     vector: ProviderConfig = ProviderConfig(provider="lancedb")
@@ -110,6 +137,7 @@ class Settings(BaseModel):
     recall: RecallConfig = RecallConfig()
     sync: SyncConfig = SyncConfig()
     explore: ExploreConfig = ExploreConfig()
+    index: IndexConfig = IndexConfig()
 
 
 def _default_data_root() -> Path:

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/provider/lancedb.py

@@ -11,12 +11,28 @@ in search results come from the ``cards`` table. SQLite holds zero search
 state; jsonl files hold zero search state.
 """
 from __future__ import annotations
+import asyncio
+import datetime as _dt
+import logging
 from pathlib import Path
 from typing import Optional
 
 import pyarrow as pa
 
 
+_log = logging.getLogger("memorytalk.lancedb")
+
+
+def _is_emfile(exc: BaseException) -> bool:
+    """Recognize Lance's wrapped EMFILE — comes through as a
+    ``RuntimeError`` whose ``str()`` contains "Too many open files".
+    We can't match on errno because Lance wraps the OS error inside
+    its own ``LanceError(IO)`` before raising. String match is fragile
+    but it's the only signal Lance gives us on this path."""
+    msg = str(exc)
+    return "Too many open files" in msg or "(os error 24)" in msg
+
+
 def _segment(text: str) -> str:
     """jieba 预分词,空格连接(jieba.cut 同步,亚毫秒级)。"""
     import jieba
@@ -38,6 +54,16 @@ class LanceStore:
         self.db = db
         self.data_dir = data_dir
         self.dim = dim
+        # Per-table "FTS index confirmed present" memo. Avoids a
+        # ``list_indices()`` round trip on every search call once we've
+        # verified the index exists. Invalidated only on process restart
+        # (we don't drop FTS indices at runtime).
+        self._fts_index_known: set[str] = set()
+        # EMFILE recovery state — see _recover_from_emfile / _search_with_recovery.
+        self._recovery_lock = asyncio.Lock()
+        self.emfile_recoveries: int = 0
+        self.last_emfile_at_iso: str | None = None
+        self.last_recovery_error: str | None = None
         self._cards_schema = pa.schema([
             pa.field("card_id", pa.string()),
             pa.field("text", pa.string()),
@@ -112,33 +138,86 @@ class LanceStore:
         table = await self.db.open_table(self.ROUNDS)
         await table.delete(f"session_id = '{session_id}'")
 
+    # ────────── compaction ──────────
+
+    async def optimize(self, table_name: str) -> dict:
+        """Compact small fragments + prune old dataset versions.
+
+        Why this is load-bearing: the ingest / backfill path is
+        append-only — every embedder batch is one ``table.add`` →
+        one new fragment + one new dataset version (manifest + txn
+        file). Left unchecked these accumulate without bound (tens of
+        thousands of files in production). Search has **no vector ANN
+        index** (the only index we build is FTS), so vector queries
+        flat-scan every fragment, opening every fragment's files at
+        once — past a few thousand fragments this blows the process
+        file-descriptor ceiling (EMFILE / "Too many open files").
+
+        ``optimize`` is LanceDB's VACUUM: merge fragments, fold new
+        data into indices, and prune old versions. We pass
+        ``cleanup_older_than=timedelta(0)`` so **every version except
+        the latest is removed** — that's what actually reclaims the
+        manifest/txn file explosion (plain compaction merges data but
+        leaves the old versions' files around until pruned). Trade-off:
+        dataset time-travel history is discarded; v3 doesn't use it.
+
+        ``delete_unverified`` stays at its safe default (False) so a
+        concurrent ingest / backfill write in flight can't be corrupted.
+
+        No-op (returns ``skipped``) when the table doesn't exist yet.
+        """
+        import datetime as _dt
+
+        if not await self._exists(table_name):
+            return {"table": table_name, "skipped": "missing"}
+        table = await self.db.open_table(table_name)
+        stats = await table.optimize(cleanup_older_than=_dt.timedelta(0))
+        # OptimizeStats shape drifts across lancedb versions; don't
+        # hard-depend on field names — stringify for the caller's log.
+        return {"table": table_name, "stats": str(stats)}
+
     # ────────── FTS index maintenance ──────────
 
     async def ensure_fts_index(self, table_name: str) -> None:
         """Create the FTS index on the ``text`` column if absent.
 
-        LanceDB's hybrid search needs an FTS index on the text column.
-        Calling this once before queries is enough (the index is shared
-        across queries; LanceDB picks up new rows automatically). Cheap
-        no-op when the index already exists.
+        Idempotent + memoized: once we've confirmed an FTS index covers
+        ``text`` for a given table in this process, future calls are
+        free. We don't drop indices at runtime, so the memo can't go
+        stale within a single process lifetime — invalidation = restart.
+
+        **Exception handling note (issue #4 §4.2 fix):** earlier this
+        function swallowed any error from ``list_indices()`` and
+        fell through to ``create_index(..., replace=True)``. Under
+        EMFILE the swallowed list call was followed by a fresh index
+        build, *adding* pressure exactly when the process was already
+        over its fd quota. Now: a successful ``list_indices()`` that
+        returns no ``text`` index is the only signal to create; any
+        IO exception from ``list_indices()`` propagates so the upstream
+        EMFILE recovery path can take over instead of compounding.
         """
+        if table_name in self._fts_index_known:
+            return
         if not await self._exists(table_name):
             return
         from lancedb.index import FTS
         table = await self.db.open_table(table_name)
-        try:
-            indices = await table.list_indices()
-            for idx in indices:
-                cols = getattr(idx, "columns", None) or []
-                if "text" in cols:
-                    return  # already indexed
-        except Exception:
-            pass  # treat as "no index" and create one
-        # whitespace tokenizer because ingest already segments via jieba.
+        # Let list_indices' exceptions bubble — see docstring.
+        indices = await table.list_indices()
+        for idx in indices:
+            cols = getattr(idx, "columns", None) or []
+            if "text" in cols:
+                self._fts_index_known.add(table_name)
+                return
+        # Confirmed absent — create. ``replace=False`` so a concurrent
+        # creator can't race us into a double build; if that ever fires
+        # the second caller gets a clear error rather than a silent
+        # second-rebuild storm.
         await table.create_index(
             "text", config=FTS(base_tokenizer="whitespace", with_position=True),
-            replace=True,
+            replace=False,
         )
+        self._fts_index_known.add(table_name)
 
     # ────────── search ──────────
 
@@ -155,10 +234,9 @@ class LanceStore:
         text/vector but callers usually just need card_id + relevance).
         Empty query → vector-only; no query and no vector → empty result.
         """
-        if not await self._exists(self.CARDS):
-            return []
-        table = await self.db.open_table(self.CARDS)
-        return await _run_hybrid(table, query, vector, top_k, where)
+        return await self._search_with_recovery(
+            self.CARDS, query, vector, top_k, where,
+        )
 
     async def search_rounds(
         self,
@@ -173,10 +251,88 @@ class LanceStore:
         responsible for aggregating per session, dereffing the text from
         jsonl for display, etc.
         """
-        if not await self._exists(self.ROUNDS):
+        return await self._search_with_recovery(
+            self.ROUNDS, query, vector, top_k, where,
+        )
+
+    # ────────── EMFILE recovery (issue #4 §6.2 fix) ──────────
+
+    async def _search_with_recovery(
+        self, table_name: str, query: str,
+        vector: list[float] | None, top_k: int, where: str | None,
+    ) -> list[dict]:
+        """Run a hybrid search; on EMFILE, drive a recovery once + retry.
+
+        Recovery (compaction + connection reset) is necessary because:
+        - compaction reclaims fragments on disk → fewer files to open;
+        - the in-process LanceDB readers hold fds to files Compaction
+          unlinked → only a fresh ``connect_async`` releases those.
+
+        Retry is gated to exactly one attempt: if the post-recovery
+        query still EMFILEs the underlying fragment / fd-budget mismatch
+        is past what we can fix in-process, and the original error
+        propagates as a 500 — operator action (restart, raise ulimit)
+        is required.
+        """
+        if not await self._exists(table_name):
             return []
-        table = await self.db.open_table(self.ROUNDS)
-        return await _run_hybrid(table, query, vector, top_k, where)
+        try:
+            table = await self.db.open_table(table_name)
+            return await _run_hybrid(table, query, vector, top_k, where)
+        except Exception as e:
+            if not _is_emfile(e):
+                raise
+            _log.warning(
+                "EMFILE on search table=%s; triggering recovery", table_name,
+            )
+            await self._recover_from_emfile()
+            # Single retry — see docstring.
+            if not await self._exists(table_name):
+                return []
+            table = await self.db.open_table(table_name)
+            return await _run_hybrid(table, query, vector, top_k, where)
+
+    async def _recover_from_emfile(self) -> None:
+        """Compact both tables + reset the LanceDB connection.
+
+        Lock-protected so concurrent EMFILE-ing requests don't pile up
+        N recoveries. The first request through the lock does the work;
+        followers see ``emfile_recoveries`` advanced and skip — they
+        proceed straight to retry, which now sees a fresh connection.
+        """
+        gen_before = self.emfile_recoveries
+        async with self._recovery_lock:
+            if self.emfile_recoveries > gen_before:
+                return  # someone else recovered while we waited
+            # 1. Compact — best-effort; failure here doesn't block retry.
+            for table_name in (self.ROUNDS, self.CARDS):
+                try:
+                    await self.optimize(table_name)
+                except Exception as e:
+                    _log.exception(
+                        "optimize during EMFILE recovery failed table=%s",
+                        table_name,
+                    )
+                    self.last_recovery_error = (
+                        f"optimize {table_name}: {e}"
+                    )
+            # 2. Reset connection — closes the held fds. Without this,
+            # post-compaction the process is still pinned to old files.
+            try:
+                import lancedb
+                try:
+                    await self.db.close()
+                except Exception:
+                    pass  # already closed / unsupported — best effort
+                self.db = await lancedb.connect_async(str(self.data_dir))
+            except Exception as e:
+                _log.exception("connection reset during EMFILE recovery failed")
+                self.last_recovery_error = f"reconnect: {e}"
+                raise
+            self.emfile_recoveries += 1
+            self.last_emfile_at_iso = _dt.datetime.now(_dt.UTC).isoformat(
+                timespec="seconds",
+            ).replace("+00:00", "Z")
 
 
 async def _run_hybrid(

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/repository/schema.py

@@ -125,6 +125,13 @@ DDL = [
         query          TEXT NOT NULL,
         where_dsl      TEXT,
         top_k          INTEGER NOT NULL,
+        mode           TEXT NOT NULL DEFAULT 'search',
+                                            -- 0.8.x: 'search' or 'recall'.
+                                            -- Audit replay distinguishes
+                                            -- normal queries from the
+                                            -- `search --recall` debug
+                                            -- lens (different ranking
+                                            -- semantics).
         created_at     TEXT NOT NULL,
         response_json  TEXT NOT NULL
     )
@@ -202,6 +209,17 @@ async def _additive_migrations(conn: aiosqlite.Connection) -> None:
             "ALTER TABLE cards ADD COLUMN tags TEXT NOT NULL DEFAULT '{}'"
         )
 
+    # 1f. ``mode`` column on search_log (0.8.x — `search --recall`
+    #     audit needs to distinguish lens). Old rows default to
+    #     'search', which matches historical behavior.
+    async with conn.execute("PRAGMA table_info(search_log)") as cursor:
+        slog_cols = {row[1] for row in await cursor.fetchall()}
+    if slog_cols and "mode" not in slog_cols:
+        await conn.execute(
+            "ALTER TABLE search_log ADD COLUMN "
+            "mode TEXT NOT NULL DEFAULT 'search'"
+        )
+
     # 2. If the legacy ``rounds_index`` table is around, derive
     #    last_round_id from it (max-idx round per session), then drop it.
     async with conn.execute(

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/repository/search_log.py

@@ -21,12 +21,13 @@ class SearchLogStore:
         top_k: int,
         created_at: str,
         response: dict,
+        mode: str = "search",
     ) -> None:
         await self.conn.execute(
             "INSERT INTO search_log "
-            "(search_id, query, where_dsl, top_k, created_at, response_json) "
-            "VALUES (?, ?, ?, ?, ?, ?)",
-            (search_id, query, where_dsl, top_k, created_at,
+            "(search_id, query, where_dsl, top_k, mode, created_at, response_json) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?)",
+            (search_id, query, where_dsl, top_k, mode, created_at,
              json.dumps(response, ensure_ascii=False)),
         )
         await self.conn.commit()

{memorytalk-0.8.0 → memorytalk-0.8.2}/memorytalk/schemas/search.py

@@ -11,12 +11,15 @@ class SearchRequest(BaseModel):
     query: str = ""
     where: str | None = None
     top_k: int | None = None  # falls back to settings.search.default_top_k
-    # When false (default), apply per-type "strong-floor" filter: if any
-    # result of that type clears the floor, hide everything below it; if
-    # nothing clears the floor, return the whole bucket. To inspect what
-    # was filtered, re-issue with show_all=true. See service/search.py
-    # for the hardcoded floors (session 0.02, card 0.1) and rationale.
-    show_all: bool = False
+    # ── 0.8.x: --recall debug lens ──────────────────────────────────
+    # When True, the search service mimics ``RecallService``:
+    # cards-only, raw RRF relevance (no ranking_formula), and (when
+    # ``recall_session_id`` is supplied) dedup against that session's
+    # recall_log. Strictly read-only — does NOT bump recall_count or
+    # write recall_log entries. Use it to tune queries against the
+    # live recall behavior without polluting state.
+    recall_mode: bool = False
+    recall_session_id: str | None = None
 
 
 class _SessionHitContext(BaseModel):
@@ -64,7 +67,11 @@ class SearchResponse(BaseModel):
     search_id: str
     query: str
     count: int
-    # Number of results filtered out by the strong-floor rule (sum across
-    # both types). 0 when ``show_all=true`` or when nothing was filtered.
-    hidden_count: int = 0
+    # 0.8.x — discriminator between normal search and the --recall
+    # debug lens, so audit / programmatic consumers can tell them apart
+    # without comparing top-level body shapes.
+    mode: Literal["search", "recall"] = "search"
+    # Set only on recall-mode + session_id supplied. Lets the JSON
+    # consumer see the dedup scope used to produce these results.
+    session_id: str | None = None
     results: list[CardResult | SessionResult] = Field(default_factory=list)