lean-memory-console 0.2.0__py3-none-any.whl

lean_memory_console/inspect_sql.py

@@ -0,0 +1,321 @@
+"""Read-only enumeration SQL over the engine's per-namespace DBs (spec §7),
+plus the fail-loud schema/sanitizer tripwires (spec §13).
+
+Connections open with file:...?mode=ro ALWAYS; immutable=1 is a per-request
+fallback tried ONLY after mode=ro raises OperationalError (spec §7). Column
+names below are verbatim from the installed lean_memory store/schema.py.
+
+Task 4 extends this module with list_facts/get_fact/list_episodes/
+get_episode/list_entities.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.resources
+import sqlite3
+from pathlib import Path
+
+from .config import sanitize_namespace
+
+
+def open_ro(path: Path) -> sqlite3.Connection:
+    """Read-only engine connection. mode=ro first; immutable=1 only on
+    OperationalError (genuinely read-only media / error-14 path, spec §7)."""
+    uri = f"file:{path}?mode=ro"
+    try:
+        conn = sqlite3.connect(uri, uri=True)
+    except sqlite3.OperationalError:
+        conn = sqlite3.connect(f"file:{path}?immutable=1", uri=True)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA busy_timeout=5000")
+    return conn
+
+
+def _fts_query(text: str) -> str:
+    """OR-query of QUOTED alnum terms (mirrors engine sqlite_store._fts_query so
+    the console's text filter matches the same tokens the engine indexes).
+
+    Quoting matters: FTS5 treats bare AND/OR/NOT/NEAR as operators, so an
+    unquoted query with such a token raises a syntax error. A quoted term is an
+    inert string literal, still matched case-insensitively by the tokenizer;
+    terms are alnum-only after the scrub, so no embedded quote can break out."""
+    terms = [
+        t for t in "".join(c if c.isalnum() else " " for c in text).split() if t
+    ]
+    if not terms:
+        return '""'
+    return " OR ".join(f'"{t}"' for t in terms)
+
+
+def list_namespaces(data_root: Path, event_log) -> list[dict]:
+    """Discover *.db (skipping _*.db), returning per-namespace counts.
+    Bare array (unpaginated), ordered by total facts DESC then name (spec §7)."""
+    data_root = Path(data_root)
+    out: list[dict] = []
+    for db_path in sorted(data_root.glob("*.db")):
+        if db_path.name.startswith("_"):
+            continue
+        name = db_path.stem
+        conn = open_ro(db_path)
+        try:
+            facts_latest = conn.execute(
+                "SELECT COUNT(*) FROM fact WHERE is_latest=1"
+            ).fetchone()[0]
+            facts_retired = conn.execute(
+                "SELECT COUNT(*) FROM fact WHERE is_latest=0"
+            ).fetchone()[0]
+            entities = conn.execute("SELECT COUNT(*) FROM entity").fetchone()[0]
+            episodes = conn.execute("SELECT COUNT(*) FROM episode").fetchone()[0]
+            chains = conn.execute(
+                "SELECT COUNT(*) FROM fact WHERE superseded_by IS NOT NULL"
+            ).fetchone()[0]
+            top_predicates = [
+                {"predicate": r["predicate"], "count": r["n"]}
+                for r in conn.execute(
+                    "SELECT predicate, COUNT(*) AS n FROM fact "
+                    "GROUP BY predicate ORDER BY n DESC, predicate LIMIT 5"
+                ).fetchall()
+            ]
+        finally:
+            conn.close()
+        file_size = db_path.stat().st_size
+        activity = event_log.activity_summary(name)
+        out.append(
+            {
+                "name": name,
+                "facts_latest": facts_latest,
+                "facts_retired": facts_retired,
+                "entities": entities,
+                "episodes": episodes,
+                "chains": chains,
+                "file_size": file_size,
+                "top_predicates": top_predicates,
+                "activity": activity,
+            }
+        )
+    out.sort(key=lambda n: (-(n["facts_latest"] + n["facts_retired"]), n["name"]))
+    return out
+
+
+def _digest_lines(text: str, predicate) -> str:
+    lines = [ln for ln in text.splitlines() if predicate(ln)]
+    return hashlib.sha256("\n".join(lines).encode("utf-8")).hexdigest()
+
+
+def compute_engine_schema_fingerprint() -> str:
+    """sha256 over the CREATE lines of the INSTALLED lean_memory store/schema.py
+    (importlib.resources, never a checked-in copy) — the §13 tripwire."""
+    text = (
+        importlib.resources.files("lean_memory.store")
+        .joinpath("schema.py")
+        .read_text(encoding="utf-8")
+    )
+    return _digest_lines(text, lambda ln: "create" in ln.lower())
+
+
+def compute_sanitizer_fingerprint() -> str:
+    """sha256 over memory.py's sanitizer lines (_SAFE_NS / the 'or "default"'
+    fallback) — guards the config.py mirror against engine drift (§13)."""
+    text = (
+        importlib.resources.files("lean_memory")
+        .joinpath("memory.py")
+        .read_text(encoding="utf-8")
+    )
+    return _digest_lines(
+        text, lambda ln: "_SAFE_NS" in ln or 'or "default"' in ln
+    )
+
+
+def _paginate(page: int, page_size: int) -> tuple[int, int, int]:
+    page = max(page, 1)
+    page_size = min(max(page_size, 1), 200)
+    return page, page_size, (page - 1) * page_size
+
+
+def list_facts(
+    db_path: Path,
+    latest_only: bool = True,
+    predicate: str | None = None,
+    entity: str | None = None,
+    min_salience: float | None = None,
+    q: str | None = None,
+    page: int = 1,
+    page_size: int = 50,
+) -> dict:
+    """Filterable fact list. Rows carry subject = entity.name (joined via
+    fact.subject_id). q is an FTS filter over fact_fts (distinct from search).
+    Order created_at DESC, id DESC; total is post-filter (spec §7)."""
+    page, page_size, offset = _paginate(page, page_size)
+    where = ["1=1"]
+    args: list = []
+    if latest_only:
+        where.append("f.is_latest = 1")
+    if predicate is not None:
+        where.append("f.predicate = ?")
+        args.append(predicate)
+    if entity is not None:
+        where.append("LOWER(e.name) = LOWER(?)")
+        args.append(entity)
+    if min_salience is not None:
+        where.append("f.salience >= ?")
+        args.append(min_salience)
+    if q is not None:
+        where.append(
+            "f.id IN (SELECT fact_id FROM fact_fts WHERE fact_fts MATCH ?)"
+        )
+        args.append(_fts_query(q))
+    clause = " AND ".join(where)
+
+    conn = open_ro(db_path)
+    try:
+        total = conn.execute(
+            f"SELECT COUNT(*) FROM fact f "
+            f"JOIN entity e ON f.subject_id = e.id WHERE {clause}",
+            args,
+        ).fetchone()[0]
+        rows = conn.execute(
+            f"SELECT f.*, e.name AS subject FROM fact f "
+            f"JOIN entity e ON f.subject_id = e.id WHERE {clause} "
+            f"ORDER BY f.created_at DESC, f.id DESC LIMIT ? OFFSET ?",
+            (*args, page_size, offset),
+        ).fetchall()
+    finally:
+        conn.close()
+    return {
+        "items": [dict(r) for r in rows],
+        "page": page,
+        "page_size": page_size,
+        "total": total,
+    }
+
+
+def get_fact(db_path: Path, fact_id) -> dict | None:
+    """Full fact row + subject name + supersession chain (oldest->newest,
+    walked both directions) + source episode (spec §7)."""
+    conn = open_ro(db_path)
+    try:
+        row = conn.execute(
+            "SELECT f.*, e.name AS subject FROM fact f "
+            "JOIN entity e ON f.subject_id = e.id WHERE f.id = ?",
+            (fact_id,),
+        ).fetchone()
+        if row is None:
+            return None
+        out = dict(row)
+
+        # Walk backward: follow superseded_by chains that point AT this fact
+        # (older facts whose superseded_by == current id), oldest first.
+        backward: list[dict] = []
+        cur = out["id"]
+        while True:
+            prev = conn.execute(
+                "SELECT * FROM fact WHERE superseded_by = ?", (cur,)
+            ).fetchone()
+            if prev is None:
+                break
+            backward.append(dict(prev))
+            cur = prev["id"]
+        backward.reverse()  # oldest -> ... -> just-before-current
+
+        # Walk forward: follow this fact's superseded_by pointer to newer facts.
+        forward: list[dict] = []
+        cur_row = dict(row)
+        while cur_row.get("superseded_by"):
+            nxt = conn.execute(
+                "SELECT * FROM fact WHERE id = ?", (cur_row["superseded_by"],)
+            ).fetchone()
+            if nxt is None:
+                break
+            forward.append(dict(nxt))
+            cur_row = dict(nxt)
+
+        chain = backward + [dict(row)] + forward
+        out["chain"] = chain
+
+        episode = conn.execute(
+            "SELECT * FROM episode WHERE id = ?", (out["episode_id"],)
+        ).fetchone()
+        out["episode"] = dict(episode) if episode is not None else None
+    finally:
+        conn.close()
+    return out
+
+
+def list_episodes(db_path: Path, page: int = 1, page_size: int = 50) -> dict:
+    """Episodes ordered t_ref DESC (spec §7)."""
+    page, page_size, offset = _paginate(page, page_size)
+    conn = open_ro(db_path)
+    try:
+        total = conn.execute("SELECT COUNT(*) FROM episode").fetchone()[0]
+        rows = conn.execute(
+            "SELECT * FROM episode ORDER BY t_ref DESC, id DESC LIMIT ? OFFSET ?",
+            (page_size, offset),
+        ).fetchall()
+    finally:
+        conn.close()
+    return {
+        "items": [dict(r) for r in rows],
+        "page": page,
+        "page_size": page_size,
+        "total": total,
+    }
+
+
+def get_episode(db_path: Path, episode_id) -> dict | None:
+    """One episode + its extracted facts (episode_id match)."""
+    conn = open_ro(db_path)
+    try:
+        row = conn.execute(
+            "SELECT * FROM episode WHERE id = ?", (episode_id,)
+        ).fetchone()
+        if row is None:
+            return None
+        out = dict(row)
+        facts = conn.execute(
+            "SELECT f.*, e.name AS subject FROM fact f "
+            "JOIN entity e ON f.subject_id = e.id WHERE f.episode_id = ? "
+            "ORDER BY f.created_at, f.id",
+            (episode_id,),
+        ).fetchall()
+        out["facts"] = [dict(r) for r in facts]
+    finally:
+        conn.close()
+    return out
+
+
+def list_entities(db_path: Path, page: int = 1, page_size: int = 50) -> dict:
+    """Entity names + fact_count (as subject), ordered fact_count DESC then name."""
+    page, page_size, offset = _paginate(page, page_size)
+    conn = open_ro(db_path)
+    try:
+        total = conn.execute("SELECT COUNT(*) FROM entity").fetchone()[0]
+        rows = conn.execute(
+            "SELECT e.id AS id, e.name AS name, e.type AS type, "
+            "COUNT(f.id) AS fact_count "
+            "FROM entity e LEFT JOIN fact f ON f.subject_id = e.id "
+            "GROUP BY e.id, e.name, e.type "
+            "ORDER BY fact_count DESC, e.name LIMIT ? OFFSET ?",
+            (page_size, offset),
+        ).fetchall()
+    finally:
+        conn.close()
+    return {
+        "items": [dict(r) for r in rows],
+        "page": page,
+        "page_size": page_size,
+        "total": total,
+    }
+
+
+# Filled once from the first run's printed digests (Step 5), then a test pins
+# equality so engine drift turns the suite red.
+EXPECTED_SCHEMA_FINGERPRINT = (
+    "a6c7f41188a196929ff4e7c257a181c100af9f0f7091aa93d428a9a7c0eec8ef"
+)
+EXPECTED_SANITIZER_FINGERPRINT = (
+    # Bumped for WP10a Task 6: Memory._maintenance_store() reuses the identical
+    # `_SAFE_NS.sub("_", namespace) or "default"` expression (memory.py:298),
+    # adding a third matching line; sanitizer semantics unchanged, mirror valid.
+    "973f203a76ab1b535ae8e81dcb830145c92bf481cf4c815b5fbca0044e5d044c"
+)

lean_memory_console/mcp_tools.py

@@ -0,0 +1,137 @@
+"""Shared MCP tool registration for the console's two surfaces (design spec §6.3).
+
+The console ships the SAME memory + maintenance tools on both its stdio server
+(``observe_mcp.py`` — what the plugin runs) and its Docker HTTP mount
+(``routes/mcp.py``). Registering them once here, against a passed-in FastMCP + the
+gateway, keeps tool names, signatures, and return shapes IDENTICAL across the two —
+and identical to the core stdio server (``lean_memory.mcp_server``), which is the §6.3
+requirement and the v0.1.3 manifest-parity lesson.
+
+Every tool reaches the engine ONLY through ``EngineGateway`` (spec §1.3.8): the four
+maintenance methods (``maintain``/``review_queue``/``decide``/``promote``) each wrap
+``retry_busy`` + the per-namespace asyncio lock + the single worker thread, exactly
+like ``add``/``search``.
+
+The review-workflow PROMPT lives on the stdio server only (``register_review_prompt``)
+— MCP prompt surfacing is a stdio-client capability; the plugin command file is the
+portable path for everyone else (§6.4).
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from mcp.server.fastmcp import FastMCP
+
+from .engine import EngineGateway
+
+# The prompt text: the client agent runs THIS workflow. It MUST NOT decide any
+# proposal without an explicit user verdict, and batch verbs ("approve all exact
+# dedups") map only to explicit user statements (§6.4). Kept as a module constant so
+# the stdio prompt and any future surface share one canonical wording.
+REVIEW_PROMPT_TEXT = """\
+You are helping the user review staged sleep-time memory-maintenance proposals for a \
+namespace. Maintenance has already run and STAGED judgment calls (near-duplicate \
+merges, summaries, evictions) as *proposals*; nothing has changed in memory yet. Your \
+job is to walk the user through them and record ONLY the decisions the user explicitly \
+makes.
+
+HARD RULE — you may NOT decide any proposal on your own. Every approve / reject / edit \
+/ promote MUST come from an explicit user verdict in this conversation. If the user has \
+not stated a verdict for an item, leave it pending. A batch verb (e.g. "approve all \
+exact dedups", "reject every eviction") is valid ONLY when the user says it in those \
+words — never infer a batch decision from a single example or from silence. Silence is \
+not consent; unreviewed proposals expire on their own.
+
+Workflow:
+1. Call `memory_review_queue(namespace=..., limit=...)` to fetch the pending queue. It \
+   comes grouped by subject entity, each proposal carrying its evidence payload.
+2. Present the proposals to the user, batched by entity and kind, showing the evidence \
+   (the before/after texts, cosine for near-dups, source facts for summaries, the \
+   value signals for evictions). Keep it scannable.
+3. Collect the user's EXPLICIT verdicts. Ask when a verdict is missing or ambiguous. Do \
+   not proceed on an item the user has not ruled on.
+4. For each item the user decided, call \
+   `memory_review_decide(namespace=..., proposal_id=..., decision=..., edited_text=...)` \
+   with decision in {approve, reject, edit, promote}. `edited_text` applies only to a \
+   summarize proposal the user chose to reword before approving.
+5. Summarize what was applied, what was rejected, and what remains pending (untouched, \
+   still awaiting the user).
+"""
+
+
+def register_maintenance_tools(mcp: FastMCP, gateway: EngineGateway) -> None:
+    """Register the four §6.3 maintenance tools on `mcp`, routed through `gateway`.
+
+    Identical names/signatures to the core stdio server: memory_maintenance_run
+    (dry-run default), memory_maintenance_status, memory_review_queue,
+    memory_review_decide.
+    """
+
+    @mcp.tool()
+    async def memory_maintenance_run(
+        namespace: str, apply: bool = False
+    ) -> dict[str, Any]:
+        """Run one sleep-time maintenance pass (§6.3). DRY-RUN by default (apply=False):
+        computes the would-do report with zero writes. apply=True runs the auto band
+        and stages proposals. Symmetric with the CLI. NOTE: the LM_MAINT_AUTO
+        auto-spawn path runs `--apply --auto-only` (auto band only, no proposals) —
+        only interactive apply=True grows the review queue. Returns the run summary."""
+        return await gateway.maintain(namespace, apply=apply)
+
+    @mcp.tool()
+    async def memory_maintenance_status(namespace: str) -> dict[str, Any]:
+        """Report the namespace's maintenance ledger — runs + pending proposals (§6.3).
+
+        A pure model-free ledger read, shaped identically to the core server's status
+        tool ({namespace, runs, pending_proposals, last_run}). Use it to see whether
+        maintenance is due and how many proposals wait."""
+        return await gateway.maintenance_status(namespace)
+
+    @mcp.tool()
+    async def memory_review_queue(
+        namespace: str, kind: str | None = None, limit: int = 20
+    ) -> dict[str, Any]:
+        """List pending maintenance proposals, grouped by entity, with evidence (§6.3).
+
+        `kind` filters to 'dedup_near' | 'summarize' | 'evict'. Each proposal includes
+        its parsed evidence payload so the reviewer sees exactly what would change."""
+        groups = await gateway.review_queue(namespace, kind=kind, limit=limit)
+        # Round-trip through JSON so numpy/None edge values serialize cleanly, matching
+        # the core server's json.dumps(..., default=str) shape.
+        return json.loads(json.dumps({"groups": groups}, default=str))
+
+    @mcp.tool()
+    async def memory_review_decide(
+        namespace: str,
+        proposal_id: str,
+        decision: str,
+        edited_text: str | None = None,
+    ) -> dict[str, Any]:
+        """Decide a maintenance proposal: approve | reject | edit | promote (§6.3).
+
+        approve applies at decide-time with apply-time re-validation; reject leaves the
+        spine byte-identical; edit (summarize only) approves the human-edited text;
+        promote (evict only) rejects the eviction and lifts the fact to the hot tier."""
+        result = await gateway.decide(
+            namespace, proposal_id, decision, edited_text=edited_text
+        )
+        return json.loads(json.dumps(result, default=str))
+
+
+def register_review_prompt(mcp: FastMCP) -> None:
+    """Register the `review-memory-maintenance` prompt on a stdio FastMCP (§6.4).
+
+    The prompt hands the client agent the review workflow but FORBIDS it from deciding
+    without an explicit user verdict. Stdio-only: prompt surfacing is a stdio-client
+    capability; the plugin command file is the portable path (§6.4)."""
+
+    @mcp.prompt(name="review-memory-maintenance")
+    def review_memory_maintenance(namespace: str = "") -> str:
+        """Walk the user through staged memory-maintenance proposals and record only the
+        decisions the user explicitly makes."""
+        header = (
+            f"Namespace to review: {namespace}\n\n" if namespace else ""
+        )
+        return header + REVIEW_PROMPT_TEXT

lean_memory_console/observe_mcp.py

@@ -0,0 +1,73 @@
+"""Observing MCP wrapper — stdio server that writes through EngineGateway.
+
+A deliberate superset of the core stdio server: memory_add gains source/t_ref
+and a structured return; memory_clear is intentionally absent (no deletion
+surface, §6). Parity is with the Memory API, not the core tool signatures.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from mcp.server.fastmcp import FastMCP
+
+from .config import ConsoleConfig
+from .engine import EngineGateway
+from .events import EventLog
+from .mcp_tools import register_maintenance_tools, register_review_prompt
+
+
+def build_mcp(gateway: EngineGateway) -> FastMCP:
+    mcp = FastMCP("lean-memory-console")
+
+    @mcp.tool()
+    async def memory_add(
+        namespace: str,
+        text: str,
+        source: str = "user",
+        t_ref: int | None = None,
+    ) -> dict[str, Any]:
+        """Ingest text into the namespace's memory (observing wrapper).
+
+        Returns the new fact ids and how many prior facts were superseded.
+        """
+        res = await gateway.add(namespace, text, source=source, t_ref=t_ref)
+        return {
+            "fact_ids": res.fact_ids,
+            "superseded_count": res.superseded_count,
+        }
+
+    @mcp.tool()
+    async def memory_search(namespace: str, query: str, k: int = 5) -> dict[str, Any]:
+        """Search a namespace's memory; returns top-k fact texts + scores.
+
+        Always latest-only (the latest_only flag is REST-only, §6).
+        """
+        res = await gateway.search(
+            namespace, query, k=k, latest_only=True, origin="agent"
+        )
+        return {
+            "hits": [
+                {"fact_text": h["fact_text"], "final_score": h["final_score"]}
+                for h in res.hits
+            ]
+        }
+
+    # The four sleep-time maintenance tools, identical to the core + HTTP surfaces
+    # (§6.3), plus the stdio-only review-workflow prompt (§6.4).
+    register_maintenance_tools(mcp, gateway)
+    register_review_prompt(mcp)
+
+    return mcp
+
+
+def run_stdio(config: ConsoleConfig) -> None:
+    """Build the gateway + wrapper for `config` and serve over stdio."""
+    event_log = EventLog(config.data_root)
+    gateway = EngineGateway(config, event_log)
+    mcp = build_mcp(gateway)
+    try:
+        mcp.run()  # blocks on stdio until the client disconnects
+    finally:
+        gateway.close()
+        event_log.close()

lean_memory_console/routes/data.py

@@ -0,0 +1,53 @@
+"""/v1/* — the REST data plane mirror (Docker mode, non-MCP agents)."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from ..config import is_reserved_namespace
+
+
+class MemoryBody(BaseModel):
+    text: str
+    source: str = "user"
+    t_ref: int | None = None
+
+
+class SearchBody(BaseModel):
+    query: str
+    k: int = Field(default=5, ge=1, le=200)
+    latest_only: bool = True
+
+
+def build_data_router() -> APIRouter:
+    router = APIRouter(prefix="/v1")
+
+    @router.post("/{namespace}/memories")
+    async def add_memory(request: Request, namespace: str, body: MemoryBody):
+        if is_reserved_namespace(namespace):
+            raise HTTPException(status_code=404, detail="unknown namespace")
+        gateway = request.app.state.gateway
+        res = await gateway.add(
+            namespace, body.text, source=body.source, t_ref=body.t_ref
+        )
+        return {
+            "fact_ids": res.fact_ids,
+            "superseded_count": res.superseded_count,
+        }
+
+    @router.post("/{namespace}/search")
+    async def search_memory(request: Request, namespace: str, body: SearchBody):
+        if is_reserved_namespace(namespace):
+            raise HTTPException(status_code=404, detail="unknown namespace")
+        gateway = request.app.state.gateway
+        res = await gateway.search(
+            namespace,
+            body.query,
+            k=body.k,
+            latest_only=body.latest_only,
+            origin="agent",
+        )
+        return {"hits": res.hits, "duration_ms": res.duration_ms}
+
+    return router