thread-archive 0.0.2__py3-none-any.whl

thread_archive/_mcp/librarian.py

@@ -0,0 +1,151 @@
+"""The librarian MCP server — the curatorial *write* surface, kept separate.
+
+A second, deliberately distinct MCP server (``thread-archive-librarian``) carrying the
+knowledge-layer write tools + the curation-read tools the librarian needs to decide
+what to write. The read-only ``thread-archive`` server (``thread_search`` /
+``thread_read``) stays read-only: mixing write tools into it would hand every
+read-only client (e.g. a reader bot) curation power. This mirrors the read/write split
+the upstream operator surface enforces.
+
+Library-native, like the read server: each tool opens the archive and dispatches
+straight to :mod:`thread_archive._knowledge.write` in-process — no HTTP, no route layer.
+The archive home comes from ``$THREAD_ARCHIVE_HOME`` (else ``~/.thread/archive``).
+Run with::
+
+    python -m thread_archive._mcp.librarian
+
+Every write is event-sourced (an append-only ``KgEvent``) and idempotent at the
+projection layer, so a re-run never double-links or double-cites.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Optional
+
+from mcp.server.fastmcp import FastMCP
+
+from .. import _api as api
+from .._knowledge import write as _write
+
+mcp = FastMCP("thread-archive-librarian")
+
+
+def _dump(value) -> str:
+    return json.dumps(value, ensure_ascii=False, default=str)
+
+
+# ── curation-read ────────────────────────────────────────────────────────────
+@mcp.tool()
+def review_queue(limit: int = 20, exclude_source_id: Optional[str] = None) -> str:
+    """Unreviewed conversation threads (the librarian backlog), newest first.
+
+    Event-bearing conversations the librarian hasn't curated yet — a thread leaves the
+    queue once it gains its first topic citation/link. Pass your own session's
+    ``source_id`` as ``exclude_source_id`` to drop your still-growing transcript from the
+    queue. In a parallel backfill (when ``$THREAD_ARCHIVE_LIBRARIAN_WORKER`` is set) this
+    transparently claims its batch under a lease, so concurrent workers don't overlap —
+    you call it the same way regardless. Returns a JSON list of ``{id, title, source_id}``."""
+    api.open_archive()
+    return _dump(_write.review_queue(limit=limit, exclude_source_id=exclude_source_id))
+
+
+@mcp.tool()
+def topic_search(query: str, limit: int = 10) -> str:
+    """Find existing live topics by title substring — search before creating duplicates.
+    Returns a JSON list of ``{topic_id, title}``."""
+    api.open_archive()
+    return _dump(_write.topic_search(query, limit=limit))
+
+
+@mcp.tool()
+def thread_user_messages(thread_id: int, limit: Optional[int] = None) -> str:
+    """A thread's user messages as ``[{event_id, text}]`` — the cheap, high-signal read
+    to cite from (cite the ``event_id`` values)."""
+    api.open_archive()
+    return _dump(_write.thread_user_messages(thread_id, limit=limit))
+
+
+# ── topics ───────────────────────────────────────────────────────────────────
+@mcp.tool()
+def topic_create(title: str, description: Optional[str] = None) -> str:
+    """Create a topic. Errors if one with that title exists — ``topic_search`` first and
+    link to the existing topic rather than duplicating."""
+    api.open_archive()
+    try:
+        return _dump(_write.create_topic(title, description))
+    except ValueError as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def topic_rename(topic_id: int, title: str, description: Optional[str] = None) -> str:
+    """Rename (and optionally re-describe) a topic."""
+    api.open_archive()
+    try:
+        return _dump(_write.rename_topic(topic_id, title, description=description))
+    except ValueError as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def topic_archive(topic_id: int) -> str:
+    """Archive a topic — it drops out of the live graph (the thread stays reachable)."""
+    api.open_archive()
+    try:
+        return _dump(_write.archive_topic(topic_id))
+    except ValueError as e:
+        return f"Error: {e}"
+
+
+@mcp.tool()
+def topic_merge(from_id: int, into_id: int) -> str:
+    """Merge ``from_id`` into ``into_id``: repoint its links + citations, archive it."""
+    api.open_archive()
+    try:
+        return _dump(_write.merge_topics(from_id, into_id))
+    except ValueError as e:
+        return f"Error: {e}"
+
+
+# ── links + citations ─────────────────────────────────────────────────────────
+@mcp.tool()
+def topic_link(
+    source_id: int, target_id: int, link_type: str = "related",
+    strength: float = 1.0, evidence: Optional[str] = None,
+) -> str:
+    """Link two threads/topics (idempotent on source+target+link_type). ``link_type`` is
+    e.g. related / implements / example-of / contrast / supersedes / works_on."""
+    api.open_archive()
+    return _dump(_write.link_threads(
+        source_id, target_id, link_type, strength=strength, evidence=evidence))
+
+
+@mcp.tool()
+def topic_unlink(source_id: int, target_id: int, link_type: str = "related") -> str:
+    """Remove a link (a tombstone event — recorded, not erased from history)."""
+    api.open_archive()
+    return _dump(_write.unlink_threads(source_id, target_id, link_type))
+
+
+@mcp.tool()
+def topic_cite(topic_id: int, event_id: int, thread_id: int, quote: str) -> str:
+    """Cite a conversation message (``event_id`` in ``thread_id``) as evidence for a
+    topic. Idempotent on (topic_id, event_id)."""
+    api.open_archive()
+    return _dump(_write.add_topic_evidence(topic_id, event_id, thread_id, quote))
+
+
+@mcp.tool()
+def topic_uncite(topic_id: int, event_id: int) -> str:
+    """Archive a citation (tombstone — sets archived_at, keeps the row + history)."""
+    api.open_archive()
+    return _dump(_write.archive_topic_evidence(topic_id, event_id))
+
+
+def main() -> None:
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

thread_archive/_mcp/server.py

@@ -0,0 +1,307 @@
+"""Library-native MCP server.
+
+Exposes ``thread_search`` + ``thread_read`` as MCP tools that call the
+:mod:`thread_archive._api` library functions directly — no web framework, no HTTP,
+no route layer. The server is library-native: it dispatches straight to the API
+functions in-process.
+
+The *tools* are read-only, but the server process also cohosts lazy catch-up
+ingest (see :func:`_maybe_catch_up` and :mod:`.._watcher.lazy`): a throttled
+background pass at startup and around tool calls keeps the archive current
+with no daemon installed, and degrades to a no-op flock probe when the
+always-on watcher owns ingest. ``THREAD_ARCHIVE_MCP_INGEST=0`` disables it.
+
+The archive home comes from ``$THREAD_ARCHIVE_HOME`` (set by the MCP client
+config), else ``~/.thread/archive``. Run with::
+
+    python -m thread_archive._mcp.server
+
+``import mcp`` below is the external MCP SDK (top-level absolute import); this
+package is ``thread_archive._mcp`` and never shadows it.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import threading
+import time
+from typing import Optional
+
+from mcp.server.fastmcp import FastMCP
+
+from .. import _api as api
+from .._retrieval import format_results, warm_models
+from .._retrieval.format import _query_terms, _term_hit_count
+
+logger = logging.getLogger(__name__)
+
+mcp = FastMCP("thread-archive")
+
+# ── lazy catch-up ingest ──────────────────────────────────────────────────────
+# The zero-daemon freshness path: the server runs a background catch-up pass at
+# startup and (throttled) around tool calls, so a bare `claude mcp add …
+# archive-mcp` searches a current archive without any LaunchAgent installed.
+# Cross-process safety lives in the pass itself (see _watcher.lazy): the
+# ingest-owner flock makes every pass a no-op probe while the always-on watcher
+# daemon — or another server's pass — owns ingest. THREAD_ARCHIVE_MCP_INGEST=0
+# turns the whole behaviour off.
+_INGEST_MIN_INTERVAL = 300.0  # seconds between catch-up attempts in this process
+_ingest_last = 0.0  # monotonic time of the last attempt (0 = never)
+_ingest_running = threading.Lock()  # one in-flight catch-up per process
+
+
+def _maybe_catch_up() -> None:
+    """Kick a background catch-up pass, throttled. Never blocks the caller and
+    never raises — retrieval must work identically with ingest disabled, owned
+    by another process, or broken."""
+    global _ingest_last
+    if os.environ.get("THREAD_ARCHIVE_MCP_INGEST", "1").strip().lower() in (
+        "0", "false", "no", "off",
+    ):
+        return
+    now = time.monotonic()
+    if _ingest_last and now - _ingest_last < _INGEST_MIN_INTERVAL:
+        return
+    if not _ingest_running.acquire(blocking=False):
+        return  # a catch-up is already in flight in this process
+    _ingest_last = now
+
+    def _run() -> None:
+        try:
+            from .._watcher import catch_up_once
+
+            catch_up_once()
+        except Exception:  # noqa: BLE001 — advisory; retrieval must not care
+            logger.exception("lazy catch-up ingest failed")
+        finally:
+            _ingest_running.release()
+
+    threading.Thread(target=_run, name="archive-lazy-ingest", daemon=True).start()
+
+# The agent-facing default search scope: USER messages plus the thread-meta docs
+# (title + stored summary) — the intentional signals of what a thread was about.
+# Assistant text, tool calls/results, and thinking are opt-in (pass an explicit
+# content_type), and content_type='all' clears the filter to search everything.
+# Mirrors the archive backend's thread_search default.
+DEFAULT_SEARCH_CONTENT_TYPES = ("user", "title", "summary")
+
+# Where the default scope widens to when it comes up dry: assistant text is where
+# conclusions/decisions live (see thread_read's mode docs), so a query the asking
+# side can't answer gets one automatic retry against it. One retry, only on the
+# default scope, only when no query term landed — an explicit content_type is a
+# deliberate choice and is never second-guessed.
+WIDENED_SEARCH_CONTENT_TYPES = DEFAULT_SEARCH_CONTENT_TYPES + ("text",)
+
+
+def _default_scope_is_weak(hits: list[dict], query: str) -> bool:
+    """True when a default-scope result set warrants the one-shot widen to
+    assistant text: no hits at all, or no query term appears in the top hit
+    (nearest-neighbour guesses)."""
+    terms = _query_terms(query)
+    if not terms:
+        return False
+    if not hits:
+        return True
+    return _term_hit_count(hits[0].get("full_content") or hits[0].get("snippet") or "", terms) == 0
+
+
+@mcp.tool()
+def thread_search(
+    query: str,
+    limit: int = 10,
+    thread_id: Optional[int] = None,
+    content_type: Optional[str] = None,
+    exclude_content_type: Optional[str] = None,
+    since: Optional[str] = None,
+    until: Optional[str] = None,
+    tool_name: Optional[str] = None,
+    source: Optional[str] = None,
+    startswith: Optional[str] = None,
+    sort: Optional[str] = None,
+    output: Optional[str] = None,
+    context_lines: int = 2,
+    context_events: Optional[str] = None,
+    rerank: Optional[bool] = None,
+) -> str:
+    """Search the local conversation archive (federated: lexical FTS5 + optional
+    semantic vectors → fusion → rank → optional cross-encoder re-rank).
+
+    Read the match signal before trusting a result: the header carries
+    ``quality=strong|partial|weak|semantic`` for the top hit, and each hit shows
+    ``K/N`` (how many query terms landed) — a weak / 0-of-N result means these are
+    nearest-neighbour guesses and the log likely lacks it, so rephrase or switch
+    store rather than piling on synonyms.
+
+    By default USER messages, thread titles, and stored thread summaries are
+    searched — the strongest signals of what a thread was about. When that scope
+    comes up dry (no query term in the top hit), the search retries once with
+    assistant text included and says so in the output. Assistant text,
+    tool calls/results, and thinking are otherwise opt-in: pass
+    ``content_type='all'`` to search everything, or a specific ``content_type``
+    (text/thinking/tool/tool_result/...) to target one.
+
+    Query grammar: natural language, "quoted phrases", boolean AND/OR/NOT,
+    pipe-OR (a|b), and code identifiers (get_session, a.b.c). Filter by
+    ``thread_id``, ``content_type`` (default user+title+summary; 'all' searches
+    everything),
+    ``exclude_content_type`` (comma-separated types to drop), ``tool_name``,
+    ``source`` (comma-separated providers, e.g. 'claude-code,cursor'), and a
+    ``since``/``until`` window (ISO timestamp or '7d').
+
+    ``startswith`` does a structural prefix scan (content LIKE 'prefix%'; query text
+    unused). ``sort='oldest'`` returns matches chronologically (find when something
+    was first discussed) instead of the default recency-biased ranking.
+    ``context_lines`` (default 2; set 0 for the raw FTS snippet) replaces each
+    snippet with a numbered ±N-line window around the match; ``context_events``
+    ('N' / 'before:after' /
+    'before:after:types', e.g. '2' or '0:1:user') appends the neighbouring events.
+    ``output='count'`` returns a per-thread tally (no snippets); ``output='linkable'``
+    returns JSON of event/thread ids. ``rerank`` forces the cross-encoder head
+    re-rank on/off (else auto-gated to conceptual queries when the ``[embeddings]``
+    extra is installed).
+    """
+    _maybe_catch_up()
+    # Default scope is user messages only; an explicit type targets it, and
+    # content_type='all' clears the filter to search everything (see the constant).
+    if content_type == "all":
+        content_types = None
+    elif content_type:
+        content_types = [content_type]
+    else:
+        content_types = list(DEFAULT_SEARCH_CONTENT_TYPES)
+    exclude = [c.strip() for c in exclude_content_type.split(",") if c.strip()] if exclude_content_type else None
+    sources = [s.strip() for s in source.split(",") if s.strip()] if source else None
+
+    def _run(cts):
+        return api.search(
+            query,
+            limit=limit,
+            thread_id=thread_id,
+            content_types=cts,
+            exclude_content_types=exclude,
+            since=since,
+            until=until,
+            tool_name=tool_name,
+            source=sources,
+            startswith=startswith,
+            sort=sort,
+            output=output,
+            context_lines=context_lines,
+            context_events=context_events,
+            rerank=rerank,
+        )
+
+    hits = _run(content_types)
+
+    # One-shot scope widen: a default-scope search whose top hit contains no query
+    # term (or that found nothing) retries once with assistant text included —
+    # conclusions live there, and internalizing the retry saves the agent a
+    # round-trip the quality note would otherwise ask of it. Ranked/plain output
+    # only: structural shapes (browse/startswith/oldest/count/linkable) have no
+    # match signal to judge weakness by.
+    widened = False
+    ranked_shape = bool((query or "").strip()) and startswith is None and sort is None and output is None
+    if content_type is None and ranked_shape and _default_scope_is_weak(hits, query):
+        wide_hits = _run(list(WIDENED_SEARCH_CONTENT_TYPES))
+        if wide_hits and not _default_scope_is_weak(wide_hits, query):
+            hits, widened = wide_hits, True
+
+    rendered = format_results(hits, query, output=output)
+    if widened:
+        rendered = ("note: no keyword match in the default scope (user/title/summary) — "
+                    "results below include assistant text\n" + rendered)
+    return rendered
+
+
+@mcp.tool()
+def thread_read(
+    thread_id: int | str,
+    limit: int = 200,
+    offset: int = 0,
+    summary: bool | str = False,
+    mode: Optional[str] = None,
+    user_only: Optional[bool] = None,
+    tool_results: bool = False,
+    max_chars: int = 0,
+    after_event: Optional[int] = None,
+) -> str:
+    """Read a thread's conversation, reconstructed from the event log.
+
+    ``thread_id`` accepts either the archive's own integer thread id OR a provider
+    **session uuid** (the id a tool like claude-code / cursor / codex knows the
+    conversation by — its ``source_id``). The uuid is resolved to the thread
+    automatically (newest match wins), so you can pass a session uuid straight
+    through without looking the integer id up first.
+
+    ``mode`` picks the view: 'user' (default) = only the USER messages — the real
+    signal of what a thread was about and what was wanted, far cheaper than the
+    transcript (for research / 'what was this thread about' that IS what you want);
+    'chat' = the readable conversation — user turns + the assistant's reasoning/text
+    with tool calls stripped out (use when you need what was *decided/concluded/built*,
+    which lives in assistant text); 'full' = the whole transcript including every
+    tool call (bulky, mostly tool noise — only when you need what the assistant *did*).
+    Tool *output* is off by default; set ``tool_results=true`` (only meaningful with
+    'full', where the calls are shown) to fold each tool's result under its call.
+
+    The read is size-budgeted (~48k chars), so it never silently overflows the MCP
+    output cap: a thread bigger than one chunk ends in a CHUNKED footer naming the
+    exact offset to read next (that's pagination, not lost data — page with
+    ``offset``, or resume from an event with ``after_event``). ``summary`` picks a
+    summary view instead of the transcript: ``true``/``'toc'`` = a compact per-message
+    TOC; ``'short'`` = the thread's stored short summary (a few sentences);
+    ``'indexed'`` = the stored indexed summary (structured, with event anchors) —
+    the stored kinds exist only where the summarizer has covered the thread.
+    ``user_only`` is a back-compat alias for ``mode`` (true→user, false→full);
+    prefer ``mode``, which wins if both are set.
+
+    Args:
+        thread_id: Integer thread id, or a provider session uuid (source_id) which
+            is resolved to the thread automatically.
+        limit: Max turns per chunk (safety cap; the char budget usually bites first).
+            Default: 200.
+        offset: Skip first N turns. Use the offset from a CHUNKED footer to read the
+            next chunk. Negative counts from end: -20 = last 20 turns. Default: 0.
+        summary: Summary view instead of full content — true/'toc' for a compact
+            TOC with previews, 'short' or 'indexed' for the stored thread summary.
+        mode: View — 'user' (default), 'chat', or 'full'. Default: user.
+        user_only: Back-compat alias for mode (true→user, false→full). Prefer mode.
+        tool_results: Include tool output under each call (default off; needs 'full').
+        max_chars: Per-chunk character budget; the read stops at a clean turn
+            boundary once hit and the footer points at the next offset. Default: ~48k.
+        after_event: Resume reading from the turn AFTER this event id (overrides
+            offset). Robust way to continue from where a previous read stopped.
+    """
+    _maybe_catch_up()
+    return api.read_thread(
+        thread_id,
+        limit=limit,
+        offset=offset,
+        summary=summary,
+        mode=mode,
+        user_only=user_only,
+        tool_results=tool_results,
+        max_chars=max_chars,
+        after_event=after_event,
+    )
+
+
+def main() -> None:
+    # Warm the embedding + cross-encoder models on a background daemon thread. The cold load
+    # is tens of seconds; when it lands inside the first conceptual search it can exceed the
+    # client's MCP request timeout (cloth defaults to 60s), which surfaces to the model as a
+    # failed tool call. Warming at startup moves that cost off the request path — the models
+    # are (usually) resident by the time the first query arrives, and _load()'s lock makes an
+    # early query that races the warm wait on one load rather than kick off a second. Daemon
+    # so it never holds up interpreter exit; warm_models is fail-soft (a missing extra / load
+    # failure just restores the old lazy behaviour).
+    threading.Thread(target=warm_models, name="archive-warm-models", daemon=True).start()
+    # Startup catch-up: whatever landed in the local stores since the last
+    # ingest (by any process) is searchable by the time the first query
+    # arrives — or shortly after; the pass is additive, never blocking.
+    _maybe_catch_up()
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()