thread-archive 0.0.2__py3-none-any.whl

thread_archive/_knowledge/write.py

@@ -0,0 +1,438 @@
+"""The curatorial write API — the librarian's hands on the topic graph.
+
+Thin, typed functions over the event-sourced knowledge layer. Each *mutating* call
+does the same three things atomically: append a ``KgEvent`` to the truth log, fold it
+into the projection (:func:`thread_archive._knowledge.materialize.apply_event`), and —
+when it owns the session — commit and drop the graph cache. One call = a durable log
+line + an updated projection. The conversation archive stays untouched; this only
+writes the knowledge overlay (topics, links, citations) that sits beside it.
+
+Topics are threads (``thread_type='topic'``), so :func:`create_topic` writes a real
+per-thread truth file and the ``topic.created`` event is the audit record; links and
+evidence have no owning conversation file, so the ``kg_events.jsonl`` log *is* their
+truth. Read helpers (:func:`review_queue`, :func:`topic_search`,
+:func:`thread_user_messages`) are the curation-read surface the librarian needs to
+decide *what* to write — kept here so the MCP and the skill share one library.
+
+All writes are idempotent at the projection layer (upsert + tombstone), so re-running
+the librarian never double-links or double-cites — a partial pass is simply redone.
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+import os
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy import or_, select
+from sqlalchemy.orm import Session
+
+from .._store import Event, KgEvent, Thread, ThreadLink, TopicMessage, use_session
+from .._truth.jsonl_log import append_kg_event, record_thread, shared_ingest_lock
+from .graph import reset_cache
+from .materialize import apply_event
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _locked_write(fn):
+    """Run an own-session mutation under the shared reindex lock.
+
+    Every mutating function here appends truth (a ``KgEvent``, sometimes a thread
+    record) and commits; racing a reindex unlocked, that write can land after the
+    rebuild's read point and commit into the database inode the swap replaces.
+    When the caller passes its own ``session`` it owns the transaction boundary —
+    and must hold the lock around it itself."""
+
+    @functools.wraps(fn)
+    def wrapper(*args, session: Optional[Session] = None, **kwargs):
+        if session is not None:
+            return fn(*args, session=session, **kwargs)
+        with shared_ingest_lock():
+            return fn(*args, session=None, **kwargs)
+
+    return wrapper
+
+
+def _topic_name(title: str) -> str:
+    """The unique ``threads.name`` for a topic. Topics share the threads table with
+    conversations, so they need a namespaced unique name distinct from a conversation's
+    ``"{source}:{source_id}"``."""
+    return f"topic:{title}"
+
+
+def _emit(
+    session: Session,
+    *,
+    event_type: str,
+    entity_type: str,
+    entity_id,
+    payload: dict,
+    actor: str,
+    actor_thread_id: Optional[int],
+) -> KgEvent:
+    """Append a ``KgEvent`` and fold it into the projection, in the caller's session.
+
+    Flush first so ``id`` / ``recorded_at`` are populated before the row is staged to
+    truth and applied; the caller owns the commit (the before-commit drain writes the
+    log line). The single seam every mutating function below goes through."""
+    ev = KgEvent(
+        event_type=event_type,
+        entity_type=entity_type,
+        entity_id=str(entity_id) if entity_id is not None else None,
+        payload=payload,
+        actor=actor,
+        actor_thread_id=actor_thread_id,
+        occurred_at=_now(),
+    )
+    session.add(ev)
+    session.flush()
+    append_kg_event(session, ev)
+    apply_event(session, ev)
+    return ev
+
+
+# ── topics ─────────────────────────────────────────────────────────────────────
+@_locked_write
+def create_topic(
+    title: str,
+    description: Optional[str] = None,
+    *,
+    topic_kind: Optional[str] = None,
+    actor: str = "librarian",
+    actor_thread_id: Optional[int] = None,
+    session: Optional[Session] = None,
+) -> dict:
+    """Create a topic (a ``thread_type='topic'`` thread) and return ``{topic_id, ...}``.
+
+    Raises ``ValueError`` if a topic with that title already exists — search first
+    (:func:`topic_search`) and link to the existing one rather than making a duplicate.
+    """
+    own = session is None
+    with use_session(session) as s:
+        existing = s.execute(select(Thread).where(Thread.name == _topic_name(title))).scalars().first()
+        if existing is not None:
+            raise ValueError(f"a topic named {title!r} already exists (id={existing.id}); link to it instead")
+        topic = Thread(
+            name=_topic_name(title), title=title, description=description,
+            thread_type="topic", topic_kind=topic_kind,
+        )
+        s.add(topic)
+        s.flush()
+        record_thread(s, topic)  # the topic's own per-thread truth file opens with its record
+        ev = _emit(
+            s, event_type="topic.created", entity_type="topic", entity_id=topic.id,
+            payload={"title": title, "description": description, "thread_id": topic.id},
+            actor=actor, actor_thread_id=actor_thread_id,
+        )
+        result = {"topic_id": topic.id, "title": title, "event_id": ev.id}
+        if own:
+            s.commit()
+            reset_cache()
+    return result
+
+
+@_locked_write
+def rename_topic(
+    topic_id: int, title: str, *, description: Optional[str] = None,
+    actor: str = "librarian", actor_thread_id: Optional[int] = None,
+    session: Optional[Session] = None,
+) -> dict:
+    """Rename (and optionally re-describe) a topic."""
+    own = session is None
+    with use_session(session) as s:
+        topic = _require_topic(s, topic_id)
+        topic.title = title
+        if description is not None:
+            topic.description = description
+        topic.updated_at = _now()
+        record_thread(s, topic)  # latest-wins metadata in the topic's per-thread file
+        payload: dict = {"title": title}
+        if description is not None:
+            payload["description"] = description
+        ev = _emit(s, event_type="topic.renamed", entity_type="topic", entity_id=topic_id,
+                   payload=payload, actor=actor, actor_thread_id=actor_thread_id)
+        result = {"topic_id": int(topic_id), "title": title, "event_id": ev.id}
+        if own:
+            s.commit()
+            reset_cache()
+    return result
+
+
+@_locked_write
+def archive_topic(
+    topic_id: int, *, actor: str = "librarian", actor_thread_id: Optional[int] = None,
+    session: Optional[Session] = None,
+) -> dict:
+    """Archive a topic (drops it from the live graph; the thread stays reachable)."""
+    own = session is None
+    with use_session(session) as s:
+        topic = _require_topic(s, topic_id)
+        topic.archived = True
+        topic.updated_at = _now()
+        record_thread(s, topic)
+        ev = _emit(s, event_type="topic.archived", entity_type="topic", entity_id=topic_id,
+                   payload={}, actor=actor, actor_thread_id=actor_thread_id)
+        result = {"topic_id": int(topic_id), "archived": True, "event_id": ev.id}
+        if own:
+            s.commit()
+            reset_cache()
+    return result
+
+
+@_locked_write
+def merge_topics(
+    from_id: int, into_id: int, *, actor: str = "librarian",
+    actor_thread_id: Optional[int] = None, session: Optional[Session] = None,
+) -> dict:
+    """Merge ``from_id`` into ``into_id``: repoint its links + citations, archive it."""
+    own = session is None
+    with use_session(session) as s:
+        from_topic = _require_topic(s, from_id)
+        _require_topic(s, into_id)
+        ev = _emit(s, event_type="topic.merged", entity_type="topic", entity_id=into_id,
+                   payload={"from_id": int(from_id), "into_id": int(into_id)},
+                   actor=actor, actor_thread_id=actor_thread_id)
+        # Persist the merged-away topic's archival to its per-thread file too (the fold
+        # archived it in the table; this records it in the truth the file carries).
+        from_topic.archived = True
+        from_topic.updated_at = ev.occurred_at
+        record_thread(s, from_topic)
+        result = {"from_id": int(from_id), "into_id": int(into_id), "event_id": ev.id}
+        if own:
+            s.commit()
+            reset_cache()
+    return result
+
+
+# ── links ──────────────────────────────────────────────────────────────────────
+@_locked_write
+def link_threads(
+    source_id: int, target_id: int, link_type: str = "related", *,
+    strength: float = 1.0, evidence: Optional[str] = None, actor: str = "librarian",
+    actor_thread_id: Optional[int] = None, session: Optional[Session] = None,
+) -> dict:
+    """Create (or update) a directed link between two threads/topics. Idempotent on
+    ``(source, target, link_type)``. Both endpoints must exist — the kg log is
+    truth, and a link to a nonexistent thread would dangle on every replay."""
+    own = session is None
+    with use_session(session) as s:
+        for tid in (source_id, target_id):
+            if s.get(Thread, int(tid)) is None:
+                raise ValueError(f"no thread with id {tid} — link endpoints must exist")
+        ev = _emit(
+            s, event_type="link.created", entity_type="link",
+            entity_id=f"{int(source_id)}:{int(target_id)}:{link_type}",
+            payload={
+                "source_thread_id": int(source_id), "target_thread_id": int(target_id),
+                "link_type": link_type, "strength": float(strength),
+                "evidence": evidence, "created_by": actor,
+            },
+            actor=actor, actor_thread_id=actor_thread_id,
+        )
+        result = {
+            "source_thread_id": int(source_id), "target_thread_id": int(target_id),
+            "link_type": link_type, "event_id": ev.id,
+        }
+        if own:
+            s.commit()
+            reset_cache()
+    return result
+
+
+@_locked_write
+def unlink_threads(
+    source_id: int, target_id: int, link_type: str = "related", *,
+    actor: str = "librarian", actor_thread_id: Optional[int] = None,
+    session: Optional[Session] = None,
+) -> dict:
+    """Remove a link (a tombstone event — the operation is recorded, not erased)."""
+    own = session is None
+    with use_session(session) as s:
+        ev = _emit(
+            s, event_type="link.deleted", entity_type="link",
+            entity_id=f"{int(source_id)}:{int(target_id)}:{link_type}",
+            payload={"source_thread_id": int(source_id), "target_thread_id": int(target_id),
+                     "link_type": link_type},
+            actor=actor, actor_thread_id=actor_thread_id,
+        )
+        result = {"source_thread_id": int(source_id), "target_thread_id": int(target_id),
+                  "link_type": link_type, "event_id": ev.id}
+        if own:
+            s.commit()
+            reset_cache()
+    return result
+
+
+# ── evidence (message → topic citations) ─────────────────────────────────────────
+@_locked_write
+def add_topic_evidence(
+    topic_id: int, event_id: int, thread_id: int, quote: str, *,
+    actor: str = "librarian", actor_thread_id: Optional[int] = None,
+    session: Optional[Session] = None,
+) -> dict:
+    """Cite a conversation message as evidence for a topic. Idempotent on
+    ``(topic_id, event_id)``.
+
+    Validated before anything is emitted: the topic must exist (and be a topic),
+    the event must exist, and ``thread_id`` must be the cited event's actual
+    thread — a citation is a claim about a real message, and the kg log is truth,
+    so a bad reference must be rejected here rather than replayed forever."""
+    own = session is None
+    with use_session(session) as s:
+        _require_topic(s, topic_id)
+        cited = s.get(Event, int(event_id))
+        if cited is None:
+            raise ValueError(
+                f"no event with id {event_id} — citations must reference a real archived message"
+            )
+        if int(cited.thread_id) != int(thread_id):
+            raise ValueError(
+                f"event {event_id} belongs to thread {cited.thread_id}, not {thread_id} — "
+                "check which message you meant to cite"
+            )
+        ev = _emit(
+            s, event_type="evidence.added", entity_type="topic_message",
+            entity_id=f"{int(topic_id)}:{int(event_id)}",
+            payload={"topic_id": int(topic_id), "event_id": int(event_id),
+                     "thread_id": int(thread_id), "quote": quote, "actor": actor},
+            actor=actor, actor_thread_id=actor_thread_id,
+        )
+        result = {"topic_id": int(topic_id), "event_id": int(event_id), "kg_event_id": ev.id}
+        if own:
+            s.commit()
+    return result
+
+
+@_locked_write
+def archive_topic_evidence(
+    topic_id: int, event_id: int, *, actor: str = "librarian",
+    actor_thread_id: Optional[int] = None, session: Optional[Session] = None,
+) -> dict:
+    """Archive a citation (tombstone — sets ``archived_at``, keeps the row + history)."""
+    own = session is None
+    with use_session(session) as s:
+        ev = _emit(
+            s, event_type="evidence.archived", entity_type="topic_message",
+            entity_id=f"{int(topic_id)}:{int(event_id)}",
+            payload={"topic_id": int(topic_id), "event_id": int(event_id)},
+            actor=actor, actor_thread_id=actor_thread_id,
+        )
+        result = {"topic_id": int(topic_id), "event_id": int(event_id), "kg_event_id": ev.id}
+        if own:
+            s.commit()
+    return result
+
+
+# ── curation-read surface ────────────────────────────────────────────────────────
+def review_queue(
+    limit: int = 20, *, exclude_source_id: Optional[str] = None,
+    exclude_ids: Optional[list[int]] = None, session: Optional[Session] = None,
+) -> list[dict]:
+    """Unreviewed conversation threads — the librarian's backlog.
+
+    Event-bearing conversation threads the librarian hasn't curated yet, newest first.
+    Pass ``exclude_source_id`` to drop the caller's own live session (its still-growing
+    transcript sits at the top of its own queue otherwise).
+
+    **State is the data, not a ledger.** 'Reviewed' is *derived from the curation a thread
+    produced* — a thread leaves the queue the instant it gains its first live topic
+    citation (or a link touching it); there is no summary column or processed-list to keep
+    in sync. That makes the queue idempotent (a half-done thread simply reappears) and safe
+    to drain even if two workers briefly overlap. Corollary: a thread genuinely read but
+    yielding nothing worth citing stays in the queue — acceptable, and in practice the
+    librarian-gate forces ≥1 citation per processed thread, so reviewed ⇒ cited.
+
+    **Parallel backfill via lease-claims.** When ``$THREAD_ARCHIVE_LIBRARIAN_WORKER`` is
+    set (the backfill driver sets a distinct id per instance), this call *claims* the batch
+    it returns — recording a timestamped lease in ``<home>/.librarian-claims.json`` so other
+    workers skip it, and reclaiming any lease older than an hour (a dead worker's). Launch N
+    instances with distinct worker ids and they self-balance with no central coordinator.
+    Interactive use (no worker id) is a plain read. ``exclude_ids`` filters explicitly and
+    bypasses the claim path (it's how the claim layer asks for the un-held remainder)."""
+    worker = os.environ.get("THREAD_ARCHIVE_LIBRARIAN_WORKER")
+    if worker and session is None and exclude_ids is None:
+        from ._claims import claim_review_batch
+
+        return claim_review_batch(worker, batch=limit, exclude_source_id=exclude_source_id)
+
+    has_events = select(Event.id).where(Event.thread_id == Thread.id).exists()
+    # 'Curated' = the librarian drew something from this thread: a live (non-archived)
+    # topic citation sourced from it, or a link touching it. Review state is
+    # derived from the curation the thread produced, never from a summary
+    # column (there is none).
+    is_curated = or_(
+        select(TopicMessage.id)
+        .where(TopicMessage.thread_id == Thread.id, TopicMessage.archived_at.is_(None))
+        .exists(),
+        select(ThreadLink.id)
+        .where(
+            or_(
+                ThreadLink.source_thread_id == Thread.id,
+                ThreadLink.target_thread_id == Thread.id,
+            )
+        )
+        .exists(),
+    )
+    conds = [
+        Thread.thread_type == "conversation",
+        ~is_curated,
+        or_(Thread.archived.is_(False), Thread.archived.is_(None)),
+        has_events,
+    ]
+    if exclude_source_id:
+        conds.append(or_(Thread.source_id.is_(None), Thread.source_id != exclude_source_id))
+    if exclude_ids:
+        conds.append(Thread.id.notin_(exclude_ids))
+    with use_session(session) as s:
+        rows = s.execute(
+            select(Thread.id, Thread.title, Thread.source_id)
+            .where(*conds).order_by(Thread.id.desc()).limit(limit)
+        ).all()
+    return [{"id": r.id, "title": r.title, "source_id": r.source_id} for r in rows]
+
+
+def topic_search(query: str, limit: int = 10, *, session: Optional[Session] = None) -> list[dict]:
+    """Find existing live topics by title substring — search before creating duplicates."""
+    like = f"%{query}%"
+    with use_session(session) as s:
+        rows = s.execute(
+            select(Thread.id, Thread.title).where(
+                Thread.thread_type == "topic",
+                or_(Thread.archived.is_(False), Thread.archived.is_(None)),
+                Thread.title.ilike(like),
+            ).order_by(Thread.id).limit(limit)
+        ).all()
+    return [{"topic_id": r.id, "title": r.title} for r in rows]
+
+
+def thread_user_messages(
+    thread_id: int, *, limit: Optional[int] = None, session: Optional[Session] = None,
+) -> list[dict]:
+    """A thread's user messages as ``[{event_id, text}]`` — the cheap, high-signal read
+    the librarian cites from (citations anchor on these ``event_id`` values)."""
+    q = select(Event.id, Event.payload).where(
+        Event.thread_id == int(thread_id),
+        Event.event_type.in_(("user_message_sent", "thread_message_sent")),
+    ).order_by(Event.id)
+    if limit:
+        q = q.limit(limit)
+    out: list[dict] = []
+    with use_session(session) as s:
+        for eid, payload in s.execute(q):
+            p = payload if isinstance(payload, dict) else json.loads(payload)
+            content = (p.get("content") or "").strip()
+            if content:
+                out.append({"event_id": eid, "text": content})
+    return out
+
+
+def _require_topic(session: Session, topic_id: int) -> Thread:
+    topic = session.get(Thread, int(topic_id))
+    if topic is None or topic.thread_type != "topic":
+        raise ValueError(f"no topic with id {topic_id}")
+    return topic

thread_archive/_launchd.py

@@ -0,0 +1,167 @@
+"""LaunchAgent management for the always-on watcher — private machinery.
+
+``archive daemon install`` materializes the watcher's LaunchAgent plist from
+inside the package — pointing at the installed ``archive`` console script,
+wherever this Python environment put it — and loads it. No repo checkout, no
+Makefile, no sed: this is the "upgrade to always-fresh" step after a plain
+``pip install thread-archive``, and it is what ``host/Makefile install-agent``
+delegates to for the operator flow (which adds the thread-family manifest on
+top; the manifest writer stays in ``host/``, repo-only by design).
+
+macOS only, deliberately (launchd is the product's process manager). The
+plist mirrors what the watcher needs and nothing else: run at login in the
+Aqua session (the watched stores live in the user's home), restart on crash,
+logs under ``<archive home>/logs``.
+"""
+
+from __future__ import annotations
+
+import plistlib
+import shutil
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Optional
+
+WATCHER_LABEL = "com.thread-archive.watcher"
+
+
+def _require_darwin() -> None:
+    if sys.platform != "darwin":
+        raise SystemExit("archive daemon: launchd management is macOS-only")
+
+
+def _plist_path() -> Path:
+    return Path.home() / "Library" / "LaunchAgents" / f"{WATCHER_LABEL}.plist"
+
+
+def _entry_path() -> Path:
+    """The ``archive`` console script this environment installed — the thing
+    the plist must point at (launchd won't inherit a venv)."""
+    candidate = Path(sys.executable).with_name("archive")
+    if candidate.is_file():
+        return candidate
+    found = shutil.which("archive")
+    if found:
+        return Path(found)
+    raise SystemExit(
+        "archive daemon: cannot find the `archive` console script next to "
+        f"{sys.executable} or on PATH — is the package installed in this environment?"
+    )
+
+
+def watcher_plist(
+    entry: Path,
+    log_dir: Path,
+    *,
+    home: Optional[str] = None,
+    web: bool = True,
+    web_port: int = 8787,
+) -> dict:
+    """The watcher LaunchAgent as a plist dict (pure — no filesystem, no launchctl)."""
+    args = [str(entry), "watch"]
+    if web:
+        # Cohost the read-only viewer in the watcher's own process so it has a
+        # persistent URL — one process, one engine; WAL makes the concurrent
+        # reads safe (see _store/_base.py).
+        args += ["--web", "--web-port", str(web_port)]
+    env = {
+        # The entry's own bin dir first so `archive` resolves its interpreter.
+        "PATH": f"{entry.parent}:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin",
+    }
+    if home:
+        env["THREAD_ARCHIVE_HOME"] = home
+    return {
+        "Label": WATCHER_LABEL,
+        "ProgramArguments": args,
+        "RunAtLoad": True,
+        # The user's GUI login session — that's where the watched home-dir
+        # stores (~/.claude, ~/.codex, Cursor, …) live.
+        "LimitLoadToSessionType": "Aqua",
+        "KeepAlive": {"Crashed": True},
+        "ThrottleInterval": 5,
+        # A live working-memory indexer: don't let App Nap stall the poll cadence.
+        "ProcessType": "Standard",
+        "WorkingDirectory": str(Path.home()),
+        "StandardOutPath": str(log_dir / "watcher-stdout.log"),
+        "StandardErrorPath": str(log_dir / "watcher-stderr.log"),
+        "EnvironmentVariables": env,
+    }
+
+
+def _uid() -> int:
+    import os
+
+    return os.getuid()
+
+
+def _launchctl(*args: str, check: bool = False) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["launchctl", *args], check=check, capture_output=True, text=True
+    )
+
+
+def install_watcher(
+    home: Optional[str] = None, *, web: bool = True, web_port: int = 8787
+) -> Path:
+    """Write the plist and (re)load the agent. Returns the plist path.
+
+    Idempotent: an existing agent is booted out first, so re-running after an
+    upgrade or a config change is the supported way to apply it.
+    """
+    _require_darwin()
+    from ._config import resolve_paths
+
+    entry = _entry_path()
+    log_dir = resolve_paths(home).home / "logs"
+    # launchd won't mkdir StandardOutPath's parent — a missing dir fails the
+    # load silently.
+    log_dir.mkdir(parents=True, exist_ok=True)
+    plist = _plist_path()
+    plist.parent.mkdir(parents=True, exist_ok=True)
+    plist.write_bytes(
+        plistlib.dumps(watcher_plist(entry, log_dir, home=home, web=web, web_port=web_port))
+    )
+    domain = f"gui/{_uid()}"
+    if _launchctl("bootout", f"{domain}/{WATCHER_LABEL}").returncode == 0:
+        # Let bootout settle before bootstrap (avoids 'Bootstrap failed: 5:
+        # Input/output error').
+        time.sleep(3)
+    result = _launchctl("bootstrap", domain, str(plist))
+    if result.returncode != 0:
+        raise SystemExit(
+            f"archive daemon: launchctl bootstrap failed: {result.stderr.strip()}"
+        )
+    return plist
+
+
+def uninstall_watcher() -> None:
+    _require_darwin()
+    _launchctl("bootout", f"gui/{_uid()}/{WATCHER_LABEL}")
+    _plist_path().unlink(missing_ok=True)
+
+
+def restart_watcher() -> None:
+    """Apply a code edit: kick the running agent (the plist itself is only
+    re-read on install)."""
+    _require_darwin()
+    result = _launchctl("kickstart", "-k", f"gui/{_uid()}/{WATCHER_LABEL}")
+    if result.returncode != 0:
+        raise SystemExit(
+            f"archive daemon: launchctl kickstart failed: {result.stderr.strip()}"
+        )
+
+
+def watcher_status() -> str:
+    """A short human-readable status: state/pid/program, or 'not loaded'."""
+    _require_darwin()
+    result = _launchctl("print", f"gui/{_uid()}/{WATCHER_LABEL}")
+    if result.returncode != 0:
+        return f"{WATCHER_LABEL}: not loaded"
+    lines = [
+        ln.strip()
+        for ln in result.stdout.splitlines()
+        if any(k in ln for k in ("state =", "pid =", "program ="))
+    ]
+    return "\n".join([WATCHER_LABEL, *lines])

thread_archive/_mcp/__init__.py

@@ -0,0 +1 @@
+"""The MCP agent surface: ``thread_search`` + ``thread_read`` over the local archive."""