thread-archive 0.0.2__py3-none-any.whl

thread_archive/_knowledge/_claims.py

@@ -0,0 +1,116 @@
+"""Lease-based work claims for parallel librarian backfill.
+
+Static `id % N` sharding is rigid: you fix the worker count up front, a dead shard
+stalls, and the modulo isn't index-sargable. Instead, workers **claim** a batch of
+thread ids — stamped with a timestamp — into a shared state file, and a claim older than
+the lease (default 1h) is treated as a dead worker's and reclaimed. So workers
+self-balance (a fast one claims more), you can launch any number independently with no
+coordinator, and a crash forfeits nothing but a one-hour wait on those few threads.
+
+The claim file (`<home>/.librarian-claims.json`) is **ephemeral operator coordination**,
+not truth — like a lock dir. Deleting it only drops in-flight leases; the archive (JSONL
+truth + the topic citations/links that actually mark a thread done) is untouched. Concurrency
+is an `flock`'d read-modify-write held only for the brief claim, never during the work.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import json
+import os
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Optional
+
+DEFAULT_LEASE_SECONDS = 3600  # a claim older than this is a presumed-dead worker's
+
+
+def claims_path() -> Path:
+    from .._config import resolve_paths
+
+    return resolve_paths().home / ".librarian-claims.json"
+
+
+def _now() -> float:
+    return time.time()
+
+
+@contextmanager
+def _locked(path: Path):
+    """Yield the claims dict under an exclusive lock; persist it on clean exit.
+
+    The lock is the claim file's own fd (``flock`` mutexes across processes *and*
+    threads on separate descriptions). A corrupt file reads as empty. If the body
+    raises, the file is left unchanged — a failed claim never half-writes."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(str(path), os.O_RDWR | os.O_CREAT, 0o644)
+    try:
+        fcntl.flock(fd, fcntl.LOCK_EX)
+        chunks = []
+        while True:
+            block = os.read(fd, 65536)
+            if not block:
+                break
+            chunks.append(block)
+        raw = b"".join(chunks)
+        try:
+            claims = json.loads(raw) if raw.strip() else {}
+        except ValueError:
+            claims = {}
+        yield claims
+        os.lseek(fd, 0, 0)
+        os.ftruncate(fd, 0)
+        os.write(fd, json.dumps(claims).encode("utf-8"))
+    finally:
+        fcntl.flock(fd, fcntl.LOCK_UN)
+        os.close(fd)
+
+
+def _prune_expired(claims: dict, lease: int, now: float) -> None:
+    for tid in list(claims):
+        if now - float(claims[tid].get("at", 0)) >= lease:
+            del claims[tid]
+
+
+def claim_review_batch(
+    worker: str, batch: int, *, lease: int = DEFAULT_LEASE_SECONDS,
+    exclude_source_id: Optional[str] = None,
+) -> list[dict]:
+    """Claim (and return) up to ``batch`` unreviewed conversations for ``worker``.
+
+    Under the lock: expire stale claims, ask the queue for eligible threads not claimed
+    by *other* live workers (resuming this worker's own still-unreviewed claims first),
+    stamp them to now, and drop this worker's claims that have since left the queue
+    (reviewed). Returns the queue rows the caller should process."""
+    from .write import review_queue
+
+    now = _now()
+    with _locked(claims_path()) as claims:
+        _prune_expired(claims, lease, now)
+        others = [int(t) for t, c in claims.items() if c.get("worker") != worker]
+        rows = review_queue(
+            limit=batch, exclude_source_id=exclude_source_id, exclude_ids=others
+        )
+        returned = {r["id"] for r in rows}
+        for r in rows:
+            claims[str(r["id"])] = {"worker": worker, "at": now}
+        # forget my claims for threads no longer eligible (reviewed, or bumped out of
+        # the window) so stale leases don't accumulate under my name
+        for tid in [t for t, c in claims.items() if c.get("worker") == worker and int(t) not in returned]:
+            del claims[tid]
+        return rows
+
+
+def has_claimable_work(
+    *, lease: int = DEFAULT_LEASE_SECONDS, exclude_source_id: Optional[str] = None
+) -> bool:
+    """Whether any unreviewed conversation is free to claim (not held by a live lease).
+    The driver's respawn condition."""
+    from .write import review_queue
+
+    now = _now()
+    with _locked(claims_path()) as claims:
+        _prune_expired(claims, lease, now)
+        active = [int(t) for t in claims]
+        return bool(review_queue(limit=1, exclude_source_id=exclude_source_id, exclude_ids=active))

thread_archive/_knowledge/_community.py

@@ -0,0 +1,75 @@
+"""Community detection for the topic graph — Leiden, with a Louvain fallback.
+
+Leiden (Traag et al. 2019) is the algorithm Neo4j GDS ran on the original graph; it
+strictly dominates Louvain — it guarantees well-connected communities (Louvain can
+leave a community internally disconnected) and converges to a better partition. We run
+it locally via ``leidenalg`` + ``python-igraph`` (base dependencies), with no graph
+server — Leiden is the community engine for the topic graph.
+
+:func:`detect_communities` still falls back to networkx's Louvain (free with the base
+``networkx`` dependency) if those C extensions ever fail to import, so a read path can
+never crash on community detection. Both paths are **seeded** for a deterministic
+partition across runs.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import networkx as nx
+from networkx.algorithms.community import louvain_communities
+
+logger = logging.getLogger(__name__)
+
+# Seeded so a given graph yields a stable partition across calls/runs (both engines).
+SEED = 42
+
+_LEIDEN_AVAILABLE: bool | None = None
+
+
+def leiden_available() -> bool:
+    """Whether the Leiden engine (``leidenalg`` + ``igraph``) can be imported. They are
+    base dependencies, so this is normally true; it guards the fail-soft fallback for
+    the rare environment where the C extensions can't load. Memoized — probes once."""
+    global _LEIDEN_AVAILABLE
+    if _LEIDEN_AVAILABLE is None:
+        try:
+            import igraph  # noqa: F401
+            import leidenalg  # noqa: F401
+
+            _LEIDEN_AVAILABLE = True
+        except Exception:
+            _LEIDEN_AVAILABLE = False
+    return _LEIDEN_AVAILABLE
+
+
+def detect_communities(graph: "nx.Graph") -> list[list[int]]:
+    """Partition ``graph`` into communities (lists of node ids), maximizing modularity.
+
+    Uses Leiden when available, else networkx Louvain. Edge ``weight`` attributes are
+    honored by both. Returns ``[]`` for an empty graph. Each community is sorted, and
+    the list is order-stable, so the partition is reproducible."""
+    if graph.number_of_nodes() == 0:
+        return []
+    if leiden_available():
+        try:
+            return _leiden(graph)
+        except Exception:  # pragma: no cover — fall back rather than fail the read path
+            logger.exception("leiden community detection failed — falling back to louvain")
+    return [sorted(c) for c in louvain_communities(graph, weight="weight", seed=SEED)]
+
+
+def _leiden(graph: "nx.Graph") -> list[list[int]]:
+    import igraph as ig
+    import leidenalg as la
+
+    nodes = list(graph.nodes())
+    index = {n: i for i, n in enumerate(nodes)}
+    edges = [(index[u], index[v]) for u, v in graph.edges()]
+    weights = [float(graph[u][v].get("weight", 1.0)) for u, v in graph.edges()]
+
+    g = ig.Graph(n=len(nodes), edges=edges, directed=False)
+    partition = la.find_partition(
+        g, la.ModularityVertexPartition, weights=weights, seed=SEED
+    )
+    return [sorted(nodes[i] for i in community) for community in partition]

thread_archive/_knowledge/graph.py

@@ -0,0 +1,208 @@
+"""In-process topic-graph projection (networkx spine, optional Leiden community engine).
+
+A pure-graph topic projection. The graph is rebuildable from ``thread_links`` + topic
+``threads``, so there is no graph server: build a networkx graph from SQLite and run
+the algorithms in process.
+
+Two projections:
+  * **PageRank** — every link, undirected, weight = ``strength``; ``pagerank(alpha=0.85)``.
+  * **Community** — drop oppositional (contrast/supersedes) + operational
+    (works_on/investigates/benchmark) edges, ×0.7 the instantiation edges
+    (implements/example-of); then :func:`thread_archive._knowledge._community.detect_communities`
+    — Leiden (the algorithm Neo4j GDS used; a base dependency), with networkx Louvain
+    as a seeded, deterministic fail-soft fallback.
+
+Topics are threads with ``thread_type='topic'``; the graph is populated as the
+librarian curates (or by a migration of existing topics + links). The graph is empty
+(and all queries return empty) until then — by design (the core archive works without
+this layer).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+import networkx as nx
+from sqlalchemy import text as sa_text
+
+from .._store import get_engine, get_session
+from ._community import detect_communities
+
+logger = logging.getLogger(__name__)
+
+# Edge-type families. The community projection excludes oppositional + operational
+# edges and down-weights instantiation.
+_OPPOSITIONAL = frozenset({"contrast", "supersedes"})
+_OPERATIONAL = frozenset({"works_on", "investigates", "benchmark"})
+_INSTANTIATION = frozenset({"implements", "example-of"})
+
+# Lazy per-engine projection cache. {id(engine): _Projection}. Reset on rebuild.
+_CACHE: dict = {}
+
+
+class _Projection:
+    """The computed topic-graph state for one engine snapshot."""
+
+    __slots__ = ("pagerank", "community", "degree", "component", "members", "titles",
+                 "_pr_graph", "_kn_graph")
+
+    def __init__(self, pr_graph: "nx.Graph", kn_graph: "nx.Graph", titles: dict[int, str]):
+        self._pr_graph = pr_graph
+        self._kn_graph = kn_graph
+        self.titles = titles
+        self.pagerank: dict[int, float] = (
+            nx.pagerank(pr_graph, alpha=0.85, weight="weight") if pr_graph.number_of_nodes() else {}
+        )
+        self.degree: dict[int, int] = dict(pr_graph.degree())
+        self.component: dict[int, int] = {}
+        for cid, comp in enumerate(nx.connected_components(pr_graph)):
+            for n in comp:
+                self.component[n] = cid
+        self.community: dict[int, int] = {}
+        self.members: dict[int, list[int]] = {}
+        for cid, comm in enumerate(detect_communities(kn_graph)):
+            self.members[cid] = sorted(comm)
+            for n in comm:
+                self.community[n] = cid
+
+    def betweenness(self) -> dict[int, float]:
+        if not self._pr_graph.number_of_nodes():
+            return {}
+        return nx.betweenness_centrality(self._pr_graph, weight="weight")
+
+
+def is_available() -> bool:
+    try:
+        return get_engine().dialect.name == "sqlite"
+    except Exception:
+        return False
+
+
+def reset_cache() -> None:
+    """Drop the cached projection (call after the KG mutates / on rebuild)."""
+    _CACHE.clear()
+
+
+def _build_projection() -> _Projection:
+    pr_graph = nx.Graph()
+    kn_graph = nx.Graph()
+    with get_session() as s:
+        titles = {
+            int(r[0]): r[1] for r in s.execute(sa_text(
+                "SELECT id, title FROM threads WHERE thread_type = 'topic' "
+                "AND (archived IS NULL OR archived = 0)"
+            ))
+        }
+        links = s.execute(sa_text(
+            "SELECT source_thread_id, target_thread_id, link_type, "
+            "COALESCE(strength, 1.0) AS w FROM thread_links"
+        )).all()
+
+    topic_ids = set(titles)
+    pr_graph.add_nodes_from(topic_ids)
+    kn_graph.add_nodes_from(topic_ids)
+    for src, tgt, link_type, w in links:
+        if src == tgt or src not in topic_ids or tgt not in topic_ids:
+            continue
+        weight = float(w)
+        _add_weighted(pr_graph, src, tgt, weight)
+        if link_type not in _OPPOSITIONAL and link_type not in _OPERATIONAL:
+            _add_weighted(kn_graph, src, tgt, weight * (0.7 if link_type in _INSTANTIATION else 1.0))
+    return _Projection(pr_graph, kn_graph, titles)
+
+
+def _add_weighted(g: "nx.Graph", a: int, b: int, w: float) -> None:
+    if g.has_edge(a, b):
+        g[a][b]["weight"] += w
+    else:
+        g.add_edge(a, b, weight=w)
+
+
+def _projection() -> Optional[_Projection]:
+    if not is_available():
+        return None
+    key = id(get_engine())
+    proj = _CACHE.get(key)
+    if proj is None:
+        proj = _build_projection()
+        _CACHE[key] = proj
+    return proj
+
+
+def get_topic_graph_metadata(thread_ids: list[int]) -> dict[int, dict]:
+    """Batch graph properties (pagerank / community / degree / link_count) for topics."""
+    proj = _projection()
+    if proj is None or not thread_ids:
+        return {}
+    out: dict[int, dict] = {}
+    for tid in thread_ids:
+        if tid not in proj.degree and tid not in proj.pagerank:
+            continue
+        deg = proj.degree.get(tid, 0)
+        out[tid] = {
+            "pagerank": proj.pagerank.get(tid, 0.0),
+            "community": proj.community.get(tid),
+            "degree": deg,
+            "link_count": deg,
+        }
+    return out
+
+
+def get_topic_graph_meta(thread_id: int) -> dict | None:
+    return get_topic_graph_metadata([thread_id]).get(thread_id)
+
+
+def get_community_topic_ids(community_id: int) -> list[int]:
+    proj = _projection()
+    if proj is None:
+        return []
+    return list(proj.members.get(community_id, []))
+
+
+def get_community_peers(thread_id: int, limit: int = 5) -> list[dict]:
+    """Other topics in the same community, highest-pagerank first."""
+    proj = _projection()
+    if proj is None:
+        return []
+    cid = proj.community.get(thread_id)
+    if cid is None:
+        return []
+    peers = [t for t in proj.members.get(cid, []) if t != thread_id]
+    peers.sort(key=lambda t: (-proj.pagerank.get(t, 0.0), t))
+    return [{"thread_id": t, "title": proj.titles.get(t), "pagerank": proj.pagerank.get(t, 0.0)}
+            for t in peers[:limit]]
+
+
+def get_bridge_topics(limit: int = 20) -> list[dict]:
+    """Highest-betweenness topics — structural bridges between communities."""
+    proj = _projection()
+    if proj is None:
+        return []
+    bc = proj.betweenness()
+    ranked = [(t, sc) for t, sc in sorted(bc.items(), key=lambda kv: (-kv[1], kv[0])) if sc > 0][:limit]
+    return [{"topic_id": t, "title": proj.titles.get(t), "score": score} for t, score in ranked]
+
+
+def get_unconnected_topics(limit: int = 50) -> list[dict]:
+    """Topics with no link — the knowledge islands."""
+    proj = _projection()
+    if proj is None:
+        return []
+    isolated = [t for t, d in proj.degree.items() if d == 0][:limit]
+    return [{"thread_id": t, "link_count": 0} for t in isolated]
+
+
+def get_status() -> dict:
+    proj = _projection()
+    if proj is None:
+        return {"available": False, "error": "store is not sqlite"}
+    from ._community import leiden_available
+
+    return {
+        "available": True,
+        "nodes": len(proj.degree),
+        "communities": len(proj.members),
+        "components": len(set(proj.component.values())),
+        "community_engine": "leiden" if leiden_available() else "louvain",
+    }

thread_archive/_knowledge/materialize.py

@@ -0,0 +1,212 @@
+"""The materializer: fold curatorial events into the knowledge projection.
+
+A ``KgEvent`` is the unit of curation truth; this module is the **fold** that turns
+the append-only event log into the ``thread_links`` / ``topic_messages`` SQLite
+tables the topic graph reads. It is the serverless analogue of a streaming
+materializer — but in-process, folding to SQLite rather than to a graph server.
+
+:func:`apply_event` dispatches one event to its projection mutation. It is called
+in two places, both with the session it must mutate:
+
+* **live write** — :mod:`thread_archive._knowledge.write` appends an event *and*
+  applies it in the same transaction, so one call yields a durable log line plus an
+  updated projection;
+* **reindex replay** — ``truth.jsonl_log`` replays the whole log in ``id`` order to
+  rebuild the projection from scratch (on top of any legacy snapshot seed).
+
+Every mutation is **upsert + tombstone**, never blind insert: ``link.created`` on an
+existing edge updates it, ``link.deleted`` removes it whether it came from the log or
+a seed snapshot, ``evidence.archived`` stamps rather than deletes. That is what makes
+the replay idempotent and order-stable — applying the same log twice, or folding a
+delta onto a seed, lands on the same projection.
+
+Timestamps come from the event (``occurred_at``), not the moment of replay, so a
+rebuild is deterministic: a link's ``created_at`` is when it was *made*, every time.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Callable
+
+from sqlalchemy import delete, select, update
+
+from .._store import Event, Thread, ThreadLink, TopicMessage
+
+logger = logging.getLogger(__name__)
+
+
+def apply_event(session, ev) -> None:
+    """Fold one ``KgEvent`` into the projection on ``session`` (no commit).
+
+    ``ev`` is any object exposing ``event_type`` / ``payload`` / ``actor`` /
+    ``actor_thread_id`` / ``occurred_at`` (a live ``KgEvent`` or a transient one
+    rebuilt from a log row). Unknown event types are logged and skipped — a forward-
+    compatible reader never aborts a replay on an event it doesn't understand."""
+    fn = _DISPATCH.get(ev.event_type)
+    if fn is None:
+        logger.warning("kg materialize: unknown event_type %r — skipped", ev.event_type)
+        return
+    fn(session, ev.payload or {}, ev)
+
+
+# ── topics (topics are threads; the event log carries audit + lifecycle) ──────
+def _topic_created(session, p: dict, ev) -> None:
+    # The topic thread itself is created + recorded by the write path (it has its own
+    # per-thread truth file); this event is the audit record. Nothing to project.
+    return
+
+
+def _topic_renamed(session, p: dict, ev) -> None:
+    fields = {k: p[k] for k in ("title", "description") if k in p}
+    if not fields:
+        return
+    _update_thread(session, _entity_int(ev), fields)
+
+
+def _topic_updated(session, p: dict, ev) -> None:
+    fields = p.get("fields", p)
+    allowed = {k: v for k, v in fields.items()
+               if k in ("title", "description", "topic_kind", "epistemological_type", "search_description")}
+    if allowed:
+        _update_thread(session, _entity_int(ev), allowed)
+
+
+def _topic_archived(session, p: dict, ev) -> None:
+    _update_thread(session, _entity_int(ev), {"archived": True})
+
+
+def _topic_merged(session, p: dict, ev) -> None:
+    """Repoint every link + evidence from ``from_id`` onto ``into_id``, then archive
+    ``from_id``. Repointing can collide with an existing edge/evidence (unique keys),
+    so it is done row-by-row with a duplicate guard rather than a blind UPDATE."""
+    from_id, into_id = int(p["from_id"]), int(p["into_id"])
+    if from_id == into_id:
+        return
+    _repoint_links(session, from_id, into_id)
+    _repoint_evidence(session, from_id, into_id)
+    _update_thread(session, from_id, {"archived": True})
+
+
+# ── links (the topic-graph edge set) ──────────────────────────────────────────
+def _link_created(session, p: dict, ev) -> None:
+    src, tgt, lt = int(p["source_thread_id"]), int(p["target_thread_id"]), p.get("link_type", "related")
+    row = _find_link(session, src, tgt, lt)
+    if row is None:
+        row = ThreadLink(source_thread_id=src, target_thread_id=tgt, link_type=lt, created_at=ev.occurred_at)
+        session.add(row)
+    row.strength = float(p.get("strength", 1.0))
+    row.evidence = p.get("evidence")
+    row.created_by = p.get("created_by") or getattr(ev, "actor", "librarian")
+    row.created_by_thread_id = getattr(ev, "actor_thread_id", None)
+    row.updated_at = ev.occurred_at
+
+
+def _link_updated(session, p: dict, ev) -> None:
+    row = _find_link(session, int(p["source_thread_id"]), int(p["target_thread_id"]),
+                     p.get("link_type", "related"))
+    if row is None:
+        return
+    for k, v in (p.get("fields") or {}).items():
+        if k in ("strength", "evidence", "link_type"):
+            setattr(row, k, v)
+    row.updated_at = ev.occurred_at
+
+
+def _link_deleted(session, p: dict, ev) -> None:
+    session.execute(delete(ThreadLink).where(
+        ThreadLink.source_thread_id == int(p["source_thread_id"]),
+        ThreadLink.target_thread_id == int(p["target_thread_id"]),
+        ThreadLink.link_type == p.get("link_type", "related"),
+    ))
+
+
+# ── evidence (message → topic citations) ──────────────────────────────────────
+def _evidence_added(session, p: dict, ev) -> None:
+    topic_id, event_id = int(p["topic_id"]), int(p["event_id"])
+    # The cited event's own row is authoritative for the thread; the payload
+    # value is the fallback for a citation whose event the store doesn't hold
+    # (a dangling reference replayed from an older, unvalidated write). Looked
+    # up before the row is added — a query would autoflush the incomplete row.
+    cited = session.get(Event, event_id)
+    row = _find_evidence(session, topic_id, event_id)
+    if row is None:
+        row = TopicMessage(topic_id=topic_id, event_id=event_id, created_at=ev.occurred_at)
+        session.add(row)
+    row.thread_id = int(cited.thread_id) if cited is not None else int(p["thread_id"])
+    row.quote = p.get("quote", "")
+    row.actor = p.get("actor") or getattr(ev, "actor", "librarian")
+    row.created_by_thread_id = getattr(ev, "actor_thread_id", None)
+    row.archived_at = None
+
+
+def _evidence_archived(session, p: dict, ev) -> None:
+    session.execute(update(TopicMessage).where(
+        TopicMessage.topic_id == int(p["topic_id"]),
+        TopicMessage.event_id == int(p["event_id"]),
+    ).values(archived_at=ev.occurred_at))
+
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+def _entity_int(ev) -> int:
+    return int(ev.entity_id)
+
+
+def _update_thread(session, thread_id: int, fields: dict) -> None:
+    session.execute(update(Thread).where(Thread.id == thread_id).values(**fields))
+
+
+def _find_link(session, src: int, tgt: int, link_type: str):
+    return session.execute(select(ThreadLink).where(
+        ThreadLink.source_thread_id == src,
+        ThreadLink.target_thread_id == tgt,
+        ThreadLink.link_type == link_type,
+    )).scalars().first()
+
+
+def _find_evidence(session, topic_id: int, event_id: int):
+    return session.execute(select(TopicMessage).where(
+        TopicMessage.topic_id == topic_id,
+        TopicMessage.event_id == event_id,
+    )).scalars().first()
+
+
+def _repoint_links(session, from_id: int, into_id: int) -> None:
+    links = session.execute(select(ThreadLink).where(
+        (ThreadLink.source_thread_id == from_id) | (ThreadLink.target_thread_id == from_id)
+    )).scalars().all()
+    for link in links:
+        new_src = into_id if link.source_thread_id == from_id else link.source_thread_id
+        new_tgt = into_id if link.target_thread_id == from_id else link.target_thread_id
+        if new_src == new_tgt:  # a self-loop after merge — drop it
+            session.delete(link)
+            continue
+        if _find_link(session, new_src, new_tgt, link.link_type) is not None:
+            session.delete(link)  # the merged-into edge already exists — collapse
+            continue
+        link.source_thread_id, link.target_thread_id = new_src, new_tgt
+
+
+def _repoint_evidence(session, from_id: int, into_id: int) -> None:
+    rows = session.execute(
+        select(TopicMessage).where(TopicMessage.topic_id == from_id)
+    ).scalars().all()
+    for row in rows:
+        if _find_evidence(session, into_id, row.event_id) is not None:
+            session.delete(row)  # citation already exists on the merged-into topic
+            continue
+        row.topic_id = into_id
+
+
+_DISPATCH: dict[str, Callable] = {
+    "topic.created": _topic_created,
+    "topic.renamed": _topic_renamed,
+    "topic.updated": _topic_updated,
+    "topic.archived": _topic_archived,
+    "topic.merged": _topic_merged,
+    "link.created": _link_created,
+    "link.updated": _link_updated,
+    "link.deleted": _link_deleted,
+    "evidence.added": _evidence_added,
+    "evidence.archived": _evidence_archived,
+}