topicsleuth 0.1.0__py3-none-any.whl

topicsleuth/add_topic.py

@@ -0,0 +1,238 @@
+"""Add a single topic to a populated corpus.
+
+`seed` only runs on an empty topics collection and `split-topic` retires a
+parent to extract children, so until now there was no supported way to add a
+topic the pipeline had not proposed on its own. This takes a topic definition
+(the seed_topics.yaml entry shape, plus an optional topic_id and an optional
+list of item URLs whose observations should move onto the new topic as its
+founding evidence), generates the embedding so vector search can match
+against it, inserts the document, and records snapshots on every topic touched.
+
+Moved observations keep their strength and connection type; their reasoning
+trace gains a line saying where they came from, so the audit trail survives.
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+
+from topicsleuth.history import removed_history
+from topicsleuth.interfaces import Embedder, Store
+from topicsleuth.synthesize import _sanitize_topic_id
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TopicDefinition:
+    title: str
+    summary: str
+    topic_id: str
+    tags: list[str] = field(default_factory=list)
+    suggested_directions: list[str] = field(default_factory=list)
+    link: list[str] = field(default_factory=list)  # item URLs to move onto the topic
+
+
+def load_topic_definition(path: Path) -> TopicDefinition:
+    raw = yaml.safe_load(Path(path).read_text()) or {}
+    if "topics" in raw:
+        raise ValueError(
+            "add-topic takes one topic definition, not a seed_topics.yaml file "
+            "with a `topics:` list"
+        )
+    title = (raw.get("title") or "").strip()
+    summary = (raw.get("summary") or "").strip()
+    if not title or not summary:
+        raise ValueError("topic definition needs a title and a summary")
+    topic_id = _sanitize_topic_id(raw.get("topic_id") or title)
+    if not topic_id:
+        raise ValueError("could not derive a topic_id from the title")
+    return TopicDefinition(
+        title=title,
+        summary=summary,
+        topic_id=topic_id,
+        tags=[str(t) for t in raw.get("tags") or []],
+        suggested_directions=[str(d) for d in raw.get("suggested_directions") or []],
+        link=[str(u) for u in raw.get("link") or []],
+    )
+
+
+def plan_moves(store: Store, urls: list[str], target_topic_id: str) -> tuple[list[dict], list[str]]:
+    """Find the observations for the given item URLs. Returns (observations,
+    missing_urls). An item with no observation cannot be moved; it would have
+    to go back through the pipeline."""
+    found: list[dict] = []
+    missing: list[str] = []
+    for url in urls:
+        obs = store.get_observations_for_item(url)
+        obs = [o for o in obs if o.get("topic_id") != target_topic_id]
+        if not obs:
+            missing.append(url)
+        found.extend(obs)
+    return found, missing
+
+
+def add_topic(
+    definition: TopicDefinition,
+    store: Store,
+    embedder: Embedder,
+    dry_run: bool = False,
+) -> dict:
+    if store.get_topic(definition.topic_id):
+        raise ValueError(f"topic already exists: {definition.topic_id}")
+
+    moves, missing = plan_moves(store, definition.link, definition.topic_id)
+    source_counts: dict[str, int] = {}
+    for o in moves:
+        source_counts[o["topic_id"]] = source_counts.get(o["topic_id"], 0) + 1
+
+    report = {
+        "dry_run": dry_run,
+        "topic_id": definition.topic_id,
+        "title": definition.title,
+        "observations_to_move": [
+            {"item_url": o["item_url"], "item_title": o.get("item_title", ""),
+             "from_topic": o["topic_id"], "strength": o.get("strength"),
+             "connection_type": o.get("connection_type")}
+            for o in moves
+        ],
+        "urls_without_observation": missing,
+        "source_topics_affected": source_counts,
+    }
+    if dry_run:
+        return report
+
+    embedding = embedder.embed_documents([f"{definition.title}\n\n{definition.summary}"])[0]
+    now = datetime.now(timezone.utc)
+    store.create_topic({
+        "topic_id": definition.topic_id,
+        "title": definition.title,
+        "summary": definition.summary,
+        "tags": definition.tags,
+        "suggested_directions": definition.suggested_directions,
+        "embedding": embedding,
+        "added_manually": True,
+    })
+    logger.info("Created topic %s", definition.topic_id)
+
+    retired = move_observations(
+        store, moves, definition.topic_id, now, reason="add-topic", retire_empty_sources=False
+    )
+    report["observations_moved"] = len(moves)
+    report["sources_retired"] = retired
+    return report
+
+
+def move_observations(
+    store: Store,
+    observations: list[dict],
+    target_topic_id: str,
+    now: datetime,
+    reason: str,
+    retire_empty_sources: bool = True,
+) -> list[str]:
+    """Re-point observations at target_topic_id, annotate their trace, and
+    re-snapshot every topic touched. With retire_empty_sources, a source
+    topic left with no observations is retired with merged_into=target so
+    the provenance survives (a topic with no evidence is not a topic).
+    Returns the ids of retired source topics."""
+    from topicsleuth.record import _compute_snapshot
+
+    sources: set[str] = set()
+    for o in observations:
+        src = o["topic_id"]
+        if src == target_topic_id:
+            continue
+        sources.add(src)
+        # An item may already be evidence on the target (it bore on both
+        # questions). One observation per item per topic: the target's
+        # survives and the moving one is folded into it, noted in the
+        # trace. The fold is one store operation: the target must be live
+        # when it commits, and the survivor's update and the source's
+        # deletion land together or not at all. The store gives the
+        # survivor the folded observation's plan and history, read at the
+        # moment of the write, so a link that only it carried, approved
+        # and not yet written, survives the fold, and so does whatever
+        # another fold into the same survivor added meanwhile.
+        holder = next((h for h in store.get_observations_for_item(o["item_url"])
+                       if h.get("topic_id") == target_topic_id), None)
+        if holder is not None:
+            note = (
+                f"\n\nFOLDED by {reason} on {now.date().isoformat()}: this item's "
+                f"observation on '{src}' (strength {o.get('strength')}, {o.get('connection_type')}) "
+                f"was merged into this one."
+            )
+            store.fold_observation(o["_id"], holder["_id"], note)
+            logger.info("Folded duplicate observation of %s from %s into %s", o["item_url"], src, target_topic_id)
+            continue
+        note = (
+            f"\n\nMOVED by {reason} on {now.date().isoformat()}: from "
+            f"'{src}' to '{target_topic_id}'."
+        )
+        store.update_observation(o["_id"], {
+            "topic_id": target_topic_id,
+            "reasoning_trace": (o.get("reasoning_trace") or "") + note,
+            "moved_from": src,
+            "removed_from": removed_history(o, src),
+        })
+
+    retired: list[str] = []
+    for tid in [target_topic_id, *sorted(sources)]:
+        store.append_snapshot(tid, _compute_snapshot(store, tid, now))
+        if (
+            retire_empty_sources
+            and tid != target_topic_id
+            and store.count_observations_for_topic(tid) == 0
+        ):
+            topic = store.get_topic(tid)
+            if topic and topic.get("status") != "retired":
+                store.update_topic(tid, {
+                    "status": "retired", "retired_at": now, "merged_into": target_topic_id,
+                })
+                retired.append(tid)
+                logger.info("Retired emptied topic %s (merged_into %s)", tid, target_topic_id)
+    return retired
+
+
+def move_observation(
+    item_url: str,
+    target_topic_id: str,
+    store: Store,
+    dry_run: bool = False,
+) -> dict:
+    """Move one item's observation(s) onto an existing topic."""
+    target = store.get_topic(target_topic_id)
+    if not target:
+        raise ValueError(f"target topic not found: {target_topic_id}")
+    if target.get("status") in ("retired", "deleting"):
+        raise ValueError(f"target topic is {target['status']}: {target_topic_id}")
+    moves, missing = plan_moves(store, [item_url], target_topic_id)
+    if not moves:
+        raise ValueError(
+            f"no observation to move for {item_url}"
+            + (" (already on the target topic or never recorded)" if missing else "")
+        )
+    will_empty = [
+        o["topic_id"] for o in moves
+        if store.count_observations_for_topic(o["topic_id"]) == 1
+    ]
+    report = {
+        "dry_run": dry_run,
+        "target_topic_id": target_topic_id,
+        "moves": [
+            {"item_title": o.get("item_title", ""), "from_topic": o["topic_id"],
+             "strength": o.get("strength"), "connection_type": o.get("connection_type")}
+            for o in moves
+        ],
+        "sources_that_would_empty": sorted(set(will_empty)),
+    }
+    if dry_run:
+        return report
+    report["sources_retired"] = move_observations(
+        store, moves, target_topic_id, datetime.now(timezone.utc), reason="move-observation"
+    )
+    return report

topicsleuth/audit.py

@@ -0,0 +1,123 @@
+"""Snapshot the current store state for before/after comparison.
+
+Produces a markdown report covering topic counts, observation distribution,
+strength buckets and connection types. Useful for
+diffing across major operations (reprocess-tail, merge-topics, prompt
+changes, etc.).
+"""
+from __future__ import annotations
+
+from collections import Counter
+from datetime import datetime, timezone
+from pathlib import Path
+
+from topicsleuth.interfaces import Store
+
+DEFAULT_OUTPUT_DIR = Path("tuning_runs")  # relative to the working directory
+
+
+def _strength_bucket(s: float) -> str:
+    if s < 0.3: return "<0.3"
+    if s < 0.5: return "0.3-0.5"
+    if s < 0.7: return "0.5-0.7"
+    if s < 0.8: return "0.7-0.8"
+    if s < 0.9: return "0.8-0.9"
+    return "0.9+"
+
+
+def _evidence_bucket(n: int) -> str:
+    if n <= 1: return "0-1"
+    if n <= 4: return "2-4"
+    if n <= 9: return "5-9"
+    if n <= 19: return "10-19"
+    if n <= 29: return "20-29"
+    return "30+"
+
+
+def build_audit_report(store: Store) -> str:
+    """Compute the audit and return it as a markdown string."""
+    topics = store.list_topics()
+    observations = store.list_observations()
+
+    lines: list[str] = []
+    lines.append(f"# Store state — {datetime.now(timezone.utc).isoformat()}")
+    lines.append("")
+
+    lines.append("## Topics")
+    status_counts = Counter(t.get("status", "?") for t in topics)
+    lines.append(f"Total: {len(topics)}")
+    for status, n in status_counts.most_common():
+        lines.append(f"  {status:<12} {n}")
+    lines.append("")
+
+    obs_by_topic: Counter = Counter()
+    for o in observations:
+        obs_by_topic[o["topic_id"]] += 1
+
+    lines.append("## Topics by observation count (non-retired)")
+    bucket_counts: Counter = Counter()
+    for topic in topics:
+        if topic.get("status") == "retired":
+            continue
+        n = obs_by_topic.get(topic["topic_id"], 0)
+        bucket_counts[_evidence_bucket(n)] += 1
+    for bucket in ("0-1", "2-4", "5-9", "10-19", "20-29", "30+"):
+        lines.append(f"  {bucket:<6} {bucket_counts.get(bucket, 0)}")
+    lines.append("")
+
+    lines.append("## Top 15 topics by observation count")
+    topic_lookup = {t["topic_id"]: t for t in topics}
+    for tid, n in obs_by_topic.most_common(15):
+        t = topic_lookup.get(tid)
+        title = t["title"] if t else tid
+        status = t.get("status", "?") if t else "?"
+        lines.append(f"  {n:>4}  [{status:<8}] {title[:60]}")
+    lines.append("")
+
+    lines.append("## Observations")
+    minor = sum(1 for o in observations if o.get("minor"))
+    lines.append(f"Total: {len(observations)}")
+    lines.append(f"  primary: {len(observations) - minor}")
+    lines.append(f"  minor:   {minor}")
+    lines.append("")
+
+    sb: Counter = Counter()
+    for o in observations:
+        sb[_strength_bucket(o.get("strength") or 0)] += 1
+    lines.append("## Strength distribution")
+    for b in ("<0.3", "0.3-0.5", "0.5-0.7", "0.7-0.8", "0.8-0.9", "0.9+"):
+        lines.append(f"  {b:<10} {sb.get(b, 0)}")
+    lines.append("")
+
+    lines.append("## Connection types")
+    ct = Counter(o.get("connection_type", "?") for o in observations)
+    for k, n in ct.most_common():
+        lines.append(f"  {k:<14} {n}")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def write_audit_report(
+    store: Store,
+    output_path: Path | None = None,
+    label: str | None = None,
+) -> Path:
+    """Run the audit and write it to disk. Returns the output path.
+
+    If output_path is None, writes to tuning_runs/audit-<label or
+    timestamp>.md.
+    """
+    report = build_audit_report(store)
+    if output_path is None:
+        DEFAULT_OUTPUT_DIR.mkdir(exist_ok=True)
+        if label:
+            output_path = DEFAULT_OUTPUT_DIR / f"audit-{label}.md"
+        else:
+            ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
+            output_path = DEFAULT_OUTPUT_DIR / f"audit-{ts}.md"
+    else:
+        output_path = Path(output_path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(report)
+    return output_path

topicsleuth/backends/__init__.py

@@ -0,0 +1,5 @@
+"""Default implementations of the interfaces, one module per service, each
+importing its SDK lazily so the core package depends on none of them.
+
+    pip install "topicsleuth[mongo,voyage,anthropic]"
+"""

topicsleuth/backends/anthropic.py

@@ -0,0 +1,28 @@
+"""Claude as the judge. ``pip install "topicsleuth[anthropic]"``."""
+from __future__ import annotations
+
+from topicsleuth.interfaces import Completion
+
+
+class AnthropicJudge:
+    def __init__(self, model: str = "claude-opus-4-8", client=None, api_key: str | None = None):
+        try:
+            import anthropic
+        except ImportError as e:  # pragma: no cover
+            raise ImportError('AnthropicJudge needs the anthropic package: pip install "topicsleuth[anthropic]"') from e
+        self.model = model
+        self._client = client or anthropic.Anthropic(api_key=api_key)
+
+    def complete(self, system: str, user: str, max_tokens: int = 2048) -> Completion:
+        response = self._client.messages.create(
+            model=self.model,
+            max_tokens=max_tokens,
+            system=system,
+            messages=[{"role": "user", "content": user}],
+        )
+        usage = getattr(response, "usage", None)
+        return Completion(
+            text=response.content[0].text,
+            input_tokens=getattr(usage, "input_tokens", None),
+            output_tokens=getattr(usage, "output_tokens", None),
+        )

topicsleuth/backends/fake.py

@@ -0,0 +1,126 @@
+"""Deterministic backends for tests, yours and ours.
+
+``HashEmbedder`` maps text to a stable unit vector, so similar strings do
+not embed similarly but the same string always does and nothing needs a
+network. ``ScriptedJudge`` returns canned completions in order, or whatever a
+callable decides from the prompt; ``ScriptedDecisionModel`` answers the pair
+questions from a table of link probabilities. Together with
+``topicsleuth.backends.memory.MemoryStore`` they run the whole decision and
+record path offline.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Callable, Iterable
+
+from topicsleuth.interfaces import Completion, PairScore
+
+
+class HashEmbedder:
+    """Stable pseudo-embeddings. Pass ``vectors`` to pin specific texts to
+    specific vectors when a test needs two items to be neighbours."""
+
+    def __init__(self, dimensions: int = 8, vectors: dict[str, list[float]] | None = None):
+        self.model = "hash"
+        self.dimensions = dimensions
+        self.vectors = dict(vectors or {})
+        self.calls: list[list[str]] = []
+
+    def _one(self, text: str) -> list[float]:
+        if text in self.vectors:
+            return list(self.vectors[text])
+        raw: list[float] = []
+        block = 0
+        while len(raw) < self.dimensions:  # a digest is 32 bytes; chain as many as needed
+            digest = hashlib.sha256(f"{block}:{text}".encode()).digest()
+            raw.extend((b / 255.0) * 2 - 1 for b in digest)
+            block += 1
+        raw = raw[: self.dimensions]
+        norm = sum(x * x for x in raw) ** 0.5 or 1.0
+        return [x / norm for x in raw]
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        self.calls.append(list(texts))
+        return [self._one(t) for t in texts]
+
+    def embed_queries(self, texts: list[str]) -> list[list[float]]:
+        return self.embed_documents(texts)
+
+
+class ScriptedJudge:
+    """Returns completions from a script: a list consumed in order, or a
+    callable ``(system, user) -> str | dict``. Dicts are JSON-encoded. Every
+    call is recorded in ``calls`` as ``(system, user)``."""
+
+    def __init__(self, script: Iterable[str | dict] | Callable[[str, str], str | dict] = (), model: str = "scripted"):
+        self.model = model
+        self._fn = script if callable(script) else None
+        self._queue = list(script) if not callable(script) else []
+        self.calls: list[tuple[str, str]] = []
+
+    def queue(self, *items: str | dict) -> "ScriptedJudge":
+        """Append completions to the script. Returns self for chaining."""
+        self._queue.extend(items)
+        return self
+
+    def complete(self, system: str, user: str, max_tokens: int = 2048) -> Completion:
+        self.calls.append((system, user))
+        if self._fn is not None:
+            out = self._fn(system, user)
+        elif self._queue:
+            out = self._queue.pop(0)
+        else:
+            raise AssertionError(f"ScriptedJudge ran out of script at call {len(self.calls)}")
+        text = json.dumps(out) if isinstance(out, (dict, list)) else str(out)
+        return Completion(text=text, input_tokens=len(system) + len(user), output_tokens=len(text))
+
+
+class ScriptedDecisionModel:
+    """Scores item-topic pairs from ``link_probabilities``, a map of
+    candidate topic title to p(belongs). Unknown titles score 0.1. Each
+    call is recorded as ``{item, topic, representative_titles}``."""
+
+    def __init__(self, link_probabilities: dict[str, float] | None = None, model: str = "scripted-decision"):
+        self.model = model
+        self.link_probabilities = dict(link_probabilities or {})
+        self.calls: list[dict] = []
+
+    def score_pair(self, item: dict, topic: dict, representative_titles: list[str]) -> PairScore:
+        self.calls.append({"item": item, "topic": topic, "representative_titles": list(representative_titles)})
+        p = self.link_probabilities.get(topic.get("title", ""), 0.1)
+        label = "same_question" if p >= 0.7 else "adjacent_question"
+        return PairScore(
+            label=label, link_probability=p, confidence=0.9,
+            probabilities={label: p, "surface_overlap": 1 - p},
+            input_tokens=100, model=self.model,
+        )
+
+
+# Process-wide instances the CLI picks up when TOPICSLEUTH_STORE=memory,
+# TOPICSLEUTH_JUDGE=scripted or TOPICSLEUTH_EMBEDDER=hash, so a test can seed
+# a store and queue completions, then run a command in-process.
+_shared: dict[str, object] = {}
+
+
+def shared(kind: str):
+    """``shared("store")``, ``shared("judge")``, ``shared("embedder")``,
+    ``shared("decision_model")``: the process-wide fake of that kind,
+    created on first use."""
+    if kind not in _shared:
+        if kind == "store":
+            from topicsleuth.backends.memory import MemoryStore
+            _shared[kind] = MemoryStore()
+        elif kind == "judge":
+            _shared[kind] = ScriptedJudge()
+        elif kind == "embedder":
+            _shared[kind] = HashEmbedder()
+        elif kind == "decision_model":
+            _shared[kind] = ScriptedDecisionModel()
+        else:
+            raise KeyError(kind)
+    return _shared[kind]
+
+
+def reset_shared() -> None:
+    _shared.clear()

topicsleuth/backends/jev.py

@@ -0,0 +1,95 @@
+"""TypeSafe Jev as the decision model.
+
+Jev scores a fixed option list in one forward pass and returns a probability
+distribution plus a confidence, with no text. There is no system prompt; the
+rubric from ``topicsleuth.rubric`` is carried in the option descriptions,
+which Jev reads field by field. No SDK: plain HTTPS.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import time
+import urllib.error
+import urllib.request
+
+from topicsleuth.interfaces import PairScore
+from topicsleuth.rubric import (
+    LINK_CRITERION,
+    RELATIONSHIP_CRITERIA,
+    RELATIONSHIP_INSTRUCTIONS,
+    pair_view,
+)
+
+logger = logging.getLogger(__name__)
+
+API_URL = "https://api.typesafe.ai/v1/systemone"
+DEFAULT_MODEL = "jev-latest"
+
+# The library's rubric in Jev's question format: a choice over the
+# relationship labels and a yes/no ("noul") on whether the item belongs.
+PAIR_QUESTIONS = {
+    "relationship": {
+        "type": "choice",
+        "instructions": RELATIONSHIP_INSTRUCTIONS,
+        "criteria": RELATIONSHIP_CRITERIA,
+    },
+    "link": {
+        "type": "noul",
+        "instructions": LINK_CRITERION,
+    },
+}
+
+
+class JevError(RuntimeError):
+    pass
+
+
+class JevDecisionModel:
+    """Implements the DecisionModel protocol over the TypeSafe API."""
+
+    def __init__(self, api_key: str, model: str = DEFAULT_MODEL, retries: int = 3, timeout: float = 30.0):
+        if not api_key:
+            raise ValueError("JevDecisionModel needs an API key")
+        self.api_key = api_key
+        self.model = model
+        self.retries = retries
+        self.timeout = timeout
+
+    def score_pair(self, item: dict, topic: dict, representative_titles: list[str]) -> PairScore:
+        resp = self.decide(pair_view(item, topic, representative_titles), PAIR_QUESTIONS)
+        answers = resp.get("answers") or {}
+        rel = answers.get("relationship") or {}
+        link = answers.get("link") or {}
+        return PairScore(
+            label=rel.get("choice"),
+            link_probability=link.get("noul"),
+            confidence=rel.get("confidence"),
+            probabilities=rel.get("probabilities"),
+            input_tokens=(resp.get("usage") or {}).get("input_tokens"),
+            model=resp.get("model", self.model),
+        )
+
+    def decide(self, state: dict, questions: dict) -> dict:
+        """One raw call: Jev's own request and reply shapes."""
+        body = json.dumps({"model": self.model, "state": state, "questions": questions}).encode()
+        req = urllib.request.Request(
+            API_URL, data=body, method="POST",
+            headers={"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"},
+        )
+        for attempt in range(self.retries):
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as r:
+                    return json.loads(r.read())
+            except urllib.error.HTTPError as e:
+                detail = e.read().decode(errors="replace")[:300]
+                if e.code in (429, 500, 502, 503, 504) and attempt < self.retries - 1:
+                    time.sleep(2 ** attempt)
+                    continue
+                raise JevError(f"HTTP {e.code}: {detail}") from None
+            except urllib.error.URLError as e:
+                if attempt < self.retries - 1:
+                    time.sleep(2 ** attempt)
+                    continue
+                raise JevError(str(e)) from None
+        raise JevError("unreachable")