abstractnote 0.1.0__py3-none-any.whl

abstractnote/process/schemas.py

@@ -0,0 +1,377 @@
+"""Structured-output schemas (docs/cognition.md, "Structured output schemas").
+
+``SCHEMAS`` maps each name to the JSON Schema dict attached as ``resp_schema``
+to the flows' ``llm_call`` nodes; the pydantic models below mirror them for
+validation in Python (``note_apply_revision`` validates the Extraction and
+Linking it receives). Identifiers never appear as free text in the Linking
+schema: entities, existing claims and open tasks are referred to by their
+INDEX in the context packet, and decisions are enumerations (ADR 0004).
+
+The flow generator (``abstractnote.flows.build``) checks that its own copy of
+these schemas has the same structure and refuses to build otherwise.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+ENTITY_KINDS = ["person", "org", "project", "concept", "place", "event", "artifact", "software", "thing"]
+NOTE_KINDS = ["idea", "task", "decision", "fact", "event", "question", "reflection", "reference", "log",
+              "conversation", "personal"]
+URGENCIES = ["bulk", "normal", "high", "urgent"]
+CERTAINTIES = ["stated", "probable", "possible", "reported", "disputed"]
+STANCES = ["assert", "revise", "retract"]
+MENTION_DECISIONS = ["reuse", "create", "merge", "unsure"]
+CLAIM_RELATIONS = ["new", "restates", "refines", "supersedes", "contradicts"]
+REFLECTION_KINDS = ["warning", "reminder", "connection", "contradiction", "evolution"]
+
+
+def _s(description: str = "") -> Dict[str, Any]:
+    return {"type": "string", "description": description} if description else {"type": "string"}
+
+
+def _ns(description: str = "") -> Dict[str, Any]:
+    out: Dict[str, Any] = {"type": ["string", "null"]}
+    if description:
+        out["description"] = description
+    return out
+
+
+def _e(values: List[str]) -> Dict[str, Any]:
+    return {"type": "string", "enum": list(values)}
+
+
+def _a(items: Dict[str, Any], description: str = "") -> Dict[str, Any]:
+    out: Dict[str, Any] = {"type": "array", "items": items}
+    if description:
+        out["description"] = description
+    return out
+
+
+def _o(properties: Dict[str, Any]) -> Dict[str, Any]:
+    return {"type": "object", "properties": properties, "required": list(properties)}
+
+
+QUICK_FILING: Dict[str, Any] = _o({
+    "title": {"type": "string", "maxLength": 80, "description": "short title, at most 80 characters"},
+    "kind": _e(NOTE_KINDS),
+    "topics": {**_a(_s(), "prefer existing topics, at most 4"), "maxItems": 4},
+    "urgency": _e(URGENCIES),
+    "language": _s("BCP-47 language tag"),
+    "one_line": _s("the note in one line"),
+})
+
+EXTRACTION: Dict[str, Any] = _o({
+    "summary": _s("1-3 sentences"),
+    "mentions": _a(_o({
+        "ref": _s("m1, m2, ..."),
+        "name": _s("as written in the note"),
+        "kind": _e(ENTITY_KINDS),
+        "aliases": _a(_s(), "other names as written in the note"),
+        "role": _s("what the note says about it, one phrase"),
+    })),
+    "claims": _a(_o({
+        "subject": _s("a mention ref"),
+        "predicate": _s("a CURIE from the vocabulary, or ex:snake_case"),
+        "object": _s("a mention ref, or a literal value"),
+        "object_is_literal": {"type": "boolean"},
+        "certainty": _e(CERTAINTIES),
+        "stance": _e(STANCES),
+        "evidence": _s("verbatim excerpt of the note"),
+        "valid_from": _ns("ISO date or null"),
+        "valid_until": _ns("ISO date or null"),
+    })),
+    "tasks": _a(_o({
+        "text": _s(),
+        "status": _e(["open", "done"]),
+        "due": _ns("ISO date or null"),
+    })),
+    "time_refs": _a(_o({"text": _s(), "iso": _ns("ISO or null")})),
+    "topics": _a(_s()),
+})
+
+LINKING: Dict[str, Any] = _o({
+    "mentions": _a(_o({
+        "ref": _s("the mention ref from the extraction"),
+        "decision": _e(MENTION_DECISIONS),
+        "candidate": {"type": ["integer", "null"], "description": "index in this mention's candidate list"},
+        "merge_candidates": _a({"type": "integer"}, "two candidate indexes that are the same entity"),
+        "name": _s("preferred name"),
+        "kind": _e(ENTITY_KINDS),
+        "aliases": _a(_s()),
+        "summary": {"type": ["string", "null"], "minLength": 1,
+                    "description": "one sentence about the entity: required for create and unsure, and for a "
+                                   "reused or merged entity this note tells something new about; null only for a "
+                                   "reused entity the note adds nothing to"},
+        "reason": _s("short"),
+    })),
+    "claims": _a(_o({
+        "index": {"type": "integer", "description": "index of the claim in the extraction"},
+        "relation_to_existing": _e(CLAIM_RELATIONS),
+        "existing": {"type": ["integer", "null"], "description": "index of the existing claim in the packet"},
+        "reason": _s("short"),
+    })),
+    "tasks": _a(_o({
+        "index": {"type": "integer", "description": "index of the task in the extraction"},
+        "closes_open_task": {"type": ["integer", "null"], "description": "index of the open task it closes"},
+    })),
+    "revision_summary": _s("one sentence"),
+})
+
+ANSWER: Dict[str, Any] = _o({
+    "answer": _s("the answer with inline citations [nt_...]"),
+    "evidence": _a(_o({"note_id": _s(), "title": _s(), "occurred_at": _s(), "excerpt": _s()})),
+    "evolution": _o({
+        "earliest": _s("the earliest stance"),
+        "latest": _s("the latest stance"),
+        "turning_points": _a(_o({"note_id": _s(), "occurred_at": _s(), "change": _s()})),
+    }),
+    "open_loops": _a(_s()),
+    "follow_ups": _a(_s()),
+})
+
+MINING_PLAN: Dict[str, Any] = _o({
+    "steps": _a(_o({
+        "question": _s("a sub-question to search for"),
+        "since": _ns("ISO or null"),
+        "until": _ns("ISO or null"),
+        "entities": _a(_s(), "names to trace"),
+    })),
+    "contradictions_to_test": _a(_s()),
+    "rounds": {"type": "integer", "minimum": 1, "maximum": 8},
+})
+
+SLICE_FINDINGS: Dict[str, Any] = _o({
+    "facts": _a(_o({"text": _s(), "citations": _a(_s(), "note ids")})),
+    "changes": _a(_s()),
+    "gaps": _a(_s()),
+})
+
+DOSSIER: Dict[str, Any] = _o({
+    "title": _s(),
+    "narrative": _s("with inline citations [nt_...]"),
+    "timeline": _a(_o({"at": _s(), "event": _s(), "note_ids": _a(_s())})),
+    "evolution": _s("how the view evolved"),
+    "contradictions": _a(_s()),
+    "forgotten": _a(_s(), "items worth revisiting"),
+    "suggestions": _a(_s()),
+    "confidence": _s("a confidence statement"),
+})
+
+REFLECTIONS: Dict[str, Any] = _o({
+    "items": _a(_o({
+        "candidate": {"type": "integer", "description": "index in the candidate list"},
+        "kind": _e(REFLECTION_KINDS),
+        "title": _s(),
+        "body": _s(),
+    })),
+})
+
+SCHEMAS: Dict[str, Dict[str, Any]] = {
+    "QuickFiling": QUICK_FILING,
+    "Extraction": EXTRACTION,
+    "Linking": LINKING,
+    "Answer": ANSWER,
+    "MiningPlan": MINING_PLAN,
+    "SliceFindings": SLICE_FINDINGS,
+    "Dossier": DOSSIER,
+    "Reflections": REFLECTIONS,
+}
+
+
+# ----------------------------------------------------------------------------- pydantic mirrors
+
+class _M(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+
+class QuickFiling(_M):
+    title: str
+    kind: Literal[tuple(NOTE_KINDS)]  # type: ignore[valid-type]
+    topics: List[str] = Field(default_factory=list, max_length=4)
+    urgency: Literal[tuple(URGENCIES)]  # type: ignore[valid-type]
+    language: str = ""
+    one_line: str = ""
+
+
+class Mention(_M):
+    ref: str
+    name: str
+    kind: Literal[tuple(ENTITY_KINDS)] = "thing"  # type: ignore[valid-type]
+    aliases: List[str] = Field(default_factory=list)
+    role: str = ""
+
+
+class ExtractedClaim(_M):
+    subject: str
+    predicate: str
+    object: str
+    object_is_literal: bool = False
+    certainty: Literal[tuple(CERTAINTIES)] = "stated"  # type: ignore[valid-type]
+    stance: Literal[tuple(STANCES)] = "assert"  # type: ignore[valid-type]
+    evidence: str = ""
+    valid_from: Optional[str] = None
+    valid_until: Optional[str] = None
+
+
+class ExtractedTask(_M):
+    text: str
+    status: Literal["open", "done"] = "open"
+    due: Optional[str] = None
+
+
+class TimeRef(_M):
+    text: str
+    iso: Optional[str] = None
+
+
+class Extraction(_M):
+    summary: str = ""
+    mentions: List[Mention] = Field(default_factory=list)
+    claims: List[ExtractedClaim] = Field(default_factory=list)
+    tasks: List[ExtractedTask] = Field(default_factory=list)
+    time_refs: List[TimeRef] = Field(default_factory=list)
+    topics: List[str] = Field(default_factory=list)
+
+
+class MentionLink(_M):
+    """A mention decision. ``summary`` is required (non-empty) when the entity is created
+    (``create``, ``unsure``); for reused entities the revision builder requires it when the
+    note changes what is known about them."""
+
+    ref: str
+    decision: Literal[tuple(MENTION_DECISIONS)]  # type: ignore[valid-type]
+    candidate: Optional[int] = None
+    merge_candidates: List[int] = Field(default_factory=list)
+    name: str = ""
+    kind: Optional[Literal[tuple(ENTITY_KINDS)]] = None  # type: ignore[valid-type]
+    aliases: List[str] = Field(default_factory=list)
+    summary: Optional[str] = None
+    reason: str = ""
+
+
+    @model_validator(mode="after")
+    def _summary_for_new_entities(self) -> "MentionLink":
+        if self.decision in ("create", "unsure") and not (self.summary or "").strip():
+            raise ValueError(f"the {self.decision} decision for {self.ref} needs a one-sentence summary")
+        return self
+
+
+class ClaimLink(_M):
+    index: int
+    relation_to_existing: Literal[tuple(CLAIM_RELATIONS)] = "new"  # type: ignore[valid-type]
+    existing: Optional[int] = None
+    reason: str = ""
+
+
+class TaskLink(_M):
+    index: int
+    closes_open_task: Optional[int] = None
+
+
+class Linking(_M):
+    mentions: List[MentionLink] = Field(default_factory=list)
+    claims: List[ClaimLink] = Field(default_factory=list)
+    tasks: List[TaskLink] = Field(default_factory=list)
+    revision_summary: str = ""
+
+
+class Evidence(_M):
+    note_id: str
+    title: str = ""
+    occurred_at: str = ""
+    excerpt: str = ""
+
+
+class TurningPoint(_M):
+    note_id: str
+    occurred_at: str = ""
+    change: str = ""
+
+
+class Evolution(_M):
+    earliest: str = ""
+    latest: str = ""
+    turning_points: List[TurningPoint] = Field(default_factory=list)
+
+
+class Answer(_M):
+    answer: str
+    evidence: List[Evidence] = Field(default_factory=list)
+    evolution: Evolution = Field(default_factory=Evolution)
+    open_loops: List[str] = Field(default_factory=list)
+    follow_ups: List[str] = Field(default_factory=list)
+
+
+class MiningStep(_M):
+    question: str
+    since: Optional[str] = None
+    until: Optional[str] = None
+    entities: List[str] = Field(default_factory=list)
+
+
+class MiningPlan(_M):
+    steps: List[MiningStep] = Field(default_factory=list)
+    contradictions_to_test: List[str] = Field(default_factory=list)
+    rounds: int = Field(1, ge=1, le=8)
+
+
+class Fact(_M):
+    text: str
+    citations: List[str] = Field(default_factory=list)
+
+
+class SliceFindings(_M):
+    facts: List[Fact] = Field(default_factory=list)
+    changes: List[str] = Field(default_factory=list)
+    gaps: List[str] = Field(default_factory=list)
+
+
+class TimelineEntry(_M):
+    at: str
+    event: str
+    note_ids: List[str] = Field(default_factory=list)
+
+
+class Dossier(_M):
+    title: str
+    narrative: str
+    timeline: List[TimelineEntry] = Field(default_factory=list)
+    evolution: str = ""
+    contradictions: List[str] = Field(default_factory=list)
+    forgotten: List[str] = Field(default_factory=list)
+    suggestions: List[str] = Field(default_factory=list)
+    confidence: str = ""
+
+
+class Reflection(_M):
+    candidate: int
+    kind: Literal[tuple(REFLECTION_KINDS)]  # type: ignore[valid-type]
+    title: str
+    body: str
+
+
+class Reflections(_M):
+    items: List[Reflection] = Field(default_factory=list)
+
+
+MODELS = {
+    "QuickFiling": QuickFiling, "Extraction": Extraction, "Linking": Linking, "Answer": Answer,
+    "MiningPlan": MiningPlan, "SliceFindings": SliceFindings, "Dossier": Dossier, "Reflections": Reflections,
+}
+
+
+def validate(name: str, data: Any) -> BaseModel:
+    """Validate model output against a schema mirror; raises ValueError with a plain sentence."""
+    from pydantic import ValidationError
+
+    if not isinstance(data, dict):
+        raise ValueError(f"The {name} must be a JSON object.")
+    try:
+        return MODELS[name].model_validate(data)
+    except ValidationError as exc:
+        first = exc.errors()[0]
+        loc = ".".join(str(x) for x in first.get("loc", ())) or name
+        raise ValueError(f"The {name} does not match its schema ({loc}: {first.get('msg')}).") from None

abstractnote/process/vocabulary.py

@@ -0,0 +1,68 @@
+"""The predicate vocabulary from the AbstractSemantics registry, with one-line glosses."""
+
+from __future__ import annotations
+
+import re
+from functools import lru_cache
+from typing import Dict, List, Optional, Tuple
+
+EX_PREDICATE = re.compile(r"^ex:[a-z][a-z0-9_]*$")
+
+
+@lru_cache(maxsize=1)
+def _registry():
+    from abstractsemantics import load_semantics_registry
+
+    return load_semantics_registry()
+
+
+def vocabulary() -> List[Dict[str, str]]:
+    """``[{predicate, gloss}]`` for every registry predicate (the extraction prompt's list)."""
+    out = []
+    for p in _registry().predicates:
+        gloss = (p.description or p.label or "").strip()
+        out.append({"predicate": p.id, "gloss": gloss})
+    return out
+
+
+def predicate_words(predicate: str) -> str:
+    """Plain words for a predicate, for text a person reads: the registry label (``schema:worksFor``
+    -> "works for"), else the spelled-out local name (``ex:led_by`` -> "led by"). Never a CURIE."""
+    raw = str(predicate or "").strip()
+    for p in _registry().predicates:
+        if p.id == raw and (p.label or "").strip():
+            return p.label.strip()
+    local = raw.split(":", 1)[1] if ":" in raw else raw
+    words = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", local).replace("_", " ").strip().lower()
+    return " ".join(words.split()) or "related to"
+
+
+def predicate_ids() -> frozenset:
+    return frozenset(_registry().predicate_ids())
+
+
+def _snake(text: str) -> str:
+    """``worksFor`` / ``works for`` -> ``works_for`` (spelling only)."""
+    s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", str(text or "").strip())
+    s = re.sub(r"[^A-Za-z0-9]+", "_", s).strip("_").lower()
+    if not s or not s[0].isalpha():
+        s = "p_" + s
+    return s
+
+
+def normalize_predicate(predicate: str) -> Tuple[str, Optional[str]]:
+    """Map an extracted predicate to a registry CURIE (exact or a registry alias); anything
+    else becomes ``ex:snake_case``. Returns ``(predicate, warning or None)``; CURIE case is kept."""
+    from abstractsemantics import normalize_kg_predicate
+
+    raw = str(predicate or "").strip()
+    if not raw:
+        return "ex:related_to", "A statement had no predicate; it was kept as ex:related_to."
+    canonical = normalize_kg_predicate(raw, _registry())
+    if canonical:
+        return canonical, None
+    if EX_PREDICATE.match(raw):
+        return raw, f"The predicate {raw} is outside the vocabulary."
+    local = raw.split(":", 1)[1] if ":" in raw else raw
+    mapped = "ex:" + _snake(local)
+    return mapped, f"The predicate {raw} is outside the vocabulary; kept as {mapped}."

abstractnote/recall/__init__.py

@@ -0,0 +1 @@
+"""Recall: quick search, expansion, then-versus-now, reflect candidates, insights and the read shapes."""

abstractnote/recall/changes.py

@@ -0,0 +1,191 @@
+"""Since you last looked (``Notebook.changes``), the landing overview and the cue cards.
+
+``changes`` is a system-time delta: everything the notebook recorded after a
+cursor. The cursor encodes the high-water ``recorded_at`` second and the items
+already returned at that second, so nothing is lost or repeated across calls.
+"""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import Any, Dict, List, Optional, Tuple
+
+from .. import ids
+from ..store.notes import clamp_limit, decode_cursor, encode_cursor
+from . import shapes
+
+
+def _since(since: Optional[str]) -> Tuple[str, set]:
+    if not since:
+        return "0000-01-01T00:00:00Z", set()
+    s = str(since).strip()
+    if s[:1].isdigit():
+        return ids.normalize_time(s, field="since"), set()
+    cur = decode_cursor(s)
+    if not cur or len(cur) != 3 or cur[0] != "chg":
+        raise ValueError("The cursor is not one this notebook issued.")
+    return str(cur[1]), set(cur[2])
+
+
+def changes(nb: Any, since: Optional[str] = None, limit: int = 50, as_received: Optional[str] = None
+            ) -> Dict[str, Any]:
+    """``{cursor, notes, entities, superseded_claims, closed_loops, failed_notes, insights}``.
+
+    ``superseded_claims`` lists every claim that stopped being believed: ``kind`` ``superseded``
+    (a later note replaced it; ``superseded_by`` names the new claim) or ``retracted`` (withdrawn
+    by a note or by the person; ``superseded_by`` is null), with ``text`` in plain words.
+
+    ``since`` is a cursor from an earlier call or an ISO time. At most ``limit``
+    items in total are returned, earliest first; the returned cursor continues
+    exactly after them.
+
+    ``as_received`` keeps only what derives from the archive as it stood on that date: notes and insights
+    received by then, entities created and claims and tasks closed by the revisions those notes caused (or
+    user actions recorded by then), failed notes received by then.
+    """
+    from ..store.archive import archive_for
+
+    limit = clamp_limit(limit, 50, 500)
+    ts, seen = _since(since)
+    q = nb.store.q
+    arch = archive_for(nb.store, as_received)
+
+    def only(col: str, kind: str) -> Tuple[str, List[Any]]:
+        if arch is None:
+            return "", []
+        frag, fp = arch.notes_in(col) if kind == "note" else arch.revisions_in(col)
+        return f" AND {frag}", fp
+
+    f_note, p_note = only("note_id", "note")
+    f_qnote, p_qnote = only("q.note_id", "note")
+    f_created, p_created = only("created_revision_id", "revision")
+    f_closed, p_closed = only("c.closed_revision_id", "revision")
+    f_task, p_task = only("closed_revision_id", "revision")
+    found: List[Tuple[str, str, str, Dict[str, Any]]] = []  # (at, key, bucket, item)
+
+    def add(bucket: str, kind: str, key: str, at: Optional[str], item: Dict[str, Any]) -> None:
+        if not at or at < ts or (at == ts and f"{kind}:{key}" in seen):
+            return
+        found.append((at, f"{kind}:{key}", bucket, item))
+
+    for r in q("SELECT * FROM notes WHERE insight_kind IS NULL AND received_at>=? AND archived_at IS NULL" + f_note +
+               " ORDER BY received_at, note_id LIMIT ?", (ts, *p_note, limit + len(seen))):
+        add("notes", "note", r["note_id"], r["received_at"], shapes.note_ref(r))
+    for r in q("SELECT * FROM entities WHERE recorded_at>=?" + f_created + " ORDER BY recorded_at, entity_id LIMIT ?",
+               (ts, *p_created, limit + len(seen))):
+        add("entities", "entity", r["entity_id"], r["recorded_at"],
+            {"entity_id": r["entity_id"], "name": r["name"], "kind": r["kind"], "first_seen": r["first_seen"]})
+    for r in q("SELECT c.*, v.recorded_at AS closed_recorded_at, v.note_id AS closing_note_id FROM claims c"
+               " JOIN revisions v ON v.revision_id=c.closed_revision_id WHERE c.status IN ('superseded','retracted')"
+               " AND v.recorded_at>=?" + f_closed + " ORDER BY v.recorded_at, c.claim_id LIMIT ?",
+               (ts, *p_closed, limit + len(seen))):
+        add("superseded_claims", "claim", r["claim_id"], r["closed_recorded_at"],
+            {"claim_id": r["claim_id"], "kind": "superseded" if r["status"] == "superseded" else "retracted",
+             "text": claim_text(nb, r), "superseded_by": r["superseded_by"] if r["status"] == "superseded" else None,
+             "note_id": r["closing_note_id"] or r["note_id"]})
+    for r in q("SELECT * FROM tasks WHERE status='done' AND recorded_closed_at>=?" + f_task +
+               " ORDER BY recorded_closed_at, task_id LIMIT ?", (ts, *p_task, limit + len(seen))):
+        add("closed_loops", "task", r["task_id"], r["recorded_closed_at"],
+            {"task_id": r["task_id"], "text": r["text"], "closed_at": r["closed_at"]})
+    for r in q("SELECT q.note_id, q.last_error, q.failed_at, n.title FROM queue q JOIN notes n ON n.note_id=q.note_id"
+               " WHERE q.state='failed' AND q.failed_at>=?" + f_qnote + " ORDER BY q.failed_at, q.note_id LIMIT ?",
+               (ts, *p_qnote, limit + len(seen))):
+        add("failed_notes", "failed", r["note_id"], r["failed_at"],
+            {"note_id": r["note_id"], "title": r["title"] or "", "error": r["last_error"] or ""})
+    for r in q("SELECT * FROM notes WHERE insight_kind IS NOT NULL AND received_at>=? AND archived_at IS NULL" +
+               f_note + " ORDER BY received_at, note_id LIMIT ?", (ts, *p_note, limit + len(seen))):
+        add("insights", "insight", r["note_id"], r["received_at"], shapes.insight(r))
+
+    found.sort(key=lambda x: (x[0], x[1]))
+    taken = found[:limit]
+    out: Dict[str, Any] = {"notes": [], "entities": [], "superseded_claims": [], "closed_loops": [],
+                           "failed_notes": [], "insights": []}
+    for _at, _key, bucket, item in taken:
+        out[bucket].append(item)
+    if taken:
+        high = taken[-1][0]
+        keys = {k for at, k, _, _ in taken if at == high}
+        if high == ts:
+            keys |= seen
+    else:
+        high, keys = ts, set(seen)
+    return {"cursor": encode_cursor("chg", high, sorted(keys)), **out}  # pinned keys only, with or without as_received
+
+
+def claim_text(nb: Any, r: Any) -> str:
+    """The claim in words a person reads: entity names and the predicate's plain words, never ids
+    ("Dana participant LUT"; an entity without a name reads "someone")."""
+    from ..process.vocabulary import predicate_words
+
+    subject = nb.store._name(r["subject_id"]) or "someone"
+    obj = (nb.store._name(r["object_id"]) or "something") if r["object_id"] else (r["object_literal"] or "something")
+    return f"{subject} {predicate_words(r['predicate'])} {obj}"
+
+
+def overview(nb: Any, days: int = 30, *, now: Optional[str] = None) -> Dict[str, Any]:
+    """``{recent, entities, insights, resurfacing, activity}`` for the landing screen."""
+    days = max(1, min(366, int(days or 30)))
+    now = ids.normalize_time(now) if now else ids.utc_now()
+    start = (ids.parse_time(now) - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    q = nb.store.q
+    recent = [shapes.note_ref(r) for r in q(
+        "SELECT * FROM notes WHERE insight_kind IS NULL AND archived_at IS NULL ORDER BY occurred_at DESC,"
+        " note_id DESC LIMIT 20")]
+    entities = [{"entity_id": r["entity_id"], "name": r["name"], "kind": r["kind"], "note_count": r["note_count"],
+                 "last_seen": r["last_seen"]} for r in q(
+        "SELECT * FROM entities WHERE merged_into IS NULL AND last_seen>=? ORDER BY note_count DESC, last_seen DESC,"
+        " entity_id LIMIT 20", (start,))]
+    insights = [shapes.insight(r) for r in q(
+        "SELECT * FROM notes WHERE insight_kind IS NOT NULL AND acked_at IS NULL AND archived_at IS NULL"
+        " ORDER BY received_at DESC, note_id DESC LIMIT 10")]
+    from .reflect import anniversary_rows
+
+    resurfacing = [shapes.note_ref(r) for r, _rule in anniversary_rows(nb, limit=10, now=now)]
+    counts: Dict[str, Dict[str, int]] = {}
+    for r in q("SELECT substr(occurred_at,1,10) AS d, COUNT(*) AS n FROM notes WHERE occurred_at>=? AND"
+               " insight_kind IS NULL GROUP BY d", (start,)):
+        counts.setdefault(r["d"], {"notes": 0, "revisions": 0})["notes"] = r["n"]
+    for r in q("SELECT substr(at,1,10) AS d, COUNT(*) AS n FROM revisions WHERE at>=? AND applied_index=1"
+               " GROUP BY d", (start,)):
+        counts.setdefault(r["d"], {"notes": 0, "revisions": 0})["revisions"] = r["n"]
+    activity = []
+    day = ids.parse_time(start)
+    end = ids.parse_time(now)
+    while day.date() <= end.date():
+        key = day.strftime("%Y-%m-%d")
+        c = counts.get(key, {"notes": 0, "revisions": 0})
+        activity.append({"day": key, "notes": c["notes"], "revisions": c["revisions"]})
+        day += timedelta(days=1)
+    return {"recent": recent, "entities": entities, "insights": insights, "resurfacing": resurfacing,
+            "activity": activity}
+
+
+def cues(nb: Any, limit: int = 3) -> Dict[str, Any]:
+    """Cue cards: unacknowledged reflect insights (not dossiers), each with the subject a reply links to.
+
+    ``{items: [{cue_id, text, kind, subject: {kind, id, title}, evidence_note_ids, entity_ids}]}``; ``cue_id`` is
+    the insight note id; ``evidence_note_ids`` and ``entity_ids`` are the insight's own (the notes it rests on and
+    the entities it is about), so a client shows why it surfaced without a second read.
+    """
+    limit = clamp_limit(limit, 3, 50)
+    items = []
+    for r in nb.store.q("SELECT * FROM notes WHERE insight_kind IS NOT NULL AND insight_kind!='dossier' AND"
+                        " acked_at IS NULL AND archived_at IS NULL ORDER BY received_at DESC, note_id DESC LIMIT ?",
+                        (limit,)):
+        ins = shapes.insight(r)
+        subject = {"kind": "cue", "id": r["note_id"], "title": ins["title"]}
+        for eid in ins["entity_ids"]:
+            e = nb.store.entity_row(nb.store.canonical(eid))
+            if e is not None:
+                subject = {"kind": "entity", "id": e["entity_id"], "title": e["name"]}
+                break
+        else:
+            for nid in ins["evidence_note_ids"]:
+                n = nb.store.note_row(nid)
+                if n is not None:
+                    subject = {"kind": "note", "id": nid, "title": n["title"] or shapes.excerpt(n["text"], 80)}
+                    break
+        items.append({"cue_id": r["note_id"], "text": ins["body"] or ins["title"], "kind": r["insight_kind"],
+                      "subject": subject, "evidence_note_ids": ins["evidence_note_ids"],
+                      "entity_ids": ins["entity_ids"]})
+    return {"items": items}