abstractnote 0.1.0__py3-none-any.whl

abstractnote/recall/expand.py

@@ -0,0 +1,379 @@
+"""Graph expansion for standard recall: entity cards as of a moment, claim and relation
+timelines, neighbours, then-versus-now pairs, and the evidence chain of a claim."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Optional, Sequence
+
+from .. import ids
+from ..store.notes import _uj, clamp_limit
+from . import shapes
+from .then_now import relation_then_now, then_now
+
+MAX_EXPAND_ENTITIES = 10
+MAX_RELATION_PAIRS = 10
+CARD_NOTES = 10
+
+
+def _members(nb: Any, entity_id: str) -> List[str]:
+    return nb.store.merged_members(nb.store.canonical(entity_id))
+
+
+def _require_entity(nb: Any, entity_id: str) -> Any:
+    row = nb.store.entity_row(entity_id)
+    if row is None:
+        raise ValueError("That entity is not in this notebook.")
+    return row
+
+
+def entity_card(nb: Any, entity_id: str, at: Optional[str] = None, as_received: Optional[str] = None
+                ) -> Dict[str, Any]:
+    """The entity as known at ``at`` (default now): name, aliases, summary and its history,
+    beliefs valid at ``at``, relations, notes, open tasks and a monthly activity sparkline.
+
+    ``as_received`` builds the card from what the archive held on that date only (notes received by then and
+    the revisions they caused, :mod:`abstractnote.store.archive`); it composes with ``at``."""
+    from ..store.archive import archive_for
+
+    _require_entity(nb, entity_id)
+    at_n = ids.optional_time(at, field="at")
+    s = nb.store
+    arch = archive_for(s, as_received)
+    if arch is not None:
+        return _archive_card(nb, arch, entity_id, at_n)
+    eid = s.canonical_at(entity_id, at_n)
+    row = s.entity_row(eid)
+    if at_n and (row["first_seen"] or row["created_at"]) > at_n:
+        raise ValueError("That entity was not known yet at that time.")
+    members = s.merged_members(eid) if at_n is None else [
+        m for m in s.merged_members(s.canonical(eid)) if s.canonical_at(m, at_n) == eid]
+    if eid not in members:
+        members.insert(0, eid)
+    ref = at_n or "9999-12-31T23:59:59Z"
+    q = ",".join("?" for _ in members)
+    name_row = s.q1("SELECT name FROM entity_names WHERE entity_id=? AND at<=? ORDER BY at DESC, revision_id DESC LIMIT 1",
+                    (eid, ref))
+    name = name_row[0] if (name_row and at_n) else row["name"]
+    aliases: List[str] = []
+    for m in members:
+        for a in s.aliases(m, at_n):
+            if ids.fold(a) != ids.fold(name) and ids.fold(a) not in {ids.fold(x) for x in aliases}:
+                aliases.append(a)
+        if m != eid:
+            mrow = s.entity_row(m)
+            if ids.fold(mrow["name"]) != ids.fold(name) and ids.fold(mrow["name"]) not in {ids.fold(x) for x in aliases}:
+                aliases.append(mrow["name"])
+    history = [{"at": r["at"], "summary": r["summary"], "note_id": r["note_id"], "revision_id": r["revision_id"]}
+               for r in s.q(f"SELECT * FROM entity_summaries WHERE entity_id IN ({q}) AND at<=? ORDER BY at, revision_id",
+                            [*members, ref])]
+    summary = history[-1]["summary"] if history else (row["summary"] if at_n is None else None)
+    notes = s.q(f"SELECT DISTINCT n.* FROM entity_notes en JOIN notes n ON n.note_id=en.note_id WHERE en.entity_id IN"
+                f" ({q}) AND en.at<=? AND n.archived_at IS NULL ORDER BY n.occurred_at DESC LIMIT ?",
+                [*members, ref, CARD_NOTES])
+    stats = s.q1(f"SELECT MIN(at), MAX(at), COUNT(DISTINCT note_id) FROM entity_notes WHERE entity_id IN ({q}) AND at<=?",
+                 [*members, ref])
+    tasks = s.q(f"SELECT * FROM tasks WHERE note_id IN (SELECT note_id FROM entity_notes WHERE entity_id IN ({q}))"
+                f" AND opened_at<=? AND (closed_at IS NULL OR closed_at>?) ORDER BY opened_at", [*members, ref, ref])
+    months: Dict[str, int] = {}
+    for r in s.q(f"SELECT substr(at,1,7), COUNT(DISTINCT note_id) FROM entity_notes WHERE entity_id IN ({q}) AND at<=?"
+                 f" GROUP BY substr(at,1,7) ORDER BY 1", [*members, ref]):
+        months[r[0]] = r[1]
+    return {
+        "entity_id": eid, "resolved_from": entity_id if entity_id != eid else None, "as_of": at_n,
+        "kind": row["kind"], "name": name, "aliases": aliases, "summary": summary, "summary_history": history,
+        "first_seen": stats[0] or row["first_seen"], "last_seen": stats[1] or row["last_seen"],
+        "note_count": int(stats[2] or 0), "merged_from": [m for m in members if m != eid],
+        "merged_into": row["merged_into"] if at_n is None else None,
+        "beliefs": [c for c in s.beliefs_at(at_n or ids.utc_now())
+                    if c["subject_id"] in members or c["object_id"] in members],
+        "relations": relations_of(nb, eid, at_n),
+        "notes": [shapes.note_ref(r) for r in notes],
+        "open_tasks": [shapes.task(r) | {"status": "open", "closed_at": None} for r in tasks],
+        "sparkline": [{"month": k, "notes": v} for k, v in months.items()],
+    }
+
+
+def _archive_card(nb: Any, arch: Any, entity_id: str, at_n: Optional[str]) -> Dict[str, Any]:
+    """:func:`entity_card` from what the archive held at ``arch.at`` (same keys, plus ``as_received``)."""
+    s = nb.store
+    eid = arch.canonical(entity_id, at_n)
+    row = s.entity_row(eid)
+    members = arch.members(eid, at_n)
+    held = [m for m in members if arch.holds_entity(m)]
+    if not held:
+        raise ValueError("That entity was not in your notebook yet on that date.")
+    ref = at_n or "9999-12-31T23:59:59Z"
+    q = ",".join("?" for _ in members)
+    in_rev, p_rev = arch.revisions_in("revision_id")
+    in_note, p_note = arch.notes_in("en.note_id")
+    stats = s.q1(f"SELECT MIN(en.at), MAX(en.at), COUNT(DISTINCT en.note_id) FROM entity_notes en WHERE en.entity_id"
+                 f" IN ({q}) AND en.at<=? AND {in_note}", [*members, ref, *p_note])
+    created = s.q1(f"SELECT MIN(created_at) FROM entities WHERE entity_id IN ({q}) AND"
+                   f" {arch.revisions_in('created_revision_id')[0]}", [*members, arch.at])[0]
+    first = min([x for x in (stats[0], created) if x], default=None)
+    if at_n and (first is None or first > at_n):
+        raise ValueError("That entity was not known yet at that time.")
+    name = arch.name(eid, at_n) or row["name"]
+    aliases: List[str] = []
+    for m in members:
+        for (a,) in s.q(f"SELECT alias FROM entity_aliases WHERE entity_id=? AND added_at<=? AND {in_rev}"
+                        " ORDER BY added_at, alias", [m, ref, *p_rev]):
+            if ids.fold(a) != ids.fold(name) and ids.fold(a) not in {ids.fold(x) for x in aliases}:
+                aliases.append(a)
+        if m != eid:
+            mname = arch.name(m, at_n) or s.entity_row(m)["name"]
+            if ids.fold(mname) != ids.fold(name) and ids.fold(mname) not in {ids.fold(x) for x in aliases}:
+                aliases.append(mname)
+    history = [{"at": r["at"], "summary": r["summary"], "note_id": r["note_id"], "revision_id": r["revision_id"]}
+               for r in s.q(f"SELECT * FROM entity_summaries WHERE entity_id IN ({q}) AND at<=? AND {in_rev}"
+                            " ORDER BY at, revision_id", [*members, ref, *p_rev])]
+    notes = s.q(f"SELECT DISTINCT n.* FROM entity_notes en JOIN notes n ON n.note_id=en.note_id WHERE en.entity_id IN"
+                f" ({q}) AND en.at<=? AND n.archived_at IS NULL AND {in_note} ORDER BY n.occurred_at DESC LIMIT ?",
+                [*members, ref, *p_note, CARD_NOTES])
+    opened, p_open = arch.revisions_in("t.opened_revision_id")
+    tasks = []
+    for r in s.q(f"SELECT t.* FROM tasks t WHERE t.note_id IN (SELECT en.note_id FROM entity_notes en WHERE"
+                 f" en.entity_id IN ({q})) AND t.opened_at<=? AND {opened} ORDER BY t.opened_at",
+                 [*members, ref, *p_open]):
+        closed = r["closed_at"] is not None and arch.has(r["closed_revision_id"]) and r["closed_at"] <= ref
+        if not closed:
+            tasks.append(shapes.task(r) | {"status": "open", "closed_at": None})
+    months: Dict[str, int] = {}
+    for r in s.q(f"SELECT substr(en.at,1,7), COUNT(DISTINCT en.note_id) FROM entity_notes en WHERE en.entity_id IN"
+                 f" ({q}) AND en.at<=? AND {in_note} GROUP BY substr(en.at,1,7) ORDER BY 1", [*members, ref, *p_note]):
+        months[r[0]] = r[1]
+    return {
+        "entity_id": eid, "resolved_from": entity_id if entity_id != eid else None, "as_of": at_n,
+        "as_received": arch.at, "kind": row["kind"], "name": name, "aliases": aliases,
+        "summary": history[-1]["summary"] if history else None, "summary_history": history,
+        "first_seen": first, "last_seen": stats[1] or first, "note_count": int(stats[2] or 0),
+        "merged_from": [m for m in members if m != eid], "merged_into": None,
+        "beliefs": [c for c in s.beliefs_at(at_n or ids.utc_now(), as_received=arch.at)
+                    if c["subject_id"] in members or c["object_id"] in members],
+        "relations": relations_of(nb, eid, at_n, as_received=arch.at),
+        "notes": [shapes.note_ref(r) for r in notes],
+        "open_tasks": tasks,
+        "sparkline": [{"month": k, "notes": v} for k, v in months.items()],
+    }
+
+
+def relations_of(nb: Any, entity_id: str, at: Optional[str] = None, *, as_received: Optional[str] = None
+                 ) -> List[Dict[str, Any]]:
+    graph = nb.store.graph(focus=[entity_id], at=at, depth=1, limit=2000, as_received=as_received)
+    if as_received:
+        from ..store.archive import Archive
+
+        eid = Archive(nb.store, as_received).canonical(entity_id, at)
+    else:
+        eid = nb.store.canonical_at(entity_id, at)
+    names = {n["id"]: n["name"] for n in graph["nodes"]}
+    out = []
+    for e in graph["edges"]:
+        if eid not in (e["source"], e["target"]):
+            continue
+        other = e["target"] if e["source"] == eid else e["source"]
+        out.append({"other_id": other, "other_name": names.get(other), "predicate": e["predicate"],
+                    "direction": "out" if e["source"] == eid else "in", "first_seen": e["first_seen"],
+                    "last_seen": e["last_seen"], "count": e["count"], "active": e["active"]})
+    out.sort(key=lambda r: (-r["count"], r["other_id"], r["predicate"]))
+    return out
+
+
+def _claim_events(nb: Any, rows: Iterable[Any]) -> List[Dict[str, Any]]:
+    out = []
+    for r in rows:
+        d = nb.store.claim_dict(r)
+        closed = None
+        if r["closed_revision_id"]:
+            rv = nb.store.revision_row(r["closed_revision_id"])
+            closed = {"revision_id": r["closed_revision_id"], "at": rv["at"] if rv else None,
+                      "note_id": rv["note_id"] if rv else None, "reason": r["closed_reason"],
+                      "kind": "retract" if r["status"] == "retracted" else "supersede"}
+        d["closed"] = closed
+        d["reinforcements"] = [{"note_id": x["note_id"], "at": x["at"]} for x in nb.store.q(
+            "SELECT * FROM claim_reinforcements WHERE claim_id=? ORDER BY at", (r["claim_id"],))]
+        out.append(d)
+    return out
+
+
+def claims_timeline(nb: Any, entity_id: str, since: Optional[str] = None, until: Optional[str] = None
+                    ) -> List[Dict[str, Any]]:
+    """Claims about an entity (as subject or object) in history order, with their supersessions."""
+    _require_entity(nb, entity_id)
+    members = _members(nb, entity_id)
+    q = ",".join("?" for _ in members)
+    sql = f"SELECT * FROM claims WHERE (subject_id IN ({q}) OR object_id IN ({q}))"
+    params: List[Any] = [*members, *members]
+    if since:
+        sql += " AND observed_at>=?"
+        params.append(ids.normalize_time(since, field="since"))
+    if until:
+        sql += " AND observed_at<=?"
+        params.append(ids.normalize_time(until, field="until"))
+    return _claim_events(nb, nb.store.q(sql + " ORDER BY observed_at, seq", params))
+
+
+def entity_timeline(nb: Any, entity_id: str, since: Optional[str] = None, until: Optional[str] = None
+                    ) -> Dict[str, Any]:
+    """Claims and notes about the entity in history order (``GET /note/entities/{id}/timeline``)."""
+    _require_entity(nb, entity_id)
+    eid = nb.store.canonical(entity_id)
+    members = _members(nb, entity_id)
+    q = ",".join("?" for _ in members)
+    since_n = ids.optional_time(since, field="since")
+    until_n = ids.optional_time(until, field="until")
+    items: List[Dict[str, Any]] = []
+    for c in claims_timeline(nb, eid, since_n, until_n):
+        items.append({"type": "claim", "at": c["observed_at"], "claim": c})
+        if c["closed"] and c["closed"]["at"]:
+            items.append({"type": c["closed"]["kind"], "at": c["closed"]["at"], "claim_id": c["claim_id"],
+                          "note_id": c["closed"]["note_id"], "reason": c["closed"]["reason"],
+                          "superseded_by": c["superseded_by"]})
+    sql = (f"SELECT DISTINCT n.* FROM entity_notes en JOIN notes n ON n.note_id=en.note_id WHERE en.entity_id IN ({q})"
+           " AND n.archived_at IS NULL")
+    params: List[Any] = list(members)
+    if since_n:
+        sql += " AND n.occurred_at>=?"
+        params.append(since_n)
+    if until_n:
+        sql += " AND n.occurred_at<=?"
+        params.append(until_n)
+    for r in nb.store.q(sql, params):
+        items.append({"type": "note", "at": r["occurred_at"], "note": shapes.note_ref(r)})
+    for r in nb.store.q(f"SELECT * FROM merges WHERE from_id IN ({q}) OR into_id IN ({q})", [*members, *members]):
+        items.append({"type": "merge", "at": r["at"], "from_id": r["from_id"], "into_id": r["into_id"],
+                      "reason": r["reason"], "undone_at": r["undone_at"]})
+    order = {"note": 0, "claim": 1, "supersede": 2, "retract": 2, "merge": 3}
+    items = [i for i in items if (not since_n or i["at"] >= since_n) and (not until_n or i["at"] <= until_n)]
+    items.sort(key=lambda i: (i["at"], order.get(i["type"], 9)))
+    return {"entity_id": eid, "since": since_n, "until": until_n, "items": items}
+
+
+def relation(nb: Any, a: str, b: str, since: Optional[str] = None, until: Optional[str] = None) -> Dict[str, Any]:
+    """The relationship between two entities through time: claims between them in history order,
+    supersessions, and a then-versus-now of the link (``GET /note/relations/{a}/{b}``)."""
+    _require_entity(nb, a)
+    _require_entity(nb, b)
+    ma, mb = _members(nb, a), _members(nb, b)
+    qa, qb = ",".join("?" for _ in ma), ",".join("?" for _ in mb)
+    sql = (f"SELECT * FROM claims WHERE ((subject_id IN ({qa}) AND object_id IN ({qb})) OR"
+           f" (subject_id IN ({qb}) AND object_id IN ({qa})))")
+    params: List[Any] = [*ma, *mb, *mb, *ma]
+    since_n = ids.optional_time(since, field="since")
+    until_n = ids.optional_time(until, field="until")
+    if since_n:
+        sql += " AND observed_at>=?"
+        params.append(since_n)
+    if until_n:
+        sql += " AND observed_at<=?"
+        params.append(until_n)
+    claims = _claim_events(nb, nb.store.q(sql + " ORDER BY observed_at, seq", params))
+    supersessions = [{"claim_id": c["claim_id"], "superseded_by": c["superseded_by"], **(c["closed"] or {})}
+                     for c in claims if c["closed"]]
+    return {"a": nb.store.entity_brief(nb.store.canonical(a)), "b": nb.store.entity_brief(nb.store.canonical(b)),
+            "since": since_n, "until": until_n, "claims": claims, "supersessions": supersessions,
+            "then_now": relation_then_now(claims)}
+
+
+def neighbours(nb: Any, entity_ids: Sequence[str], at: Optional[str] = None, limit: int = 20) -> List[Dict[str, Any]]:
+    seen = {nb.store.canonical_at(e, at) for e in entity_ids}
+    out: Dict[str, Dict[str, Any]] = {}
+    for e in seen:
+        for r in relations_of(nb, e, at):
+            if r["other_id"] in seen:
+                continue
+            n = out.setdefault(r["other_id"], {"entity_id": r["other_id"], "name": r["other_name"], "via": [],
+                                               "count": 0})
+            n["via"].append({"entity_id": e, "predicate": r["predicate"], "direction": r["direction"]})
+            n["count"] += r["count"]
+    ranked = sorted(out.values(), key=lambda n: (-n["count"], n["entity_id"]))[: clamp_limit(limit, 20, 200)]
+    for n in ranked:
+        row = nb.store.entity_row(n["entity_id"])
+        n["kind"] = row["kind"] if row else None
+    return ranked
+
+
+def expand(nb: Any, note_ids: Any = None, entity_ids: Any = None, as_of: Optional[str] = None) -> Dict[str, Any]:
+    """``note_expand``: cards, claim timelines, relation timelines, neighbours and then-versus-now
+    for the given entities plus the entities the given notes mention."""
+    as_of_n = ids.optional_time(as_of, field="as_of")
+    wanted: List[str] = []
+    for e in list(entity_ids or []):
+        if nb.store.entity_row(str(e)) is not None:
+            c = nb.store.canonical_at(str(e), as_of_n)
+            if c not in wanted:
+                wanted.append(c)
+    for nid in list(note_ids or []):
+        for r in nb.store.q("SELECT entity_id FROM entity_notes WHERE note_id=? ORDER BY entity_id", (str(nid),)):
+            c = nb.store.canonical_at(r[0], as_of_n)
+            if c not in wanted:
+                wanted.append(c)
+    wanted = wanted[:MAX_EXPAND_ENTITIES]
+    cards, timelines, thens = [], [], []
+    for e in wanted:
+        try:
+            cards.append(entity_card(nb, e, as_of_n))
+        except ValueError:
+            continue  # not known yet at as_of
+        tl = claims_timeline(nb, e, until=as_of_n)
+        timelines.append({"entity_id": e, "claims": tl})
+        thens.append(then_now(nb, e))
+    pairs = []
+    known = {c["entity_id"] for c in cards}
+    for i, a in enumerate(sorted(known)):
+        for b in sorted(known)[i + 1:]:
+            if nb.store.q1("SELECT 1 FROM relations WHERE (subject_id=? AND object_id=?) OR (subject_id=? AND"
+                           " object_id=?)", (a, b, b, a)):
+                pairs.append((a, b))
+    rel = [relation(nb, a, b, until=as_of_n) for a, b in pairs[:MAX_RELATION_PAIRS]]
+    return {"as_of": as_of_n, "entities": cards, "claims_timeline": timelines, "relation_timelines": rel,
+            "neighbours": neighbours(nb, list(known), as_of_n), "then_now": thens}
+
+
+def claim_chain(nb: Any, claim_id: str) -> Dict[str, Any]:
+    """Why do I believe this: the claim, what it superseded or was superseded by, the notes and
+    attachment excerpts behind it (``GET /note/claims/{id}``)."""
+    row = nb.store.claim_row(claim_id)
+    if row is None:
+        raise ValueError("That statement is not in this notebook.")
+    [claim] = _claim_events(nb, [row])
+    supersedes = _claim_events(nb, nb.store.q("SELECT * FROM claims WHERE superseded_by=? ORDER BY observed_at",
+                                              (claim_id,)))
+    superseded_by = None
+    if row["superseded_by"]:
+        r2 = nb.store.claim_row(row["superseded_by"])
+        superseded_by = _claim_events(nb, [r2])[0] if r2 else None
+    disputed = [nb.store.claim_dict(x) for x in (nb.store.claim_row(c) for c in _uj(row["disputed_with_json"], []))
+                if x is not None]
+    note_ids = [row["note_id"]] if row["note_id"] else []
+    note_ids += [r["note_id"] for r in nb.store.q("SELECT note_id FROM claim_reinforcements WHERE claim_id=? AND"
+                                                  " note_id IS NOT NULL ORDER BY at", (claim_id,))]
+    notes = []
+    for nid in dict.fromkeys(note_ids):
+        n = nb.store.note_row(nid)
+        if n is None:
+            continue
+        ref = shapes.note_ref(n)
+        ref["attachments"] = []
+        for a in nb.store.attachments_of(nid):
+            path = nb.root / "extracted" / f"{a['sha256']}.txt"
+            text = path.read_text(encoding="utf-8") if a["extracted_chars"] and path.exists() else ""
+            ref["attachments"].append({"attachment_id": a["attachment_id"], "filename": a["filename"],
+                                       "mime": a["mime"], "extracted_by": a["extracted_by"],
+                                       "excerpt": _around(text, row["evidence"])})
+        notes.append(ref)
+    rev = nb.store.revision_row(row["revision_id"])
+    return {"claim": claim, "supersedes": supersedes, "superseded_by": superseded_by, "disputed_with": disputed,
+            "notes": notes, "revision": {k: v for k, v in nb.store.revision_dict(rev).items() if k != "ops"}
+            if rev else None}
+
+
+def _around(text: str, evidence: Optional[str], width: int = 400) -> str:
+    """An excerpt of attachment text around the evidence quote when it occurs verbatim, else its start."""
+    if not text:
+        return ""
+    pos = text.find(evidence) if evidence else -1
+    if pos < 0:
+        return shapes.excerpt(text, width)
+    start = max(0, pos - width // 2)
+    return ("…" if start else "") + text[start: start + width] + ("…" if start + width < len(text) else "")

abstractnote/recall/insights.py

@@ -0,0 +1,62 @@
+"""``note_insight_write``: system-authored insight and synthesis notes with provenance."""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional, Sequence
+
+from .. import ids
+from ..ingest.normalise import text_norm
+from ..model import INSIGHT_KINDS
+from ..store.notes import _j, note_vector_text
+
+
+def write_insight(nb: Any, kind: str, title: str, body: str, evidence_note_ids: Optional[Sequence[str]] = None,
+                  entity_ids: Optional[Sequence[str]] = None, run_id: Optional[str] = None,
+                  author: str = "abstractnote") -> Dict[str, Any]:
+    """Store an insight (``dossier`` -> a ``synthesis`` note). Unknown evidence notes or entities are
+    left out and named in ``warnings``. Writing the same insight twice returns the same note."""
+    nb.require_embedder()
+    if kind not in INSIGHT_KINDS:
+        raise ValueError(f"The insight kind must be one of {', '.join(INSIGHT_KINDS)}.")
+    title = str(title or "").strip()
+    body = str(body or "").strip()
+    if not title and not body:
+        raise ValueError("An insight needs a title or a body.")
+    warnings: List[str] = []
+    evidence: List[str] = []
+    for nid in evidence_note_ids or []:
+        nid = str(nid)
+        if nb.store.note_row(nid) is None:
+            warnings.append(f"The evidence note {nid} is not in this notebook; it was left out.")
+        elif nid not in evidence:
+            evidence.append(nid)
+    ents: List[str] = []
+    for eid in entity_ids or []:
+        eid = str(eid)
+        if nb.store.entity_row(eid) is None:
+            warnings.append(f"The entity {eid} is not in this notebook; it was left out.")
+        else:
+            c = nb.store.canonical(eid)
+            if c not in ents:
+                ents.append(c)
+    note_id = ids.note_id_for_insight(kind, title, body, evidence)
+    if nb.store.note_row(note_id) is not None:
+        return {"note_id": note_id, "status": "duplicate", "warnings": warnings}
+    now = ids.utc_now()
+    vtext = note_vector_text(title, body, [], 8000)
+    [vkey] = nb.cache.ensure([vtext]) if vtext.strip() else [None]
+    with nb.store.tx() as c:
+        nb.store.insert_note(c, {
+            "note_id": note_id, "kind": "synthesis" if kind == "dossier" else "insight", "title": title or None,
+            "text": body, "text_norm": text_norm(body), "text_hash": None, "occurred_at": now, "received_at": now,
+            "processed_at": now, "source_json": _j({"client": "agent", "author": author}), "client": "agent",
+            "priority": 1, "status": "processed", "run_id": run_id, "size": len(body), "vector_key": vkey,
+            "insight_kind": kind, "evidence_note_ids_json": json.dumps(evidence), "entity_ids_json": json.dumps(ents),
+        })
+        for nid in evidence:
+            c.execute("INSERT OR IGNORE INTO note_links(src_note_id, dst_note_id, predicate, score, matched_by,"
+                      " detail_json, created_at) VALUES (?,?,'cito:citesAsEvidence',1.0,'insight',NULL,?)",
+                      (note_id, nid, now))
+        nb.store.refresh_note_fts(c, note_id, [])
+    return {"note_id": note_id, "status": "new", "warnings": warnings}

abstractnote/recall/reflect.py

@@ -0,0 +1,202 @@
+"""Deterministic reflect candidates (docs/cognition.md, "note-reflect").
+
+Each function returns ``{"items": [...]}``; every item names the ``rule`` that
+produced it and carries ``note_ids`` and ``entity_ids`` so the reflect flow can
+write an insight with provenance. Thresholds are explicit constants; the model
+decides what deserves attention.
+"""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import Any, Dict, List, Optional, Tuple
+
+from .. import ids
+from ..store.notes import _uj, clamp_limit
+from . import shapes
+
+OPEN_TASK_AGE_DAYS = 14
+QUESTION_AGE_DAYS = 14
+TASK_MENTIONS_MIN = 2          # an open task whose note was revisited this many times
+SILENT_DAYS = 60               # an entity or topic with no note for this long has gone quiet
+RECENT_DAYS = 7                # "recent" notes for resurfacing
+
+
+def _now(now: Optional[str]) -> str:
+    return ids.normalize_time(now) if now else ids.utc_now()
+
+
+def _minus(ts: str, days: int) -> str:
+    return (ids.parse_time(ts) - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _days(a: str, b: str) -> int:
+    return int((ids.parse_time(b) - ids.parse_time(a)).total_seconds() // 86400)
+
+
+def _since(since: Optional[str]) -> Optional[str]:
+    return ids.optional_time(since, field="since")
+
+
+def _entities_of_note(nb: Any, note_id: Optional[str]) -> List[str]:
+    if not note_id:
+        return []
+    return sorted({nb.store.canonical(r[0]) for r in nb.store.q("SELECT entity_id FROM entity_notes WHERE note_id=?",
+                                                                (note_id,))})
+
+
+def open_loops(nb: Any, limit: int = 20, since: Optional[str] = None, *, now: Optional[str] = None
+               ) -> Dict[str, Any]:
+    limit = clamp_limit(limit, 20, 200)
+    now = _now(now)
+    since_n = _since(since)
+    items: List[Dict[str, Any]] = []
+    q = nb.store.q
+    sql = "SELECT * FROM tasks WHERE status='open' AND opened_at<=?"
+    params: List[Any] = [_minus(now, OPEN_TASK_AGE_DAYS)]
+    if since_n:
+        sql += " AND opened_at>=?"
+        params.append(since_n)
+    for r in q(sql + " ORDER BY opened_at, task_id", params):
+        mentions = 0
+        if r["note_id"]:
+            mentions = q("SELECT COUNT(*) FROM note_links WHERE dst_note_id=? AND predicate='ex:revisits'",
+                         (r["note_id"],))[0][0]
+        rule = "task_mentioned_repeatedly" if mentions >= TASK_MENTIONS_MIN else f"open_task_older_than_{OPEN_TASK_AGE_DAYS}_days"
+        items.append({"rule": rule, "kind": "task", "task_id": r["task_id"], "text": r["text"],
+                      "opened_at": r["opened_at"], "age_days": _days(r["opened_at"], now), "mentions": mentions + 1,
+                      "note_ids": [r["note_id"]] if r["note_id"] else [], "entity_ids": _entities_of_note(nb, r["note_id"])})
+    sql = ("SELECT * FROM notes n WHERE n.kind='question' AND n.archived_at IS NULL AND n.occurred_at<=? AND NOT"
+           " EXISTS (SELECT 1 FROM note_links l WHERE l.dst_note_id=n.note_id AND l.predicate IN"
+           " ('ex:revisits','dcterms:references'))")
+    params = [_minus(now, QUESTION_AGE_DAYS)]
+    if since_n:
+        sql += " AND n.occurred_at>=?"
+        params.append(since_n)
+    for r in q(sql + " ORDER BY n.occurred_at, n.note_id", params):
+        items.append({"rule": "question_without_followup", "kind": "question", "text": r["title"] or
+                      shapes.excerpt(r["text"], 160), "opened_at": r["occurred_at"],
+                      "age_days": _days(r["occurred_at"], now), "note_ids": [r["note_id"]],
+                      "entity_ids": _entities_of_note(nb, r["note_id"])})
+    items.sort(key=lambda it: (0 if it["rule"] == "task_mentioned_repeatedly" else 1, it["opened_at"]))
+    return {"items": items[:limit]}
+
+
+def contradictions(nb: Any, limit: int = 20, since: Optional[str] = None) -> Dict[str, Any]:
+    limit = clamp_limit(limit, 20, 200)
+    since_n = _since(since)
+    q = nb.store.q
+    items: List[Dict[str, Any]] = []
+    seen = set()
+    sql = "SELECT * FROM claims WHERE status='active' AND disputed_with_json!='[]'"
+    params: List[Any] = []
+    if since_n:
+        sql += " AND observed_at>=?"
+        params.append(since_n)
+    for r in q(sql + " ORDER BY observed_at, claim_id", params):
+        for other in _uj(r["disputed_with_json"], []) or []:
+            pair = tuple(sorted((r["claim_id"], other)))
+            if pair in seen:
+                continue
+            o = nb.store.claim_row(other)
+            if o is None or o["status"] != "active":
+                continue
+            seen.add(pair)
+            claims = [nb.store.claim_dict(x) for x in (r, o)]
+            items.append({"rule": "disputed_pair", "claims": claims,
+                          "text": " / ".join(_claim_line(c) for c in claims),
+                          "note_ids": sorted({c["note_id"] for c in claims if c["note_id"]}),
+                          "entity_ids": sorted({nb.store.canonical(c["subject_id"]) for c in claims})})
+    groups: Dict[Tuple[str, str], List[Any]] = {}
+    for r in q("SELECT * FROM claims WHERE status='active' AND valid_until IS NULL ORDER BY observed_at, claim_id"):
+        groups.setdefault((nb.store.canonical(r["subject_id"]), r["predicate"]), []).append(r)
+    for (subject, predicate), rows in sorted(groups.items()):
+        objects = {(nb.store.canonical(x["object_id"]) if x["object_id"] else None, x["object_literal"]) for x in rows}
+        if len(objects) < 2:
+            continue
+        if since_n and max(x["observed_at"] for x in rows) < since_n:
+            continue
+        claims = [nb.store.claim_dict(x) for x in rows]
+        items.append({"rule": "several_active_objects", "subject_id": subject, "predicate": predicate,
+                      "claims": claims, "text": " / ".join(_claim_line(c) for c in claims),
+                      "note_ids": sorted({c["note_id"] for c in claims if c["note_id"]}), "entity_ids": [subject]})
+    return {"items": items[:limit]}
+
+
+def _claim_line(c: Dict[str, Any]) -> str:
+    obj = c.get("object_name") or c.get("object_literal") or c.get("object_id")
+    return f"{c.get('subject_name') or c['subject_id']} {c['predicate']} {obj} ({c['observed_at'][:10]})"
+
+
+def resurface(nb: Any, limit: int = 20, since: Optional[str] = None, *, now: Optional[str] = None
+              ) -> Dict[str, Any]:
+    limit = clamp_limit(limit, 20, 200)
+    now = _now(now)
+    recent_from = _since(since) or _minus(now, RECENT_DAYS)
+    quiet_before = _minus(now, SILENT_DAYS)
+    q = nb.store.q
+    recent_entities = {nb.store.canonical(r[0]) for r in q("SELECT DISTINCT entity_id FROM entity_notes WHERE at>=?",
+                                                           (recent_from,))}
+    items: List[Dict[str, Any]] = []
+    if recent_entities:
+        marks = ",".join("?" for _ in recent_entities)
+        rel = q(f"SELECT subject_id, object_id FROM relations WHERE subject_id IN ({marks}) OR object_id IN ({marks})",
+                [*recent_entities, *recent_entities])
+        neighbours: Dict[str, set] = {}
+        for r in rel:
+            for a, b in ((r[0], r[1]), (r[1], r[0])):
+                if a in recent_entities and b not in recent_entities:
+                    neighbours.setdefault(b, set()).add(a)
+        for eid, via in sorted(neighbours.items()):
+            e = nb.store.entity_row(eid)
+            if e is None or e["merged_into"] or not e["last_seen"] or e["last_seen"] >= quiet_before:
+                continue
+            last_notes = [r[0] for r in q("SELECT note_id FROM entity_notes WHERE entity_id=? ORDER BY at DESC LIMIT 3",
+                                          (eid,))]
+            items.append({"rule": "silent_but_related", "kind": "entity", "entity_id": eid, "text": e["name"],
+                          "last_seen": e["last_seen"], "silent_days": _days(e["last_seen"], now),
+                          "related_to": sorted(via), "note_ids": last_notes, "entity_ids": [eid, *sorted(via)]})
+    for r in q("SELECT DISTINCT t.topic FROM note_topics t JOIN notes n ON n.note_id=t.note_id WHERE n.occurred_at>=?",
+               (recent_from,)):
+        prev = q("SELECT MAX(n.occurred_at), GROUP_CONCAT(n.note_id) FROM note_topics t JOIN notes n ON"
+                 " n.note_id=t.note_id WHERE t.topic=? AND n.occurred_at<?", (r[0], recent_from))[0]
+        if prev[0] and prev[0] < _minus(recent_from, SILENT_DAYS):
+            items.append({"rule": "topic_back_after_silence", "kind": "topic", "topic": r[0], "text": r[0],
+                          "last_seen": prev[0], "silent_days": _days(prev[0], recent_from),
+                          "note_ids": (prev[1] or "").split(",")[-3:], "entity_ids": []})
+    items.sort(key=lambda it: -it["silent_days"])
+    return {"items": items[:limit]}
+
+
+def anniversary_rows(nb: Any, limit: int = 20, since: Optional[str] = None, *, now: Optional[str] = None
+                     ) -> List[Tuple[Any, str]]:
+    """Notes from today's month and day in earlier years, then today's day of month in the past 12 months."""
+    now = _now(now)
+    since_n = _since(since)
+    md, d, ym = now[5:10], now[8:10], now[:7]
+    out: List[Tuple[Any, str]] = []
+    base = "SELECT * FROM notes WHERE insight_kind IS NULL AND archived_at IS NULL"
+    params: List[Any] = []
+    if since_n:
+        base += " AND occurred_at>=?"
+        params.append(since_n)
+    for r in nb.store.q(base + " AND substr(occurred_at,6,5)=? AND substr(occurred_at,1,4)<? ORDER BY occurred_at DESC",
+                        [*params, md, now[:4]]):
+        out.append((r, "same_day_previous_year"))
+    year_ago = _minus(now, 366)
+    for r in nb.store.q(base + " AND substr(occurred_at,9,2)=? AND substr(occurred_at,1,7)<? AND occurred_at>=?"
+                        " AND substr(occurred_at,6,5)!=? ORDER BY occurred_at DESC", [*params, d, ym, year_ago, md]):
+        out.append((r, "same_day_previous_month"))
+    return out[: clamp_limit(limit, 20, 200)]
+
+
+def anniversaries(nb: Any, limit: int = 20, since: Optional[str] = None, *, now: Optional[str] = None
+                  ) -> Dict[str, Any]:
+    now_n = _now(now)
+    items = []
+    for r, rule in anniversary_rows(nb, limit, since, now=now_n):
+        ref = shapes.note_ref(r)
+        items.append({"rule": rule, "kind": "note", **ref, "text": ref["title"] or ref["excerpt"],
+                      "years_ago": int(now_n[:4]) - int(r["occurred_at"][:4]),
+                      "note_ids": [r["note_id"]], "entity_ids": _entities_of_note(nb, r["note_id"])})
+    return {"items": items}