abstractnote 0.1.0__py3-none-any.whl

abstractnote/ingest/normalise.py

@@ -0,0 +1,95 @@
+"""Text normalisation and sectioning.
+
+``text_norm`` is the hashing and full-text form of a note: Unicode NFC,
+whitespace collapsed, lower-cased. ``sections`` splits a long text for
+processing by Markdown headings, then paragraphs, packing pieces up to a
+character budget; every character of the input is kept (no truncation).
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from typing import Dict, List
+
+DEFAULT_SECTION_CHARS = 12000
+
+_HEADING = re.compile(r"^#{1,6}\s")  # Markdown heading line (document structure, not meaning)
+
+
+def text_norm(text: str) -> str:
+    return " ".join(unicodedata.normalize("NFC", str(text or "")).split()).lower()
+
+
+def normalize_topic(topic: str) -> str:
+    """Topic spelling: no leading ``#``, words joined with ``-``, lower case."""
+    t = str(topic or "").strip().lstrip("#").strip()
+    return "-".join(t.lower().split())
+
+
+def _blocks(text: str) -> List[str]:
+    """Heading-led blocks, each split further into paragraphs (blank-line separated)."""
+    lines = text.splitlines(keepends=True)
+    blocks: List[List[str]] = [[]]
+    for line in lines:
+        if _HEADING.match(line) and blocks[-1]:
+            blocks.append([])
+        blocks[-1].append(line)
+    out: List[str] = []
+    for b in blocks:
+        chunk = "".join(b)
+        if not chunk:
+            continue
+        paras = re.split(r"(\n[ \t]*\n)", chunk)
+        # keep separators attached to the paragraph before them so nothing is lost
+        cur = ""
+        for piece in paras:
+            cur += piece
+            if re.fullmatch(r"\n[ \t]*\n", piece):
+                out.append(cur)
+                cur = ""
+        if cur:
+            out.append(cur)
+    return out
+
+
+def _split_hard(piece: str, budget: int) -> List[str]:
+    """Split an over-budget paragraph at line ends, then sentence ends, then the budget itself."""
+    out: List[str] = []
+    rest = piece
+    while len(rest) > budget:
+        window = rest[:budget]
+        cut = max(window.rfind("\n"), window.rfind(". "), window.rfind("? "), window.rfind("! "))
+        cut = cut + 1 if cut > budget // 4 else budget
+        out.append(rest[:cut])
+        rest = rest[cut:]
+    if rest:
+        out.append(rest)
+    return out
+
+
+def sections(text: str, budget: int = DEFAULT_SECTION_CHARS) -> List[Dict[str, object]]:
+    """``[{index, start, end, chars, text}]`` covering ``text`` exactly once, each <= budget chars."""
+    text = str(text or "")
+    budget = max(200, int(budget))
+    if len(text) <= budget:
+        return [{"index": 0, "start": 0, "end": len(text), "chars": len(text), "text": text}] if text else []
+    pieces: List[str] = []
+    for block in _blocks(text):
+        pieces.extend(_split_hard(block, budget) if len(block) > budget else [block])
+    packed: List[str] = []
+    cur = ""
+    for p in pieces:
+        if cur and len(cur) + len(p) > budget:
+            packed.append(cur)
+            cur = ""
+        cur += p
+    if cur:
+        packed.append(cur)
+    out = []
+    pos = 0
+    for i, s in enumerate(packed):
+        out.append({"index": i, "start": pos, "end": pos + len(s), "chars": len(s), "text": s})
+        pos += len(s)
+    assert pos == len(text), "sectioning must cover the text exactly"
+    return out

abstractnote/model.py

@@ -0,0 +1,229 @@
+"""The records AbstractNote keeps (docs/data-model.md), as pydantic v2 models.
+
+These types are the read and write shapes of the library. The SQLite schema
+in ``abstractnote.store.schema`` stores them; ``abstractnote.store.kg`` projects
+claims, names, aliases and merges into AbstractMemory.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+NoteKind = Literal[
+    "idea", "task", "decision", "fact", "event", "question", "reflection", "reference", "log",
+    "conversation", "personal", "synthesis", "insight",
+]
+NOTE_KINDS: tuple = NoteKind.__args__  # type: ignore[attr-defined]
+FILING_KINDS: tuple = tuple(k for k in NOTE_KINDS if k not in ("synthesis", "insight"))
+
+NoteStatus = Literal["received", "queued", "processing", "processed", "failed", "duplicate", "archived"]
+Client = Literal["web", "watch", "iphone", "cli", "agent", "importer"]
+CLIENTS: tuple = Client.__args__  # type: ignore[attr-defined]
+EntityKind = Literal["person", "org", "project", "concept", "place", "event", "artifact", "software", "thing"]
+ENTITY_KINDS: tuple = EntityKind.__args__  # type: ignore[attr-defined]
+Certainty = Literal["stated", "probable", "possible", "reported", "disputed"]
+CERTAINTIES: tuple = Certainty.__args__  # type: ignore[attr-defined]
+Stance = Literal["assert", "revise", "retract"]
+ClaimStatus = Literal["active", "superseded", "retracted"]
+TaskStatus = Literal["open", "done", "dropped"]
+InsightKind = Literal["warning", "reminder", "connection", "contradiction", "evolution", "dossier", "cue"]
+INSIGHT_KINDS: tuple = InsightKind.__args__  # type: ignore[attr-defined]
+ExtractedBy = str  # plain | markdown | pdf | vision:<model> | stt:<model> | media-metadata | none
+
+CERTAINTY_CONFIDENCE: Dict[str, float] = {
+    "stated": 0.9, "probable": 0.7, "possible": 0.5, "reported": 0.6, "disputed": 0.3,
+}
+URGENCY_PRIORITY: Dict[str, int] = {"bulk": 0, "normal": 1, "high": 2, "urgent": 3}
+
+OP_NAMES = (
+    "entity.create", "entity.alias", "entity.rename", "entity.summary", "entity.merge", "entity.unmerge",
+    "entity.mention", "claim.add", "claim.supersede", "claim.retract", "claim.reinforce", "topic.assign",
+    "task.open", "task.close", "task.drop", "note.summary",
+)
+
+
+class _Model(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class Location(_Model):
+    lat: float
+    lon: float
+    accuracy_m: Optional[float] = None
+    label: Optional[str] = None
+
+
+class Provenance(_Model):
+    """The ``source`` of a note or a user action."""
+
+    model_config = ConfigDict(extra="allow")
+    client: Client
+    importer: Optional[str] = None
+    origin: Dict[str, Any] = Field(default_factory=dict)
+    author: Optional[str] = None
+    account: Optional[str] = None
+
+
+class Attachment(_Model):
+    attachment_id: str
+    note_id: str
+    sha256: str
+    size: int
+    mime: str
+    filename: Optional[str] = None
+    artifact_ref: Optional[Dict[str, Any]] = None
+    blob_path: str
+    extracted_text: Optional[str] = None
+    extracted_by: ExtractedBy = "none"
+    extracted_chars: int = 0
+
+
+class Note(_Model):
+    note_id: str
+    kind: Optional[NoteKind] = None
+    title: Optional[str] = None
+    text: str = ""
+    text_norm: str = ""
+    summary: Optional[str] = None
+    occurred_at: str
+    received_at: str
+    processed_at: Optional[str] = None
+    source: Provenance
+    location: Optional[Location] = None
+    language: Optional[str] = None
+    topics: List[str] = Field(default_factory=list)
+    priority: int = 1
+    status: NoteStatus = "received"
+    duplicate_of: Optional[str] = None
+    session_id: Optional[str] = None
+    run_id: Optional[str] = None
+    archived_at: Optional[str] = None
+    attachments: List[Attachment] = Field(default_factory=list)
+    # Insight notes (kind insight|synthesis)
+    insight_kind: Optional[InsightKind] = None
+    evidence_note_ids: List[str] = Field(default_factory=list)
+    entity_ids: List[str] = Field(default_factory=list)
+    acked_at: Optional[str] = None
+
+
+class Entity(_Model):
+    entity_id: str
+    kind: EntityKind
+    name: str
+    aliases: List[str] = Field(default_factory=list)
+    summary: Optional[str] = None
+    first_seen: Optional[str] = None
+    last_seen: Optional[str] = None
+    note_count: int = 0
+    merged_into: Optional[str] = None
+
+
+class Claim(_Model):
+    claim_id: str
+    subject_id: str
+    predicate: str
+    object_id: Optional[str] = None
+    object_literal: Optional[str] = None
+    certainty: Certainty = "stated"
+    stance: Stance = "assert"
+    observed_at: str
+    valid_from: Optional[str] = None
+    valid_until: Optional[str] = None
+    status: ClaimStatus = "active"
+    superseded_by: Optional[str] = None
+    evidence: Optional[str] = None
+    note_id: Optional[str] = None
+    revision_id: Optional[str] = None
+    assertion_id: Optional[str] = None
+    disputed_with: List[str] = Field(default_factory=list)
+
+
+class Relation(_Model):
+    subject_id: str
+    object_id: str
+    predicate: str
+    first_seen: str
+    last_seen: str
+    count: int
+    active_claim_ids: List[str] = Field(default_factory=list)
+
+
+class RevisionOp(_Model):
+    op: Literal[OP_NAMES]  # type: ignore[valid-type]
+    payload: Dict[str, Any] = Field(default_factory=dict)
+
+
+class Revision(_Model):
+    revision_id: str
+    note_id: Optional[str] = None
+    attempt: Optional[int] = None
+    action: Optional[str] = None
+    at: str
+    recorded_at: Optional[str] = None
+    run_id: Optional[str] = None
+    ops: List[RevisionOp] = Field(default_factory=list)
+    summary: str = ""
+    source: Optional[Dict[str, Any]] = None
+    warnings: List[str] = Field(default_factory=list)
+    applied_index: bool = False
+    applied_kg: bool = False
+
+
+class QueueItem(_Model):
+    note_id: str
+    priority: int
+    enqueued_at: str
+    occurred_at: str
+    size: int = 0
+    state: Literal["queued", "leased", "done", "failed"] = "queued"
+    leased_by: Optional[str] = None
+    lease_until: Optional[str] = None
+    attempts: int = 0
+    failures: int = 0
+    next_attempt_at: Optional[str] = None
+    last_error: Optional[str] = None
+
+
+class Task(_Model):
+    task_id: str
+    note_id: Optional[str] = None
+    text: str
+    status: TaskStatus = "open"
+    due: Optional[str] = None
+    opened_at: Optional[str] = None
+    closed_at: Optional[str] = None
+    closed_by_note_id: Optional[str] = None
+
+
+class Topic(_Model):
+    topic: str
+    note_count: int = 0
+    first_seen: Optional[str] = None
+    last_seen: Optional[str] = None
+
+
+class Insight(_Model):
+    note_id: str
+    insight_kind: InsightKind
+    title: str
+    body: str
+    evidence_note_ids: List[str] = Field(default_factory=list)
+    entity_ids: List[str] = Field(default_factory=list)
+    occurred_at: str
+    acked_at: Optional[str] = None
+    run_id: Optional[str] = None
+
+
+class CaptureResult(_Model):
+    note_id: str
+    status: Literal["new", "duplicate"]
+    duplicate_of: Optional[str] = None
+    first_saved_at: Optional[str] = None
+    received_at: str
+    revisits: List[Dict[str, Any]] = Field(default_factory=list)
+    similar: List[Dict[str, Any]] = Field(default_factory=list)
+    topics: List[str] = Field(default_factory=list)
+    attachments: List[Dict[str, Any]] = Field(default_factory=list)
+    warnings: List[str] = Field(default_factory=list)

abstractnote/process/__init__.py

@@ -0,0 +1,5 @@
+"""Processing: the pure functions behind note_context, note_resolve_candidates and note_apply_revision.
+
+The model calls themselves are flow nodes (abstractnote.flows); this package
+provides their schemas, prompts and the deterministic steps around them.
+"""

abstractnote/process/context.py

@@ -0,0 +1,218 @@
+"""``note_context``: what the notebook already knows that is relevant to one note.
+
+The packet lists candidate entities (exact name or alias found in the note's
+words, then vector neighbours of the note), their active claims and the open
+tasks, each with an ``index``. The Linking step refers to existing claims and
+open tasks only by those indexes; the packet is stored so that
+``note_apply_revision`` can check every reference by set membership.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Dict, List, Optional, Sequence
+
+from .. import ids
+from ..ingest.normalise import DEFAULT_SECTION_CHARS, sections
+from ..recall.shapes import excerpt
+from ..store.embeddings import top_k
+from .vocabulary import vocabulary
+
+MAX_CANDIDATE_ENTITIES = 15
+MAX_CLAIMS_PER_ENTITY = 8
+MAX_OPEN_TASKS = 20
+SIMILAR_NOTES = 5
+ENTITY_VECTOR_MIN = 0.3
+
+_WORD = re.compile(r"\w+", re.UNICODE)
+
+
+def words(text: str) -> List[str]:
+    return _WORD.findall(ids.fold(text))
+
+
+# ----------------------------------------------------------------------------- packet storage
+
+def load_packet(nb: Any, note_id: str) -> Optional[Dict[str, Any]]:
+    row = nb.store.q1("SELECT packet_json FROM context_packets WHERE note_id=?", (note_id,))
+    return json.loads(row[0]) if row else None
+
+
+def save_packet(nb: Any, note_id: str, packet: Dict[str, Any]) -> None:
+    with nb.store.lock:
+        nb.store.conn.execute("INSERT OR REPLACE INTO context_packets(note_id, created_at, packet_json) VALUES (?,?,?)",
+                              (note_id, ids.utc_now(), json.dumps(packet, sort_keys=True)))
+
+
+def empty_packet() -> Dict[str, Any]:
+    return {"candidate_entities": [], "existing_claims": [], "open_tasks": [], "mentions": {}}
+
+
+# ----------------------------------------------------------------------------- views
+
+def claim_view(nb: Any, row: Any, index: int) -> Dict[str, Any]:
+    subject = nb.store._name(row["subject_id"]) or row["subject_id"]
+    obj = (nb.store._name(row["object_id"]) or row["object_id"]) if row["object_id"] else row["object_literal"]
+    return {"index": index, "claim_id": row["claim_id"], "subject": subject, "predicate": row["predicate"],
+            "object": obj, "object_is_literal": row["object_id"] is None, "observed_at": row["observed_at"],
+            "certainty": row["certainty"], "evidence": row["evidence"]}
+
+
+def active_claim_rows(nb: Any, entity_id: str, limit: int = MAX_CLAIMS_PER_ENTITY) -> List[Any]:
+    members = nb.store.merged_members(nb.store.canonical(entity_id))
+    q = ",".join("?" for _ in members)
+    return nb.store.q(
+        f"SELECT * FROM claims WHERE status='active' AND valid_until IS NULL AND (subject_id IN ({q}) OR object_id IN"
+        f" ({q})) ORDER BY observed_at DESC, claim_id LIMIT ?", [*members, *members, int(limit)])
+
+
+def entity_view(nb: Any, entity_id: str) -> Dict[str, Any]:
+    e = nb.store.entity_row(entity_id)
+    return {"entity_id": e["entity_id"], "name": e["name"], "kind": e["kind"], "aliases": nb.store.aliases(entity_id),
+            "summary": e["summary"], "last_seen": e["last_seen"]}
+
+
+def attach_claims(nb: Any, packet: Dict[str, Any], entity_id: str, views: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Active claims of an entity as indexed views, appending new ones to the packet."""
+    out = []
+    for row in active_claim_rows(nb, entity_id):
+        cid = row["claim_id"]
+        if cid in packet["existing_claims"]:
+            idx = packet["existing_claims"].index(cid)
+        else:
+            idx = len(packet["existing_claims"])
+            packet["existing_claims"].append(cid)
+            views.append(claim_view(nb, row, idx))
+        out.append(claim_view(nb, row, idx))
+    return out
+
+
+# ----------------------------------------------------------------------------- name index
+
+def name_index(nb: Any) -> Dict[str, List[str]]:
+    """Case-folded entity names and aliases -> canonical entity ids (exact matching only)."""
+    out: Dict[str, List[str]] = {}
+    for r in nb.store.q("SELECT entity_id, name FROM entities"):
+        out.setdefault(ids.fold(r[1]), []).append(r[0])
+    for r in nb.store.q("SELECT entity_id, alias_fold FROM entity_aliases"):
+        out.setdefault(r[1], []).append(r[0])
+    return out
+
+
+def entities_named_in(nb: Any, text: str) -> List[str]:
+    """Entities whose exact name or alias (case-folded, as whole words) occurs in the text."""
+    token_set = set(words(text))
+    if not token_set:
+        return []
+    folded = " " + " ".join(words(text)) + " "
+    found: List[str] = []
+    for name, eids in name_index(nb).items():
+        parts = words(name)
+        if not parts or any(p not in token_set for p in parts):
+            continue
+        if len(parts) > 1 and (" " + " ".join(parts) + " ") not in folded:
+            continue
+        for eid in eids:
+            c = nb.store.canonical(eid)
+            if c not in found:
+                found.append(c)
+    return found
+
+
+# ----------------------------------------------------------------------------- note_context
+
+def note_context(nb: Any, note_id: str, max_chars: Optional[int] = None) -> Dict[str, Any]:
+    row = nb.store.note_row(note_id)
+    if row is None:
+        raise ValueError("That note is not in this notebook.")
+    budget = int(max_chars) if max_chars else DEFAULT_SECTION_CHARS
+    atts = nb.store.attachments_of(note_id)
+    secs: List[Dict[str, Any]] = []
+    for s in sections(row["text"] or "", budget):
+        secs.append({"index": len(secs), "source": "note", "chars": s["chars"], "text": s["text"]})
+    full_text = [row["title"] or "", row["text"] or ""]
+    for a in atts:
+        path = nb.root / "extracted" / f"{a['sha256']}.txt"
+        if not (a["extracted_chars"] and path.exists()):
+            continue
+        body = path.read_text(encoding="utf-8")
+        full_text.append(body)
+        for s in sections(body, budget):
+            secs.append({"index": len(secs), "source": f"attachment:{a['filename'] or a['attachment_id']}",
+                         "chars": s["chars"], "text": s["text"]})
+    if not secs and row["title"]:
+        secs.append({"index": 0, "source": "title", "chars": len(row["title"]), "text": row["title"]})
+
+    source = json.loads(row["source_json"] or "{}")
+    about = json.loads(row["about_json"]) if row["about_json"] else None
+    note = {
+        "note_id": note_id, "title": row["title"], "kind": row["kind"], "occurred_at": row["occurred_at"],
+        "language": row["language"], "topics": nb.store.note_topics(note_id),
+        "source": {k: source[k] for k in ("client", "importer", "author") if source.get(k)},
+        "location": (json.loads(row["location_json"]).get("label") if row["location_json"] else None),
+        "attachments": [{"attachment_id": a["attachment_id"], "filename": a["filename"], "mime": a["mime"],
+                         "extracted_by": a["extracted_by"]} for a in atts],
+        "chars": sum(s["chars"] for s in secs), "about": about,
+    }
+
+    vector = nb.cache.get(row["vector_key"]) if row["vector_key"] else None
+    similar: List[Dict[str, Any]] = []
+    if vector is not None:
+        for nid, score in nb.note_vectors.nearest(vector, SIMILAR_NOTES * 3, 0.2, exclude=[note_id],
+                                                  notes_only=True):
+            n = nb.store.note_row(nid)
+            if n["archived_at"] or len(similar) >= SIMILAR_NOTES:
+                continue
+            similar.append({"note_id": nid, "title": n["title"], "occurred_at": n["occurred_at"],
+                            "summary": excerpt(n["summary"] or n["one_line"] or n["text"], 200),
+                            "score": round(score, 4)})
+
+    packet = empty_packet()
+    ordered: List[tuple] = []
+    if about and about["kind"] == "entity" and nb.store.entity_row(about["id"]) is not None:
+        ordered.append((nb.store.canonical(about["id"]), "about"))
+    for eid in entities_named_in(nb, "\n".join(full_text)):
+        ordered.append((eid, "name_in_text"))
+    if vector is not None:
+        rows = nb.store.q("SELECT e.entity_id, v.vector FROM entities e JOIN vectors v ON v.vector_key=e.vector_key"
+                          " WHERE e.merged_into IS NULL")
+        for eid, _score in top_k(vector, [(r[0], r[1]) for r in rows], MAX_CANDIDATE_ENTITIES, ENTITY_VECTOR_MIN):
+            ordered.append((eid, "vector"))
+    candidates: List[Dict[str, Any]] = []
+    claim_views: List[Dict[str, Any]] = []
+    for eid, how in ordered:
+        if eid in packet["candidate_entities"] or len(candidates) >= MAX_CANDIDATE_ENTITIES:
+            continue
+        packet["candidate_entities"].append(eid)
+        view = entity_view(nb, eid)
+        view.update({"index": len(candidates), "matched_by": how,
+                     "active_claims": attach_claims(nb, packet, eid, claim_views)})
+        candidates.append(view)
+
+    task_rows: List[Any] = []
+    if about and about["kind"] == "task":
+        t = nb.store.q1("SELECT * FROM tasks WHERE task_id=? AND status='open'", (about["id"],))
+        if t is not None:
+            task_rows.append(t)
+    if packet["candidate_entities"]:
+        members: List[str] = []
+        for eid in packet["candidate_entities"]:
+            members.extend(nb.store.merged_members(eid))
+        q = ",".join("?" for _ in members)
+        task_rows.extend(nb.store.q(
+            f"SELECT * FROM tasks WHERE status='open' AND note_id IN (SELECT note_id FROM entity_notes WHERE entity_id"
+            f" IN ({q})) ORDER BY opened_at DESC LIMIT ?", [*members, MAX_OPEN_TASKS]))
+    task_rows.extend(nb.store.q("SELECT * FROM tasks WHERE status='open' ORDER BY opened_at DESC LIMIT ?",
+                                (MAX_OPEN_TASKS,)))
+    open_tasks: List[Dict[str, Any]] = []
+    for t in task_rows:
+        if t["task_id"] in packet["open_tasks"] or len(open_tasks) >= MAX_OPEN_TASKS:
+            continue
+        packet["open_tasks"].append(t["task_id"])
+        open_tasks.append({"index": len(open_tasks), "task_id": t["task_id"], "text": t["text"],
+                           "opened_at": t["opened_at"], "due": t["due"], "note_id": t["note_id"]})
+    save_packet(nb, note_id, packet)
+    return {"note": note, "sections": secs, "similar_notes": similar, "candidate_entities": candidates,
+            "existing_claims": claim_views, "open_tasks": open_tasks, "topics": nb.store.topic_names(),
+            "vocabulary": vocabulary()}

abstractnote/process/merge.py

@@ -0,0 +1,60 @@
+"""Merge partial Extractions of a multi-section note before linking.
+
+Mentions are unioned by exact case-folded name and kind (refs renumbered m1,
+m2, ...; aliases unioned); claims, tasks and time references are concatenated
+with their refs remapped; topics are unioned; summaries are joined. This is
+the same rule as the flow's merge step.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Sequence, Tuple
+
+
+def merge_extractions(parts: Sequence[Dict[str, Any]]) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+    """Returns ``(merged extraction, mention list for note_resolve_candidates)``."""
+    parts = [p for p in (parts or []) if isinstance(p, dict)]
+    if not parts:
+        raise ValueError("There is no extraction to merge.")
+    if len(parts) == 1:
+        merged = dict(parts[0])
+    else:
+        mentions: List[Dict[str, Any]] = []
+        refmap: Dict[Tuple[int, Any], str] = {}
+        by_key: Dict[Tuple[str, str], Dict[str, Any]] = {}
+        claims: List[Dict[str, Any]] = []
+        tasks: List[Dict[str, Any]] = []
+        time_refs: List[Dict[str, Any]] = []
+        topics: List[str] = []
+        summaries: List[str] = []
+        for i, part in enumerate(parts):
+            for m in part.get("mentions") or []:
+                key = (str(m.get("name") or "").casefold(), str(m.get("kind") or ""))
+                if key in by_key:
+                    kept = by_key[key]
+                    for a in m.get("aliases") or []:
+                        if a not in kept["aliases"]:
+                            kept["aliases"].append(a)
+                else:
+                    kept = dict(m, ref="m" + str(len(mentions) + 1), aliases=list(m.get("aliases") or []))
+                    by_key[key] = kept
+                    mentions.append(kept)
+                refmap[(i, m.get("ref"))] = kept["ref"]
+            for c in part.get("claims") or []:
+                c2 = dict(c)
+                c2["subject"] = refmap.get((i, c.get("subject")), c.get("subject"))
+                if not c.get("object_is_literal"):
+                    c2["object"] = refmap.get((i, c.get("object")), c.get("object"))
+                claims.append(c2)
+            tasks.extend(part.get("tasks") or [])
+            time_refs.extend(part.get("time_refs") or [])
+            for t in part.get("topics") or []:
+                if t not in topics:
+                    topics.append(t)
+            if part.get("summary"):
+                summaries.append(str(part["summary"]))
+        merged = {"summary": " ".join(summaries), "mentions": mentions, "claims": claims, "tasks": tasks,
+                  "time_refs": time_refs, "topics": topics}
+    names = [{"ref": m.get("ref"), "name": m.get("name"), "kind": m.get("kind"), "aliases": m.get("aliases") or []}
+             for m in merged.get("mentions") or []]
+    return merged, names

abstractnote/process/prompts/__init__.py

@@ -0,0 +1,39 @@
+"""Prompt templates (Markdown with ``{{var}}`` placeholders) and their tiny renderer."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any, Dict
+
+_DIR = Path(__file__).resolve().parent
+_VAR = re.compile(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}")
+
+NAMES = ("quick_filing", "extract", "link", "synthesize", "mine_plan", "mine_findings", "mine_dossier", "reflect")
+
+
+def load(name: str) -> str:
+    if name not in NAMES:
+        raise ValueError(f"There is no prompt named {name!r}.")
+    return (_DIR / f"{name}.md").read_text(encoding="utf-8")
+
+
+def variables(template: str) -> set:
+    return set(_VAR.findall(template))
+
+
+def _text(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, ensure_ascii=False, indent=1)
+
+
+def render(template: str, values: Dict[str, Any]) -> str:
+    """Replace ``{{var}}`` with the value (JSON for non-strings); a missing variable is an error."""
+    missing = variables(template) - set(values)
+    if missing:
+        raise ValueError(f"The prompt needs values for: {', '.join(sorted(missing))}.")
+    return _VAR.sub(lambda m: _text(values[m.group(1)]), template)

abstractnote/process/prompts/extract.md

@@ -0,0 +1,30 @@
+You read one section of a person's note and write down what it says, so it can be linked to what they already know.
+
+The note: {{note}}
+
+Section to read:
+{{section}}
+
+What the notebook already knows (for orientation only; do not copy it into the output):
+- similar notes: {{similar_notes}}
+- entities that may be mentioned: {{candidate_entities}}
+- open tasks: {{open_tasks}}
+- existing topics: {{topics}}
+
+Predicate vocabulary (use one of these CURIEs when it fits; otherwise write `ex:snake_case`):
+{{vocabulary}}
+
+Return:
+- `summary`: one to three sentences about this section;
+- `mentions`: every person, organisation, project, concept, place, event, artifact (document, dataset, repository), piece of software or other thing the section talks about, each with a `ref` (m1, m2, ...), the `name` exactly as written, its `kind`, other names used for it in the note (`aliases`), and its `role` in one phrase;
+- `claims`: one per statement, with `subject` a mention ref, a `predicate`, and an `object` that is a mention ref, or a literal value with `object_is_literal: true`; `certainty` (stated, probable, possible, reported, disputed), `stance` (assert; revise when the note corrects earlier knowledge; retract when it says something is no longer true), `evidence` (a verbatim excerpt), and `valid_from` / `valid_until` only when the note gives or implies those dates from its own date;
+  Quantities, amounts, dates, deadlines and decisions are facts about their subject: write them as literal-object claims with an explicit `ex:` predicate rather than forcing a vocabulary predicate that does not mean the same thing. For example, "We capped the LUT budget at 25k; Dana needs the Pareto by 24 October" gives `{"subject": "m1", "predicate": "ex:budget_cap", "object": "25k", "object_is_literal": true}`, `{"subject": "m1", "predicate": "ex:deadline", "object": "2026-10-24", "object_is_literal": true}` and `{"subject": "m1", "predicate": "ex:decided", "object": "cap the budget at 25k", "object_is_literal": true}` (with m1 = LUT), not `schema:nextItem` or `schema:knowsAbout`;
+- `tasks`: things to do (`open`) and things reported done (`done`), with a due date when one is given;
+- `time_refs`: explicit time references ("last Tuesday", "in 2024") with their ISO date when the note's own date resolves them;
+- `topics`: the topics the section belongs to, preferring existing ones.
+
+Rules:
+- Use only the note and the provided context; never invent entities or dates.
+- Keep names as written by the author; put variants in aliases.
+- Quote evidence verbatim; one claim per statement; no inferred hierarchy.
+- Output JSON only, matching the schema.