primarysource 0.1.0__py3-none-any.whl

primarysource/rfc/collection.py

@@ -0,0 +1,19 @@
+"""The RFC series as a primarysource collection. Importing this needs nothing beyond the core."""
+
+from __future__ import annotations
+
+from primarysource.model import Unit
+from primarysource.refs import RefGrammar
+from primarysource.registry import Collection
+
+# Tool names stay bare (search, read, cite, ...): rfclib's published interface. A prefix is the owner's call
+# (PLAN.md §7, "Tool prefix").
+COLLECTION = Collection(
+    name="rfc",
+    title="RFC series (RFC, BCP, STD, FYI) from the RFC Editor",
+    prefix="",
+    unit=Unit.PARAGRAPH,
+    grammar=RefGrammar(r"(?:RFC|BCP|STD|FYI)\d+"),
+    server="primarysource.rfc.mcp.server:mcp",
+    cli="primarysource.rfc.cli:main",
+)

primarysource/rfc/config.py

@@ -0,0 +1,138 @@
+"""Paths, network policy and official source constants.
+
+All URLs point at RFC Editor first-party infrastructure (rfc-editor.org),
+the authoritative publisher/archive of the RFC Series for all five streams.
+See RFC 9280 (RFC Editor Model v3), RFC 4714, RFC 7841.
+
+Where the library lives: $RFCLIB_HOME if set, else the platform's per-user
+data directory (platformdirs: %LOCALAPPDATA%\\rfclib on Windows, not
+roaming, since the library is gigabytes; $XDG_DATA_HOME/rfclib or
+~/.local/share/rfclib on Linux; ~/Library/Application Support/rfclib on
+macOS). Never the working directory: MCP clients start servers from
+wherever they like, so a cwd-relative home would silently point every
+session at a different, empty library.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from primarysource.errors import LibError
+
+from . import __version__
+
+BASE_URL = "https://www.rfc-editor.org"
+MANIFEST_URL = f"{BASE_URL}/rfc-index.xml"
+MANIFEST_XSD_URL = f"{BASE_URL}/rfc-index.xsd"
+RSS_URL = f"{BASE_URL}/rfcrss.xml"
+ATOM_URL = f"{BASE_URL}/rfcatom.xml"
+RSYNC_HOST = "rsync.rfc-editor.org"
+RSYNC_DOC_URL = f"{BASE_URL}/series/rfc-download/"
+
+USER_AGENT = f"rfclib/{__version__} (local RFC library sync; polite bulk fetch)"
+
+NETWORK_MODES = ("auto", "on-demand", "offline")
+
+
+def _default_home() -> Path:
+    # RFCLIB_HOME, else $PRIMARYSOURCE_HOME/rfc, else an existing per-user
+    # rfclib library, else the per-user primarysource/rfc
+    # (primarysource.home.collection_home).
+    from primarysource.home import collection_home
+    return collection_home("rfc", legacy_app="rfclib").expanduser()
+
+
+def legacy_home() -> Path | None:
+    """A pre-platformdirs library in ./data (the old default), if one is
+    sitting in the working directory and RFCLIB_HOME is not set: the CLI
+    points at it instead of silently starting an empty library."""
+    if os.environ.get("RFCLIB_HOME"):
+        return None
+    old = Path.cwd() / "data"
+    if (old / "rfclib.db").is_file() and old.resolve() != \
+            _default_home().resolve():
+        return old
+    return None
+
+
+class NetworkDisabled(LibError, RuntimeError):
+    """A fetch was needed but RFCLIB_NETWORK=offline (or read-only mode)
+    forbids it. `action` says what a human runs to get the document in."""
+
+
+def read_only() -> bool:
+    """RFCLIB_READONLY=1 (or `rfclib mcp --read-only`): this process must not
+    write the library. The MCP server then opens the database read-only and
+    never fetches, so a shared HTTP server stays the only writer."""
+    return any(
+        (os.environ.get(var) or "").strip().lower() in ("1", "true", "yes", "on")
+        for var in ("RFCLIB_READONLY", "PRIMARYSOURCE_READONLY"))
+
+
+def network_mode() -> str:
+    """RFCLIB_NETWORK: auto (default: fetch documents on demand, refresh the
+    catalog when stale) | on-demand (fetch only when a document is asked
+    for) | offline (never touch the network; answer from the cache).
+    A read-only process is always offline: a fetch would have to be stored."""
+    if read_only():
+        return "offline"
+    mode = (os.environ.get("RFCLIB_NETWORK") or "auto").strip().lower()
+    return mode if mode in NETWORK_MODES else "auto"
+
+
+def require_network(what: str) -> None:
+    if read_only():
+        raise NetworkDisabled(
+            f"{what} is not in this library and this server is read-only "
+            "(RFCLIB_READONLY)", "ask the shared rfclib server for it",
+            "rfclib sync (without RFCLIB_READONLY)")
+    if network_mode() == "offline":
+        raise NetworkDisabled(
+            f"{what} is not available locally and RFCLIB_NETWORK=offline "
+            "forbids fetching it", "",
+            "rfclib sync while online, or unset RFCLIB_NETWORK")
+
+
+@dataclass(frozen=True)
+class Paths:
+    """Layout of the local library. Override the root with RFCLIB_HOME.
+
+    home/
+      rfclib.db        SQLite index (metadata, relations, file inventory)
+      meta/            rfc-index.xml manifest copy, subject page cache
+      mirror/          document files, rfc-editor layout (rfc/, bcp/, ...)
+      models/          embedding model, loaded offline
+    """
+
+    home: Path
+
+    @classmethod
+    def default(cls) -> "Paths":
+        return cls(_default_home())
+
+    @property
+    def db(self) -> Path:
+        return self.home / "rfclib.db"
+
+    @property
+    def meta(self) -> Path:
+        return self.home / "meta"
+
+    @property
+    def manifest_xml(self) -> Path:
+        return self.meta / "rfc-index.xml"
+
+    @property
+    def mirror(self) -> Path:
+        return self.home / "mirror"
+
+    @property
+    def models(self) -> Path:
+        return self.home / "models"
+
+    def ensure(self) -> "Paths":
+        for p in (self.home, self.meta, self.mirror):
+            p.mkdir(parents=True, exist_ok=True)
+        return self

primarysource/rfc/errata.py

@@ -0,0 +1,203 @@
+"""Verified-errata overlay from the RFC Editor errata service.
+
+Two sources. The primary one is the RFC Editor's published bulk API,
+www.rfc-editor.org/api/v1/errata.json (every erratum, ~12 MB, one request;
+robots.txt allows /api/): `sync_bulk` loads it so every document's errata
+status is authoritative offline, including "none". The fallback is the
+per-document errata.rfc-editor.org search page, fetched on demand only (that
+host forbids crawling), parsed from its records presentation.
+"""
+
+from __future__ import annotations
+
+import html as htmllib
+import re
+import sqlite3
+import time
+from datetime import datetime, timezone
+
+from .config import USER_AGENT, network_mode, require_network
+from .sync.state import _now, get_meta, set_meta
+
+ERRATA_URL = "https://errata.rfc-editor.org/search/?rfc_number={num}"
+BULK_URL = "https://www.rfc-editor.org/api/v1/errata.json"
+BULK_META = "errata_bulk_synced_at"
+
+_RECORD_SPLIT = re.compile(r'Errata-ID:\s*<a href="/eid(\d+)/?"')
+_TAG_RE = re.compile(r"<[^>]+>")
+
+
+def _text(fragment: str) -> str:
+    return htmllib.unescape(_TAG_RE.sub("", fragment)).strip()
+
+
+def _field(block: str, label: str) -> str | None:
+    """Value of a <dt>label</dt><dd>value</dd> pair inside the metadata dl."""
+    m = re.search(
+        rf"<dt[^>]*>\s*{re.escape(label)}\s*</dt>\s*<dd[^>]*>(.*?)</dd>",
+        block, re.S)
+    return _text(m.group(1)) if m else None
+
+
+def parse_errata(page: str) -> list[dict]:
+    """Parse the errata service HTML (Full records view) into dicts."""
+    parts = _RECORD_SPLIT.split(page)
+    # parts[0] = preamble; then alternating eid, block
+    out: list[dict] = []
+    for eid, block in zip(parts[1::2], parts[2::2]):
+        # Cut anything after the last record (page footer).
+        block = block.split("Report New Erratum")[0]
+        entry: dict = {"eid": int(eid),
+                       "status": _field(block, "Status:") or "",
+                       "type": _field(block, "Type:") or "",
+                       "reported_by": _field(block, "Reported By:"),
+                       "reported_date": _field(block, "Date Reported:"),
+                       "section": None, "says": None,
+                       "should_say": None, "notes": None,
+                       "handler": None, "handler_date": None}
+        if (v := _field(block, "Verified by:")):
+            entry["handler"] = v
+            entry["handler_date"] = _field(block, "Date Verified:")
+        elif (v := _field(block, "Rejected by:")):
+            entry["handler"] = v
+            entry["handler_date"] = _field(block, "Date Rejected:")
+
+        # "Section X says:" paragraph followed by a <pre> quote
+        m = re.search(r"<p>\s*(?:Section\s+([^<]*?))?\s*says:\s*</p>\s*"
+                      r"<pre[^>]*>(.*?)</pre>", block, re.S)
+        if m:
+            entry["section"] = (m.group(1) or "").strip().rstrip(":") or None
+            entry["says"] = _text(m.group(2))
+        m = re.search(r"<p>\s*It should say:\s*</p>\s*<pre[^>]*>(.*?)</pre>",
+                      block, re.S)
+        if m:
+            entry["should_say"] = _text(m.group(1))
+        m = re.search(r"<p>\s*Notes:\s*</p>\s*<pre[^>]*>(.*?)</pre>",
+                      block, re.S)
+        if m:
+            entry["notes"] = _text(m.group(1))
+        out.append(entry)
+    return out
+
+
+def fetch(rfc_number: int) -> str:
+    """One polite request to the errata service; caller throttles."""
+    import httpx  # lazy: serving from an existing library never needs it
+    with httpx.Client(headers={"User-Agent": USER_AGENT},
+                      follow_redirects=True,
+                      timeout=httpx.Timeout(30.0)) as client:
+        r = client.get(ERRATA_URL.format(num=rfc_number))
+        r.raise_for_status()
+        return r.text
+
+
+def store(conn: sqlite3.Connection, doc_id: str,
+          entries: list[dict]) -> int:
+    conn.execute("DELETE FROM errata WHERE doc_id=?", (doc_id,))
+    rows = [(e["eid"], doc_id, e["status"], e.get("type"), e.get("section"),
+             e.get("says"), e.get("should_say"), e.get("notes"),
+             e.get("reported_by"), e.get("reported_date"),
+             e.get("handler"), e.get("handler_date"), _now())
+            for e in entries]
+    conn.executemany(
+        """INSERT OR REPLACE INTO errata(eid, doc_id, status, type, section,
+               says, should_say, notes, reported_by, reported_date,
+               handler, handler_date, fetched_at)
+           VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)""", rows)
+    conn.commit()
+    return len(rows)
+
+
+def sync_bulk(conn: sqlite3.Connection, data: list[dict] | None = None) -> dict:
+    """Replace the errata table with the RFC Editor's bulk feed."""
+    if data is None:
+        require_network("the errata feed")
+        import httpx  # lazy: serving from an existing library never needs it
+        with httpx.Client(headers={"User-Agent": USER_AGENT},
+                          follow_redirects=True,
+                          timeout=httpx.Timeout(120.0)) as client:
+            r = client.get(BULK_URL)
+            r.raise_for_status()
+            data = r.json()
+    now = _now()
+    rows = []
+    for e in data:
+        doc = (e.get("doc-id") or "").upper().replace(" ", "")
+        eid = str(e.get("errata_id") or "")
+        if not doc or not eid.isdigit():
+            continue
+        rows.append((int(eid), doc, e.get("errata_status_code") or "",
+                     e.get("errata_type_code"), e.get("section"),
+                     e.get("orig_text"), e.get("correct_text"),
+                     e.get("notes"), e.get("submitter_name"),
+                     e.get("submit_date"), e.get("verifier_name"),
+                     e.get("update_date"), now))
+    conn.execute("DELETE FROM errata")
+    conn.executemany(
+        """INSERT OR REPLACE INTO errata(eid, doc_id, status, type, section,
+               says, should_say, notes, reported_by, reported_date,
+               handler, handler_date, fetched_at)
+           VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)""", rows)
+    conn.commit()
+    set_meta(conn, BULK_META, now)
+    by = {}
+    for r in conn.execute("SELECT status, COUNT(*) c FROM errata "
+                          "GROUP BY status"):
+        by[r["status"]] = r["c"]
+    return {"errata": len(rows), "documents": conn.execute(
+        "SELECT COUNT(DISTINCT doc_id) FROM errata").fetchone()[0],
+            "by_status": by, "synced_at": now}
+
+
+def bulk_synced_at(conn: sqlite3.Connection) -> str | None:
+    return get_meta(conn, BULK_META)
+
+
+def cached(conn: sqlite3.Connection, doc_id: str,
+           status: str | None = None) -> list[dict]:
+    q = "SELECT * FROM errata WHERE doc_id=?"
+    args: list = [doc_id]
+    if status:
+        q += " AND status=?"
+        args.append(status)
+    q += " ORDER BY eid"
+    return [dict(r) for r in conn.execute(q, args)]
+
+
+def refreshed_at(conn: sqlite3.Connection, doc_id: str) -> str | None:
+    row = conn.execute(
+        "SELECT MIN(fetched_at) t FROM errata WHERE doc_id=?",
+        (doc_id,)).fetchone()
+    return row["t"] if row and row["t"] else None
+
+
+def ensure(conn: sqlite3.Connection, doc_id: str, refresh: bool = False,
+           max_age_days: float = 7.0) -> dict:
+    """Cached errata for doc_id, fetching from the service when absent
+    (or when refresh=True / cache older than max_age_days)."""
+    if not doc_id.startswith("RFC"):
+        return {"doc_id": doc_id, "results": cached(conn, doc_id),
+                "source": "cache",
+                "note": "errata exist for RFC-series documents only"}
+    # a bulk sync is authoritative for every document, including "none"
+    fetched_at = refreshed_at(conn, doc_id) or bulk_synced_at(conn)
+    fresh = False
+    if fetched_at:
+        try:
+            t = datetime.strptime(fetched_at, "%Y-%m-%dT%H:%M:%SZ")\
+                 .replace(tzinfo=timezone.utc)
+            fresh = ((datetime.now(timezone.utc) - t).total_seconds()
+                     < max_age_days * 86400)
+        except ValueError:
+            fresh = False
+    if fresh and not refresh:
+        return {"doc_id": doc_id, "results": cached(conn, doc_id),
+                "source": "cache", "fetched_at": fetched_at}
+    if network_mode() == "offline":
+        return {"doc_id": doc_id, "results": cached(conn, doc_id),
+                "source": "cache (offline)", "fetched_at": fetched_at}
+    time.sleep(0.25)  # politeness floor on the upstream service
+    page = fetch(int(doc_id[3:]))
+    store(conn, doc_id, parse_errata(page))
+    return {"doc_id": doc_id, "results": cached(conn, doc_id),
+            "source": "live", "fetched_at": refreshed_at(conn, doc_id)}

primarysource/rfc/fsutil.py

@@ -0,0 +1,82 @@
+"""Filesystem helpers that behave the same on Windows, macOS and Linux.
+
+atomic_write_bytes: write to a temp file in the destination directory, then
+os.replace() it into place, so a reader (another MCP server process, the
+reader UI) never sees a half-written RFC and a crash leaves no torn file.
+On Windows os.replace fails while another process holds the target open
+(antivirus, search indexers, a concurrent reader), so it retries briefly.
+
+bulk_lock: one long-running writer at a time per library (init, sync,
+index build). Short writes such as on-demand indexing of a single RFC rely
+on SQLite's own locking (busy_timeout) instead.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import time
+from pathlib import Path
+
+STALE_LOCK_SECONDS = 6 * 3600
+
+
+def atomic_write_bytes(dest: Path, data: bytes, retries: int = 10) -> None:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    tmp = dest.with_name(f".{dest.name}.{os.getpid()}.tmp")
+    with open(tmp, "wb") as fh:
+        fh.write(data)
+        fh.flush()
+        os.fsync(fh.fileno())
+    for attempt in range(retries):
+        try:
+            os.replace(tmp, dest)
+            return
+        except PermissionError:
+            if attempt == retries - 1:
+                with contextlib.suppress(OSError):
+                    tmp.unlink()
+                raise
+            time.sleep(0.05 * (attempt + 1))
+
+
+class LockHeld(RuntimeError):
+    pass
+
+
+@contextlib.contextmanager
+def bulk_lock(home: Path, name: str = "write"):
+    """Exclusive lock file <home>/.<name>.lock (O_EXCL create, works on every
+    OS and on network filesystems). A lock older than STALE_LOCK_SECONDS is
+    taken over: its holder crashed."""
+    home.mkdir(parents=True, exist_ok=True)
+    lock = home / f".{name}.lock"
+    for _ in range(2):
+        try:
+            fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+            break
+        except FileExistsError:
+            try:
+                age = time.time() - lock.stat().st_mtime
+            except OSError:
+                continue
+            if age > STALE_LOCK_SECONDS:
+                with contextlib.suppress(OSError):
+                    lock.unlink()
+                continue
+            holder = ""
+            with contextlib.suppress(OSError):
+                holder = lock.read_text("utf-8").strip()
+            raise LockHeld(f"another rfclib process is updating this library "
+                           f"({holder or lock}); wait for it to finish") \
+                from None
+    else:
+        raise LockHeld(f"could not take {lock}")
+    try:
+        os.write(fd, f"pid {os.getpid()} since {time.strftime('%H:%M:%S')}"
+                 .encode())
+        os.close(fd)
+        yield lock
+    finally:
+        with contextlib.suppress(OSError):
+            lock.unlink()

primarysource/rfc/graph.py

@@ -0,0 +1,213 @@
+"""Cross-reference graph mined from the local RFC .txt corpus.
+
+RFCs cite each other with bracketed tags ([RFC9000]) inside numbered
+References sections (normative vs informative by subsection), and point
+at specific sections with "Section 7.4.1 of [RFC9110]" / "[RFC9110],
+Section 4.2" phrasing. Mining those patterns from text we already
+store yields the full document-level citation graph plus section-level
+anchors, with zero extra downloads (no XML mirror needed).
+
+Build is incremental: ref_src keeps each source txt's sha256 so
+re-running only re-mines changed documents.
+"""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+from pathlib import Path
+
+from .sync.downloader import sha256_file
+from .sync.state import file_path, mirror_of
+
+_SERIES = r"(?:RFC|BCP|STD|FYI)"
+
+# [RFC9000], [BCP 14], [STD 1] ... (spaced or not)
+_REF_TAG = re.compile(rf"\[({_SERIES})\s?(\d{{1,5}})\]")
+
+# Body header of the references area, e.g. "19.  References",
+# "     19.1.  Normative References". TOC lines are excluded because
+# they end with dot-leaders + page number, not with "References".
+_HDR_REF = re.compile(
+    r"^\s*(\d+(?:\.\d+)*)?\.?\s{1,3}(Normative|Informative)?\s*"
+    r"References:?\s*$")
+
+# Any numbered body header (used to leave the references area).
+_HDR_NUM = re.compile(r"^\s*(\d+(?:\.\d+)*)\.?\s{1,3}[A-Z]")
+
+# Section-anchored citations, the common phrasings:
+#   "Section 7.4.1 of [RFC9110]"        "Sections 4.1 ... of RFC 9001"
+#   "[RFC9110], Section 4.2"            "[RFC9110] Section 4.2"
+#   "RFC 9001, Section 5.4"             "Appendix C of [RFC6962]"
+_SEC_OF = re.compile(
+    rf"\b[Ss]ections?\s+((?:\d+|[A-Z])[\w.\-]{{0,12}})\s+of\s+\[?\s*"
+    rf"({_SERIES})\s?(\d{{1,5}})\s*\]?")
+_SEC_TAG = re.compile(
+    rf"\[({_SERIES})\s?(\d{{1,5}})\](?:,|\s+)?[Ss]ections?\s+"
+    r"((?:\d+|[A-Z])[\w.\-]{0,12})")
+_SEC_PLAIN = re.compile(
+    rf"\b({_SERIES})\s?(\d{{1,5}})\s*[,\u2014]+\s+[Ss]ections?\s+"
+    r"((?:\d+|[A-Z])[\w.\-]{0,12})")
+_SEC_APPX = re.compile(
+    rf"\bAppendix\s+([A-Z](?:\.\d+)?)\s+of\s+\[?\s*({_SERIES})\s?"
+    r"(\d{1,5})\s*\]?")
+
+
+def _clean_sec(sec: str) -> str:
+    return sec.strip().rstrip(".,:;")[:16]
+
+
+def extract_refs(text: str, known: set[str]) -> list[tuple[str, str, str]]:
+    """Mine (dst, kind, sec) edges from one RFC's txt.
+
+    kind is normative|informative|reference for tags inside the
+    References section, mention for section-anchored citations found
+    anywhere in the document. dst is only emitted when the target id
+    exists in `known`.
+    """
+    edges: set[tuple[str, str, str]] = set()
+    zone: str | None = None         # None|reference|normative|informative
+    zone_prefix: str | None = None  # e.g. '19' while inside section 19(.x)
+
+    for line in text.splitlines():
+        m = _HDR_REF.match(line)
+        if m:
+            num, sub = m.group(1), m.group(2)
+            zone = ("normative" if sub == "Normative" else
+                    "informative" if sub == "Informative" else "reference")
+            zone_prefix = num
+            continue
+        h = _HDR_NUM.match(line)
+        if h and zone is not None:
+            num = h.group(1)
+            if not (zone_prefix and (num == zone_prefix
+                                     or num.startswith(zone_prefix + "."))):
+                zone = None         # left the references section
+            continue
+
+        # TOC lines (dot leaders) mirror headings + can contain tags;
+        # they never carry real citations, skip them entirely.
+        if ". ." in line:
+            continue
+
+        if zone is not None:
+            for series, num in _REF_TAG.findall(line):
+                dst = series + num
+                if dst in known:
+                    edges.add((dst, zone, ""))
+
+        for sec, series, num in _SEC_OF.findall(line):
+            dst = series + num
+            if dst in known:
+                edges.add((dst, "mention", _clean_sec(sec)))
+        for sec, series, num in _SEC_APPX.findall(line):
+            dst = series + num
+            if dst in known:
+                edges.add((dst, "mention", _clean_sec(sec)))
+        for series, num, sec in _SEC_TAG.findall(line):
+            dst = series + num
+            if dst in known:
+                edges.add((dst, "mention", _clean_sec(sec)))
+        for series, num, sec in _SEC_PLAIN.findall(line):
+            dst = series + num
+            if dst in known:
+                edges.add((dst, "mention", _clean_sec(sec)))
+
+    return sorted(edges)
+
+
+def build(conn: sqlite3.Connection, doc_ids: list[str] | None = None,
+          force: bool = False, progress=None) -> dict:
+    """(Re-)mine ref edges for docs with a local txt; sha-incremental."""
+    known = {r[0] for r in conn.execute("SELECT doc_id FROM docs")}
+    src_sha = {r["doc_id"]: r["sha256"] for r in
+               conn.execute("SELECT doc_id, sha256 FROM ref_src")}
+    rows = conn.execute("SELECT doc_id, path FROM files WHERE fmt='txt'")\
+             .fetchall()
+    if doc_ids:
+        want = set(doc_ids)
+        rows = [r for r in rows if r["doc_id"] in want]
+    built = skipped = total_edges = 0
+    mirror = mirror_of(conn)
+    for i, r in enumerate(rows, 1):
+        path = file_path(conn, r["path"], mirror)
+        if not path.is_file():
+            continue
+        sha = sha256_file(path)
+        if not force and src_sha.get(r["doc_id"]) == sha:
+            skipped += 1
+            continue
+        text = path.read_text(encoding="utf-8", errors="replace")
+        edges = extract_refs(text, known)
+        conn.execute("DELETE FROM ref_edges WHERE src=?", (r["doc_id"],))
+        conn.executemany(
+            "INSERT OR IGNORE INTO ref_edges(src,dst,kind,sec) VALUES(?,?,?,?)",
+            [(r["doc_id"], d, k, s) for d, k, s in edges])
+        conn.execute(
+            "INSERT INTO ref_src(doc_id,sha256) VALUES(?,?) "
+            "ON CONFLICT(doc_id) DO UPDATE SET sha256=excluded.sha256",
+            (r["doc_id"], sha))
+        built += 1
+        total_edges += len(edges)
+        if progress:
+            progress(i, len(rows), r["doc_id"], len(edges))
+        if i % 250 == 0:
+            conn.commit()
+    conn.commit()
+    return {"built": built, "skipped_unchanged": skipped,
+            "new_edges": total_edges,
+            "total_edges": conn.execute(
+                "SELECT COUNT(*) FROM ref_edges").fetchone()[0]}
+
+
+_RANK = {"normative": 0, "informative": 1, "reference": 2, "mention": 3}
+
+
+def _num(doc_id: str) -> int:
+    m = re.match(r"\D*(\d+)", doc_id)
+    return int(m.group(1)) if m else 0
+
+
+def _sec_key(s: str):
+    return [int(p) if p.isdigit() else p
+            for p in re.split(r"[.\-]", s)]
+
+
+def get_refs(conn: sqlite3.Connection, doc_id: str,
+             limit: int = 100) -> dict:
+    """Merged citation view: outbound (this doc cites) and inbound."""
+    def side(col_own: str, col_peer: str) -> tuple[list[dict], int]:
+        agg: dict[str, dict] = {}
+        for r in conn.execute(
+                f"SELECT {col_peer} peer, kind, sec FROM ref_edges "
+                f"WHERE {col_own}=?", (doc_id,)):
+            e = agg.setdefault(r["peer"], {"doc_id": r["peer"],
+                                           "kinds": set(), "sections": set()})
+            e["kinds"].add(r["kind"])
+            if r["sec"]:
+                e["sections"].add(r["sec"])
+        titles = {}
+        if agg:
+            titles = {t["doc_id"]: (t["title"], bool(t["is_current"]))
+                      for t in conn.execute(
+                          "SELECT doc_id, title, is_current FROM docs "
+                          "WHERE doc_id IN (%s)"
+                          % ",".join("?" * len(agg)), list(agg))}
+        out = []
+        for peer, e in agg.items():
+            title, current = titles.get(peer, (None, True))
+            kinds = sorted(e["kinds"], key=lambda k: _RANK.get(k, 9))
+            out.append({"doc_id": peer, "title": title,
+                        "is_current": current, "kinds": kinds,
+                        "sections": sorted(e["sections"], key=_sec_key)[:12]})
+        out.sort(key=lambda x: (_RANK.get(x["kinds"][0], 9),
+                                -_num(x["doc_id"])))
+        return out[:limit], len(out)
+
+    outbound, n_out = side("src", "dst")
+    inbound, n_in = side("dst", "src")
+    return {"doc_id": doc_id,
+            "outbound": outbound, "outbound_count": n_out,
+            "inbound": inbound, "inbound_count": n_in,
+            "built": conn.execute(
+                "SELECT COUNT(*) FROM ref_src").fetchone()[0] > 0}

primarysource/rfc/index/__init__.py

@@ -0,0 +1 @@
+"""Chunking and retrieval subpackage: section chunker, FTS5, vectors."""