primarysource 0.1.0__py3-none-any.whl

primarysource/refs.py

@@ -0,0 +1,87 @@
+"""One ref grammar, parameterised by what a document id looks like.
+
+The three libraries that have a ref string agree on its shape and differ only in the document id:
+
+    RFC9000§9.6¶2      X.690§11.6¶2      devdocs_en_flask§blueprints/index#why¶3
+
+So the grammar is `<doc> § <section> [#anchor] [¶ n]`, or `<doc> : first-last` for a line range. `::` is the
+ASCII spelling of `§`. A library supplies the pattern for its document ids and, if its sections are not dotted
+numbers, the pattern for those.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Protocol
+
+from primarysource.errors import LibError
+
+DOTTED_SECTION = r"[0-9A-Za-z]+(?:\.[0-9A-Za-z]+)*"
+
+
+@dataclass(frozen=True, slots=True)
+class Ref:
+    """A parsed ref. Every part but one may be None; `lines` is (first, last), 1-based and inclusive."""
+
+    doc: str | None = None
+    section: str | None = None
+    para: int | None = None
+    anchor: str | None = None
+    lines: tuple[int, int] | None = None
+
+
+class RefParser(Protocol):
+    """What a collection's refs need: text to `Ref` and back. `RefGrammar` is one; a collection whose refs are not
+    `<doc>§<section>` (an id-keyed registry such as `usbids:1d6b:0002`) supplies its own object with these two
+    methods, raising `LibError` for text that is not a ref."""
+
+    def parse(self, text: str) -> Ref: ...
+
+    def format(self, ref: Ref) -> str: ...
+
+
+class RefGrammar:
+    """Parses and formats refs for one library."""
+
+    def __init__(self, doc_pattern: str, section_pattern: str = DOTTED_SECTION) -> None:
+        self._pattern = re.compile(
+            rf"\s*(?P<doc>{doc_pattern})?\s*"
+            rf"(?:(?:§|::)\s*(?P<section>{section_pattern}))?"
+            r"(?:#(?P<anchor>[^\s¶#]+))?"
+            r"(?:\s*¶\s*(?P<para>\d+))?"
+            r"(?::(?P<first>\d+)-(?P<last>\d+))?\s*"
+        )
+
+    def parse(self, text: str) -> Ref:
+        """Parse `text`, or raise LibError. A string that names nothing is refused, never guessed at."""
+        match = self._pattern.fullmatch(text)
+        if match is None or not any(match.group(name) for name in ("doc", "section", "anchor", "first")):
+            raise LibError(f"not a ref: {text!r}", "expected <doc>§<section>[#anchor][¶n] or <doc>:<first>-<last>")
+        lines: tuple[int, int] | None = None
+        if match.group("first"):
+            if match.group("section") or match.group("para") or match.group("anchor"):
+                raise LibError(f"not a ref: {text!r}", "a line range cannot be combined with a section")
+            lines = (int(match.group("first")), int(match.group("last")))
+            if lines[0] < 1 or lines[1] < lines[0]:
+                raise LibError(f"not a line range: {text!r}", "lines are 1-based and first <= last")
+        para = int(match.group("para")) if match.group("para") else None
+        if para is not None and para < 1:
+            raise LibError(f"not a ref: {text!r}", "¶ is 1-based")
+        if para is not None and not (match.group("section") or match.group("anchor")):
+            raise LibError(f"not a ref: {text!r}", "¶ needs a section or an anchor to count within")
+        return Ref(match.group("doc"), match.group("section"), para, match.group("anchor"), lines)
+
+    @staticmethod
+    def format(ref: Ref) -> str:
+        """The canonical spelling: no spaces, `§` for the section."""
+        out = ref.doc or ""
+        if ref.lines is not None:
+            return f"{out}:{ref.lines[0]}-{ref.lines[1]}"
+        if ref.section is not None:
+            out += f"§{ref.section}"
+        if ref.anchor is not None:
+            out += f"#{ref.anchor}"
+        if ref.para is not None:
+            out += f"¶{ref.para}"
+        return out

primarysource/registry.py

@@ -0,0 +1,161 @@
+"""The collection registry: how the core finds a library without importing it.
+
+A collection (`nist`, `itu`, `ieee`, ...) is one corpus with its own database, tool-name prefix, declared unit
+and ref grammar (PLAN.md §5.8). The core never imports a collection by name. It reads two sources of
+`name -> "module:attribute"` targets and imports a target only when that collection is asked for:
+
+- `BUILTIN`: the collections that ship in this package as subpackages, each behind its own extra;
+- the `primarysource.collections` entry-point group: a separately installed package that registers itself the
+  same way, for a collection with its own licence or release cadence.
+
+Either way the target is a module attribute holding a `Collection`. A collection whose extra is not installed
+fails with a `LibError` that names the install command, never with a bare `ImportError`.
+"""
+
+from __future__ import annotations
+
+import importlib
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from importlib.metadata import entry_points
+from typing import Any
+
+from primarysource.errors import LibError
+from primarysource.model import Unit
+from primarysource.refs import RefParser
+
+ENTRY_POINT_GROUP = "primarysource.collections"
+
+# name -> "module:attribute" or "module:attribute [extra]", the entry-point spelling. A collection is added here
+# when it moves in (PLAN.md §6). Without "[extra]" the extra is the collection's name; two collections that share
+# one extra (posix and ieee802 in `[ieee]`) both name it.
+BUILTIN: dict[str, str] = {
+    "rfc": "primarysource.rfc.collection:COLLECTION [rfc]",
+    "nist": "primarysource.nist.collection:COLLECTION",
+    "usb": "primarysource.usb.collection:COLLECTION",
+    "itu": "primarysource.itu.collection:COLLECTION",
+    "posix": "primarysource.ieee:POSIX [ieee]",
+    "ieee802": "primarysource.ieee:IEEE802 [ieee]",
+    "iana": "primarysource.iana:COLLECTION [iana]",
+    "cavp": "primarysource.cavp:COLLECTION",
+    "usbids": "primarysource.usbids:COLLECTION",
+    "draft": "primarysource.draft.collection:COLLECTION [draft]",
+    "c2sp": "primarysource.c2sp:C2SP",
+    "bbf": "primarysource.bbf:BBF [bbf]",
+    "ecma": "primarysource.ecma:ECMA [ecma]",
+    "gitdocs": "primarysource.gitdocs:COLLECTION",
+    "debpolicy": "primarysource.debpolicy:COLLECTION",
+    "rpmdocs": "primarysource.rpmdocs:COLLECTION",
+    "pms": "primarysource.pms:COLLECTION",
+    "pep": "primarysource.pep:COLLECTION",
+    "kconfig": "primarysource.kconfig:COLLECTION",
+    "iso": "primarysource.iso:ISO [iso]",
+    "zim": "primarysource.zim.collection:COLLECTION [zim]",
+    "cve": "primarysource.cve.collection:COLLECTION",
+}
+
+_NAME = re.compile(r"[a-z][a-z0-9]*")
+_SPEC = re.compile(r"\s*(?P<target>[\w.]+:[\w.]+)\s*(?:\[\s*(?P<extra>[\w-]+)\s*\])?\s*")
+
+
+def _split(spec: str) -> tuple[str, str | None]:
+    """`module:attribute [extra]` -> (`module:attribute`, `extra` or None)."""
+    match = _SPEC.fullmatch(spec)
+    if match is None:
+        raise LibError(f"malformed collection target {spec!r}", "expected 'module:attribute [extra]'")
+    return match["target"], match["extra"]
+
+
+@dataclass(frozen=True, slots=True)
+class Collection:
+    """What the core needs to know to serve a collection. Builders (index, readers, tools) are added as the core
+    grows; this is the part every collection has today.
+
+    `server` and `cli` are "module:attribute" targets, imported only when used, so a collection can be listed and
+    described without its extra installed: `server` names the `MCPServer` object, `cli` a `main(argv)` function
+    that runs the collection's own commands."""
+
+    name: str
+    title: str
+    prefix: str
+    unit: Unit
+    grammar: RefParser
+    server: str = ""
+    cli: str = ""
+    identify: str = ""
+    """A "module:function" taking an `identify.Probe` and returning `identify.Candidate`s: how this collection
+    recognises its own documents in a supplied file. Empty when it has none."""
+
+    def __post_init__(self) -> None:
+        if not _NAME.fullmatch(self.name):
+            raise ValueError(f"collection name {self.name!r} must be lowercase letters and digits")
+        if self.prefix not in ("", f"{self.name}_"):
+            raise ValueError(f"tool prefix for {self.name!r} must be {self.name + '_'!r} or empty")
+
+
+@dataclass(frozen=True, slots=True)
+class _Target:
+    spec: str
+    install: str
+    # the hint when the collection's MCP server is what is missing: its own extra plus primarysource[mcp]
+    server_install: str = ""
+
+
+def _targets() -> dict[str, _Target]:
+    """Every registered collection, without importing any of them. A built-in wins over a plugin of the same
+    name, so an installed package cannot shadow one shipped here."""
+    found: dict[str, _Target] = {}
+    for ep in entry_points(group=ENTRY_POINT_GROUP):
+        dist = ep.dist.name if ep.dist is not None else ep.name
+        target, extra = _split(ep.value)
+        package = f"{dist}[{extra}]" if extra else dist
+        found[ep.name] = _Target(target, f'pip install "{package}"', f'pip install "{package}" "primarysource[mcp]"')
+    for name, spec in BUILTIN.items():
+        target, extra = _split(spec)
+        found[name] = _Target(
+            target,
+            f'pip install "primarysource[{extra or name}]"',
+            f'pip install "primarysource[{extra or name},mcp]"',
+        )
+    return found
+
+
+def available() -> tuple[str, ...]:
+    """Names of every registered collection, sorted. Nothing is imported."""
+    return tuple(sorted(_targets()))
+
+
+def load(name: str, targets: Mapping[str, _Target] | None = None) -> Collection:
+    """Import and return one collection, or raise a `LibError` that says what to do."""
+    known = _targets() if targets is None else targets
+    target = known.get(name)
+    if target is None:
+        names = ", ".join(sorted(known)) or "none"
+        raise LibError(f"no collection named {name!r} (registered: {names})", "primarysource collections")
+    module_name, _, attribute = target.spec.partition(":")
+    try:
+        module = importlib.import_module(module_name)
+    except ModuleNotFoundError as exc:
+        raise LibError(f"collection {name!r} needs {exc.name!r}, which is not installed", target.install) from exc
+    collection = getattr(module, attribute, None)
+    if not isinstance(collection, Collection) or collection.name != name:
+        raise LibError(f"{target.spec} does not hold the {name!r} collection", "")
+    return collection
+
+
+def attribute(collection: Collection, spec: str, what: str) -> Any:
+    """Import a collection's `server` or `cli` target, mapping a missing dependency to the install hint."""
+    if not spec:
+        raise LibError(f"collection {collection.name!r} has no {what}", "")
+    module_name, _, name = spec.partition(":")
+    try:
+        module = importlib.import_module(module_name)
+    except ModuleNotFoundError as exc:
+        target = _targets().get(collection.name)
+        if target is None:
+            hint = f'pip install "primarysource[{collection.name}]"'
+        else:
+            hint = target.server_install if what == "server" and target.server_install else target.install
+        raise LibError(f"the {collection.name!r} {what} needs {exc.name!r}, which is not installed", hint) from exc
+    return getattr(module, name)

primarysource/render.py

@@ -0,0 +1,49 @@
+"""The text shape of a tool result: body first, then footer lines in one fixed order.
+
+Decision recorded in PLAN.md: results are text, because the passage dominates the tokens and a JSON string
+escapes every newline in it. What a JSON result carried as fields is carried as `key: value` footer lines, in
+the order below, so a reader (or a regex) finds them in the same place in every library.
+
+The `next:` value is exactly what to pass back (PLAN.md §5.11): a ref string for a read, which `read` accepts as
+its `ref`, and `start=<n>` for a list (search, outline, browse, grep), a 0-based offset for the same tool's
+`start` argument. `next_start` and `parse_next_start` are the two halves of the list cursor.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+
+FOOTER_ORDER = ("cite", "unit", "span", "next", "relaxed", "confidence", "partial", "warning")
+
+_NEXT_START = re.compile(r"start=(0|[1-9][0-9]*)")
+
+
+def render(body: str, footer: Mapping[str, str]) -> str:
+    """Body, a blank line, then the known footer keys in FOOTER_ORDER. Unknown keys are a bug, not output."""
+    unknown = sorted(set(footer) - set(FOOTER_ORDER))
+    if unknown:
+        raise ValueError(f"unknown footer keys: {', '.join(unknown)}")
+    lines = [f"{key}: {footer[key]}" for key in FOOTER_ORDER if footer.get(key)]
+    for line in lines:
+        if "\n" in line:
+            raise ValueError("a footer value must be one line")
+    return body.rstrip("\n") + ("\n\n" + "\n".join(lines) if lines else "") + "\n"
+
+
+def next_start(offset: int | None) -> str:
+    """The `next:` value of a list page that continues at the 0-based `offset`: `start=<offset>`. None (the last
+    page) gives "", which `render` drops, so a footer with no `next:` line means there is nothing more."""
+    if offset is None:
+        return ""
+    if offset < 0:
+        raise ValueError("a list cursor is a 0-based offset")
+    return f"start={offset}"
+
+
+def parse_next_start(value: str) -> int:
+    """The offset in a list `next:` value, the inverse of `next_start`. Anything else is a bug in the caller."""
+    match = _NEXT_START.fullmatch(value.strip())
+    if match is None:
+        raise ValueError(f"not a list cursor: {value!r} (expected 'start=<n>')")
+    return int(match[1])

primarysource/rfc/__init__.py

@@ -0,0 +1,3 @@
+"""rfclib: local RFC library with official RFC Editor sync and MCP interface."""
+
+__version__ = "0.2.0"

primarysource/rfc/__main__.py

@@ -0,0 +1,6 @@
+"""Allow `python -m primarysource.rfc ...` as an alias for the `rfclib` console script."""
+
+from .cli import main
+
+if __name__ == "__main__":
+    main()

primarysource/rfc/cite.py

@@ -0,0 +1,204 @@
+"""Citation primitives: reference the spec, do not restate it.
+
+Why this module exists: prose written *about* an RFC inside a library is a
+second, uncheckable copy of the spec. It drifts when the RFC is updated, it
+drifts when someone misremembers it, and nothing ever notices. A citation —
+document, section, canonical URL, hash of the exact span — stays checkable
+forever, and check() re-verifies it in a lookup.
+
+So the policy this pushes toward:
+  * cite the section for a rule (`RFC 2658 section 3.2`), never a summary of it;
+  * quote verbatim (quote()) only when the exact wording is load-bearing, and
+    carry the span hash so an upstream change becomes detectable;
+  * do not paraphrase normative text (MUST/SHOULD sentences, ABNF, wire
+    formats) into comments, docstrings or identifiers. That is the copy that
+    rots silently, so lint() flags near-verbatim prose and uncited normative
+    claims instead of pretending they are documentation.
+"""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+from . import query as q
+from .index import search as _s
+
+NORM_KEYWORDS = ("MUST NOT", "MUST", "SHALL NOT", "SHALL", "SHOULD NOT",
+                 "SHOULD NOT", "SHOULD", "MAY", "REQUIRED")
+_WORDS = re.compile(r"[A-Za-z0-9][A-Za-z0-9.'\-_]*")
+_MONTHS = ["jan", "feb", "mar", "apr", "may", "jun",
+           "jul", "aug", "sep", "oct", "nov", "dec"]
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def label(doc_id: str) -> str:
+    """'RFC2167' -> 'RFC 2167' (how a human writes it); 'BCP14' -> 'BCP 14'."""
+    m = re.fullmatch(r"(RFC|BCP|STD|FYI)(\d+)", doc_id)
+    return f"{m.group(1)} {int(m.group(2))}" if m else doc_id
+
+
+def errata_note(conn: sqlite3.Connection, doc_id: str, sec: str = "") -> str:
+    """Errata status for a doc or one section. Distinguishes 'none' from
+    'nobody looked' — asserting zero verified errata when the service was
+    never queried would be a false assurance."""
+    rows = conn.execute(
+        "SELECT status, section FROM errata WHERE doc_id=?",
+        (doc_id,)).fetchall()
+    if not rows:
+        from .errata import bulk_synced_at
+        at = bulk_synced_at(conn)
+        return (f"errata: none (as of {at[:10]})" if at
+                else f"errata: not cached (errata(ref='{doc_id}'))")
+    verified = [r for r in rows if r["status"] == "Verified"]
+    if not verified:
+        return f"errata: 0 verified ({len(rows)} on file)"
+    if sec:
+        want = _s.norm_sec(sec)
+        hit = [r for r in verified
+               if want and want in (r["section"] or "").replace("\n", " ")]
+        if hit:
+            # the exact call, so an agent that has not loaded the errata
+            # tool's schema copies the right argument name
+            return (f"errata: {len(hit)} VERIFIED on this section — read "
+                    f"them before implementing: errata(ref='{doc_id}§{want}')")
+        return (f"errata: {len(verified)} verified elsewhere, "
+                f"none on this section")
+    return f"errata: {len(verified)} verified"
+
+
+def _span(conn: sqlite3.Connection, doc_id: str, section: str | None,
+          lines: str | None) -> dict:
+    """Line span + anchor for a section reference, or an explicit 'a-b' line
+    range when the caller has no section number to offer."""
+    if section:
+        if not conn.execute("SELECT 1 FROM chunks WHERE doc_id=? LIMIT 1",
+                            (doc_id,)).fetchone():
+            return {"ok": False, "reason": f"{doc_id} is not section-indexed "
+                                           "(run `rfclib index build`)"}
+        span = q._resolve_section_arg(conn, doc_id, section)
+        if not span.get("ok"):
+            return {"ok": False, "reason": span.get("reason"),
+                    "suggestions": span.get("suggestions", []),
+                    "outline": span.get("outline", [])}
+        return {"ok": True, "sec_num": span["sec_num"],
+                "sec_title": span.get("sec_title", ""),
+                "start_line": span["start_line"], "end_line": span["end_line"],
+                "frag": span.get("url_frag", "")}
+    if lines:
+        m = re.fullmatch(r"(\d+)\s*[-:]\s*(\d+)", lines.strip())
+        if not m:
+            return {"ok": False,
+                    "reason": f"bad line range {lines!r} (want '1240-1288')"}
+        return {"ok": True, "sec_num": "", "sec_title": "",
+                "start_line": int(m.group(1)), "end_line": int(m.group(2)),
+                "frag": ""}
+    return {"ok": True, "sec_num": "", "sec_title": "", "start_line": 1,
+            "end_line": 0, "frag": ""}
+
+
+def span_text(conn: sqlite3.Connection, doc_id: str, *, mirror_root: Path,
+              section: str | None = None, lines: str | None = None,
+              mode: str = "clean") -> tuple[str, dict]:
+    """The exact text of a span plus its resolved anchor metadata.
+
+    mode='clean' (default) is the canonical form span_hash is defined over;
+    mode='raw' is byte-exact against the RFC Editor file.
+    """
+    from .index.chunker import strip_page_furniture
+    text, served, _p = q._served_text(conn, doc_id, "txt", mirror_root)
+    sp = _span(conn, doc_id, section, lines)
+    if text is None:
+        return "", dict(sp, ok=False, reason="no local text copy")
+    if not sp.get("ok"):
+        return "", sp
+    ls = text.splitlines()
+    end = sp["end_line"] or len(ls)
+    body = "\n".join(ls[sp["start_line"] - 1:end])
+    if mode == "clean":
+        body = strip_page_furniture(body)
+    return body, dict(sp, ok=True, served_via=served, total_lines=len(ls))
+
+
+def _authors_meta(conn, meta) -> str:
+    return " and ".join(meta.get("authors") or [])
+
+
+def cite(conn: sqlite3.Connection, doc_id: str, section: str | None = None,
+         lines: str | None = None, style: str = "verbose", *,
+         mirror_root: Path | None = None) -> str:
+    """A canonical, checkable reference — the cheap substitute for pasting
+    spec text into code.
+
+    style: verbose (default) | short | markdown | comment | bibtex.
+    'verbose' and 'comment' carry the span hash when mirror_root is given, so
+    the citation can later be re-verified with verify().
+    """
+    row = q.get_doc_row(conn, doc_id)
+    meta = q._row_to_meta(conn, row)
+    sp = _span(conn, doc_id, section, lines)
+    if not sp.get("ok") and style in ("verbose", "short", "comment",
+                                      "markdown"):
+        cands = ", ".join(sp.get("suggestions", [])) or "see get_toc"
+        return (f"CANNOT CITE {label(doc_id)} \u00a7{section}: {sp['reason']}\n"
+                f"candidates: {cands}")
+    sec = sp.get("sec_num", "")
+    title = meta.get("title") or ""
+    date = (meta.get("pub_date") or "")[:7]
+    url = q.anchor_url(meta, sp.get("frag") or "", q.resolve(conn, doc_id))
+    secpart = f" \u00a7{sec}" + (f' "{sp["sec_title"]}"'
+                                 if sp.get("sec_title") else "")
+    linepart = (f" lines {sp['start_line']}-{sp['end_line']}" if lines else "")
+    status = meta.get("current_status") or meta.get("status") or ""
+    successor = ", ".join(meta["relations"].get("obsoleted_by", []))
+    state = "current" if meta["is_current"] else \
+        f"OBSOLETE by {successor or 'unknown'}"
+
+    if style == "short":
+        return f"{label(doc_id)}{secpart}{linepart}"
+    if style == "markdown":
+        return f"[{label(doc_id)}{secpart}{linepart}]({url})"
+    if style == "bibtex":
+        num = re.sub(r"\D", "", doc_id)
+        series = ("Request for Comments" if doc_id.startswith("RFC")
+                  else {"BCP": "Best Current Practice",
+                        "STD": "Internet Standards",
+                        "FYI": "FYI"}.get(doc_id[:3], "Request for Comments"))
+        month = (_MONTHS[int(date[5:7]) - 1]
+                 if len(date) >= 7 and date[5:7].isdigit() else "")
+        rows = [f"@misc{{rfc{num},",
+                f"  series =       {{{series}}},",
+                f"  number =       {num},",
+                f"  howpublished = {{{label(doc_id)}}},",
+                "  publisher =    {RFC Editor},",
+                f"  doi =          {{{meta.get('doi') or ''}}},",
+                f"  url =          {{{meta.get('canonical_url') or ''}}},",
+                f"  author =       {{{_authors_meta(conn, meta) or ''}}},",
+                f"  title =        {{{{{title}}}}},"]
+        if year := date[:4]:
+            rows.append(f"  year =         {year},")
+        if month:
+            rows.append(f"  month =        {month},")
+        rows.append("}")
+        return "\n".join(rows)
+
+    sha = ""
+    if mirror_root is not None and (sec or lines):
+        body, _ = span_text(conn, doc_id, mirror_root=mirror_root,
+                            section=section, lines=lines)
+        sha = f" | span sha256:{q.span_hash(body)}"
+    tail = (f"{label(doc_id)}{secpart}{linepart} — {title} ({date}, "
+            f"{status}, {state})\n{url} | "
+            f"{errata_note(conn, doc_id, sec)}{sha}")
+    if style == "comment":
+        return "\n".join(("# " + ln) if ln else "#" for ln in tail.split("\n"))
+    if not meta["is_current"]:
+        tail += (f"\nWARNING: {label(doc_id)} is obsolete"
+                 + (f"; cite {successor} instead" if successor else ""))
+    return tail
+