primarysource 0.1.0__py3-none-any.whl

primarysource/itu/text.py

@@ -0,0 +1,609 @@
+"""PDF to clauses. The unit of citation in an ITU recommendation is the subclause, not the page.
+
+Extraction is lossy in ways that matter to a citation: ITU's PDFs emit "Rec. ITU -T X.680" with a stray space,
+hyphenate "set -of" and "ISO/IEC 8824 -1". `tidy` repairs the three artefacts actually observed in 02/2021, and the
+repairs are named here rather than hidden, because a quote that has been silently normalised is a quote that
+cannot be checked against the paper. Anything else the extractor mangles stays broken and visible.
+
+Clause boundaries are recognised as a numbered heading: `8.1.2.4 For tags with a number greater than or
+equal to 31 ...`. Table-of-contents lines carry dot leaders and are discarded rather than indexed, since an
+index whose top hit for "BOOLEAN" is a contents entry answers no question.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+# pypdf (the [itu] extra) is imported where a PDF is opened, not here: the collection and its server import
+# without it (PLAN.md §5.13), and an index already built needs no PDF reader.
+
+# Numeric (8.1.2.4) or annex (A.1, C.3.2) numbering. Annex letters stop at I: past that they collide with
+# recommendation names ("G.711", "H.323") far more often than they head a clause.
+# The extractor sometimes drops the space after the number ("22.11.2.1If ..."); a capitalised word is accepted then.
+CLAUSE_NUMBER = re.compile(r"(?<![\w.])((?:\d{1,2}|[A-I])(?:\.\d{1,2}){1,5})(?:\s|(?=[A-Z][a-z]))")
+# A row of the document-history table ("1.1 ITU-T X.690 (1994) Cor. 1 ...") looks like clause 1.1.
+HISTORY_ROW = re.compile(r"ITU-T\s+[A-Z]\.\d+\s*\(?\d{4}")
+# "ISO/IEC 8824 -1": the extractor puts a space before the part number of a joint designation.
+ISO_PART = re.compile(r"(\d{4}) -(\d)")
+LEADER = re.compile(r"\.{5,}")
+SPACE = re.compile(r"\s+")
+
+
+@dataclass(frozen=True)
+class Clause:
+    number: str
+    title: str
+    body: str
+    heading: bool = False  # title is the document's own heading, not the first sentence
+
+    def cite(self, edition_key: str) -> str:
+        return f"Rec. ITU-T {edition_key} §{self.number}"
+
+
+def tidy(text: str) -> str:
+    """Repair the three extraction artefacts observed in 02/2021. Lossy, and said so on purpose."""
+    text = text.replace("ITU -T", "ITU-T").replace("set -of", "set-of")
+    return SPACE.sub(" ", ISO_PART.sub(r"\1-\2", text)).strip()
+
+
+def pages(path: object) -> list[str]:
+    from pypdf import PdfReader
+
+    reader = PdfReader(str(path))
+    return strip_running_lines([page.extract_text() or "" for page in reader.pages])
+
+
+def strip_running_lines(page_texts: list[str]) -> list[str]:
+    """Drop running page headers and footers ("ISO/IEC 8825-1:2021 (E) 14", "Rec. ITU-T X.690 (02/2021)").
+
+    They land in the middle of a clause when a clause crosses a page, which corrupts quoting and makes the same
+    sentence differ between editions. A line is running text when, with its digits masked, it contains a digit
+    (a page or edition number) and repeats on at least a quarter of the pages. Requiring the digit keeps a
+    legitimately repeated word such as a table caption.
+    """
+    if len(page_texts) < 4:
+        return page_texts
+
+    def masked(line: str) -> str:
+        return re.sub(r"\d+", "#", line.strip())
+
+    seen: dict[str, int] = {}
+    for page in page_texts:
+        for key in {masked(line) for line in page.splitlines() if line.strip()}:
+            seen[key] = seen.get(key, 0) + 1
+    threshold = max(3, len(page_texts) // 4)
+    # A clause heading masks to the same string every time it repeats ("10.1 General", "18.1 General" ->
+    # "#.# General"), so lines that start with a clause number are never running text.
+    numbered = re.compile(r"^(?:#|[A-I])(?:\.#)+\s")
+    running = {
+        key
+        for key, count in seen.items()
+        if count >= threshold and "#" in key and len(key) <= 90 and not numbered.match(key)
+    }
+    return ["\n".join(line for line in page.splitlines() if masked(line) not in running) for page in page_texts]
+
+
+_TITLE_END = re.compile(r"(?<!Rec)(?<!e\.g)(?<!i\.e)(?<!Fig)(?<!No)(?<!cf)(?<!Cor)(?<!Amd)\. ")
+
+
+def first_sentence(chunk: str) -> str:
+    """First sentence of the chunk, capped. A heading and its first rule are not separable in extracted
+    text — `11.6 Set-of components The encodings of…` runs straight on — so the title is offered as a label
+    and `body` stays authoritative."""
+    cut = _TITLE_END.search(chunk)
+    head = chunk if cut is None else chunk[: cut.start()]
+    return head[:96].rstrip()
+
+
+_REFERENCE_WORDS = frozenset(
+    [
+        "clause",
+        "clauses",
+        "subclause",
+        "subclauses",
+        "sub-clause",
+        "see",
+        "in",
+        "of",
+        "and",
+        "or",
+        "to",
+        "with",
+        "by",
+        "from",
+        "per",
+        "under",
+        "table",
+        "figure",
+        "annex",
+        "than",
+        "also",
+        "both",
+        "either",
+        "between",
+        "through",
+    ]
+)
+_PRECEDING_WORD = re.compile(r"([A-Za-z-]+|§|[,(])\s*$")
+
+
+def _is_reference(flat: str, start: int) -> bool:
+    """True when a dotted number is a pointer ("see 8.1.2") rather than the start of a heading.
+
+    A heading follows a sentence end or the previous clause's text; a pointer follows a word that introduces
+    one. Treating the pointer as a heading split clauses mid-sentence and produced duplicate numbers, so that
+    `read` could return a fragment of a cross-reference instead of the clause. The cost of the opposite error
+    (a real heading right after one of these words) is a clause merged into its neighbour, which is visible in
+    the outline rather than silently wrong.
+    """
+    match = _PRECEDING_WORD.search(flat[max(start - 20, 0) : start])
+    if not match:
+        return False
+    word = match.group(1)
+    if len(word) > 1 and word.isupper():
+        return False  # an acronym ("PER", "BER"), not the word "per" shouted
+    return word in {"§", ",", "("} or word.lower() in _REFERENCE_WORDS
+
+
+def _plausible_heading(flat: str, start: int, number: str) -> bool:
+    """Annex numbers (A.1) are only headings at a sentence boundary: elsewhere they are far more likely to be
+    a reference ("Part 2, D.2)"), a recommendation name, or an enumerator."""
+    if number[0].isdigit():
+        return True
+    before = flat[:start].rstrip()
+    return not before or before[-1] in ".:;)"
+
+
+_HEADING_TITLE = re.compile(r"^((?:\d{1,2}|[A-I])(?:\.\d{1,2})*)\s+(\S.*)$")
+_ANNEX_TITLE = re.compile(r"^Annex\s+([A-I])\b\s*[-\u2013\u2014:.]*\s*(.*)$")
+_TOC_LINE = re.compile(r"^\s*((?:\d{1,2}|[A-I])(?:\.\d{1,2})*)\s+(.*?)[\s.]*\.{3,}[\s.\d]*$")
+
+
+def headings(path: object, page_text: list[str] | None = None) -> list[tuple[str, str]]:
+    """The recommendation's own list of numbered headings, in document order: (number, title).
+
+    The PDF bookmarks are the source: ITU's PDFs carry one per heading down to three or four levels, with the
+    exact title. The contents pages are the fallback when a PDF has none. Deeper numbered paragraphs
+    (8.1.2.4.2) are in neither, and are found by `clauses` from their position instead.
+    """
+    found: list[tuple[str, str]] = []
+
+    def walk(items: list[object]) -> None:
+        for item in items:
+            if isinstance(item, list):
+                walk(item)
+                continue
+            title = SPACE.sub(" ", str(getattr(item, "title", ""))).strip()
+            match = _HEADING_TITLE.match(title)
+            annex = _ANNEX_TITLE.match(title)
+            if match:
+                found.append((match.group(1), tidy(match.group(2))))
+            elif annex:
+                found.append((annex.group(1), tidy(annex.group(2)) or f"Annex {annex.group(1)}"))
+
+    from pypdf import PdfReader
+    from pypdf.errors import PdfReadError
+
+    try:
+        walk(list(PdfReader(str(path)).outline))
+    except (PdfReadError, ValueError, OSError):
+        found.clear()  # not a readable PDF: fall through to the contents lines
+    if found:
+        return found
+    for line in "\n".join(page_text if page_text is not None else pages(path)).splitlines():
+        toc = _TOC_LINE.match(line)
+        if toc:
+            found.append((toc.group(1), tidy(toc.group(2))))
+    return found
+
+
+def _key(number: str) -> tuple[int | str, ...]:
+    return tuple(int(part) if part.isdigit() else part for part in number.split("."))
+
+
+def _is_successor(last: tuple[int | str, ...], candidate: tuple[int | str, ...]) -> bool:
+    """Whether `candidate` can directly follow `last` in a numbered document: its first child (8.1 -> 8.1.1), its
+    next sibling (8.1.2 -> 8.1.3), or the next sibling of an ancestor (8.1.2.4 -> 8.1.3). Anything else in the
+    middle of a clause is a pointer, an enumerator or a stray number, which is exactly what the unconstrained
+    splitter was mistaking for headings."""
+    if candidate[:-1] == last and candidate[-1] == 1:
+        return True
+    for depth in range(1, len(last) + 1):
+        parent, tail = last[: depth - 1], last[depth - 1]
+        if isinstance(tail, int) and candidate == (*parent, tail + 1):
+            return True
+    return False
+
+
+def _title_pattern(title: str) -> re.Pattern[str] | None:
+    words = tidy(title).split()
+    if not words:
+        return None
+    return re.compile(r"\s*".join(re.escape(word) for word in words), re.IGNORECASE)
+
+
+def _locate(flat: str, listed: list[tuple[str, str]]) -> list[tuple[str, str, int, int]]:
+    """Find each listed heading in the text, in order: (number, title, start, end-of-title).
+
+    The number alone is not enough (every pointer to "8.1" matches it), so the title has to follow, and the search
+    only moves forward, so a heading cannot be satisfied by an earlier cross-reference.
+    """
+    out: list[tuple[str, str, int, int]] = []
+    cursor = 0
+    for number, title in listed:
+        pattern = _title_pattern(title)
+        # A top-level annex is bookmarked as "A" but printed as "Annex A".
+        label = r"Annex\s+" + number + r"[\s\-–—:.]*" if number.isalpha() else re.escape(number) + r"\s+"
+        head = re.compile(r"(?<![\w.])" + label)
+        for match in head.finditer(flat, cursor):
+            # A heading that follows the previous heading's title with nothing between is not a pointer, whatever
+            # that title ends in ("15 The renames clause 15.1 Explicitly ...").
+            adjacent = not flat[cursor : match.start()].strip()
+            if not adjacent and _is_reference(flat, match.start()):
+                continue
+            tail = pattern.match(flat, match.end()) if pattern else None
+            if tail is None and pattern is not None:
+                short = _title_pattern(" ".join(title.split()[:4]))
+                tail = short.match(flat, match.end()) if short else None
+            if tail is not None:
+                out.append((number, title, match.start(), tail.end()))
+                cursor = tail.end()
+                break
+    return out
+
+
+def _body_after_title(flat: str, start: int, stop: int) -> str:
+    return flat[start:stop].strip()
+
+
+def clauses(source: list[str] | str, listed: list[tuple[str, str]] | None = None) -> tuple[Clause, ...]:
+    """Split recommendation text into clauses.
+
+    With `listed` (the recommendation's own headings, from `headings`) the structure comes from the document:
+    every listed heading found in order becomes a clause with its exact title, headings with no text of their
+    own included, and numbered paragraphs inside it are accepted only as strict successors. Without it, the
+    older heuristic applies: any dotted number that is not a pointer.
+    """
+    raw = source if isinstance(source, str) else "\n".join(source)
+    # Contents lines are dropped whole. Stripping the dot leaders instead leaves "11.2 Unused bits 20" behind,
+    # which is a heading-shaped fragment that runs on into the next section and gets indexed as if it were
+    # the rule itself.
+    kept = [line for line in raw.splitlines() if not LEADER.search(line)]
+    flat = tidy("\n".join(kept))
+    if listed:
+        return _clauses_from_headings(flat, listed)
+    marks = [
+        (match.group(1), match.start(), match.end())
+        for match in CLAUSE_NUMBER.finditer(flat)
+        if not match.group(1).endswith(".0")
+        and not _is_reference(flat, match.start())
+        and _plausible_heading(flat, match.start(), match.group(1))
+    ]
+    found: list[Clause] = []
+    for index, (number, _start, end) in enumerate(marks):
+        stop = marks[index + 1][1] if index + 1 < len(marks) else len(flat)
+        body = flat[end:stop].strip()
+        if len(body) < 20 or HISTORY_ROW.match(body):
+            continue
+        found.append(Clause(number=number, title=first_sentence(body), body=body))
+    return tuple(found)
+
+
+# Words that can follow a pointer ("applying 13.4 to the set", "then 16.8 to 16.11 apply") and never open a
+# numbered paragraph or a definition entry. A definition's term is lowercase too ("3.1 canonical encoding:"), which
+# is why this is a list of function words and not "anything lowercase": of the 276 accepted paragraphs that began
+# with a lowercase word in the 02/2021 corpus, only four (all "to") began with one of these; all four were pointers.
+_POINTER_FOLLOWERS = frozenset(
+    [
+        *("to", "and", "or", "shall", "should", "may", "must", "applies", "apply", "is", "are", "of", "do", "does"),
+        *("not", "still", "which", "that", "if", "then", "in", "for", "from", "as", "with", "by", "also", "above"),
+        *("below", "can", "cannot", "will", "was", "were", "has", "have", "being", "but"),
+    ]
+)
+_NEXT_WORD = re.compile(r"\s*([a-z][a-z-]*)\b")
+
+
+def _is_pointer_by_follower(flat: str, end: int) -> bool:
+    match = _NEXT_WORD.match(flat, end)
+    return match is not None and match.group(1) in _POINTER_FOLLOWERS
+
+
+def _is_annex_entry(
+    flat: str, last: tuple[int | str, ...], candidate: tuple[int | str, ...], at: int, floor: int
+) -> bool:
+    """`C.1` straight after the text of the previous annex or clause, where PDF bookmarks skip the annex's own
+    heading ("Annex C", its title and "(This annex forms an integral part ...)"). Accepted only for a later annex
+    letter and only when the words "Annex C" occur since the previous numbered paragraph, so a pointer such as
+    "defined in C.10" cannot open an annex."""
+    if len(candidate) != 2 or candidate[1] != 1 or not isinstance(candidate[0], str):
+        return False
+    letter = candidate[0]
+    if not isinstance(last[0], int) and letter <= last[0]:
+        return False  # annexes only advance; a letter has no heading to open twice
+    # Later than the current one, not necessarily the next: an annex without numbered paragraphs ("Annex B") is
+    # skipped over.
+    return re.search(rf"Annex\s+{letter}\b", flat[floor:at]) is not None
+
+
+def _paragraph_marks(
+    flat: str, number: str, start: int, stop: int, taken: frozenset[str] = frozenset()
+) -> list[tuple[str, int, int]]:
+    """Numbered paragraphs inside one heading's region, accepted only as successors of the previous one.
+
+    `taken` holds the numbers of headings located elsewhere in the document. Such a number cannot also be a
+    paragraph of this region, so an occurrence here is a pointer or a list entry ("RepetitionStructure 16.4
+    ConcatenationStructure 16.5") however well it fits the sequence.
+    """
+    marks: list[tuple[str, int, int]] = []
+    last = _key(number)
+    floor = start
+    for match in CLAUSE_NUMBER.finditer(flat, start, stop):
+        candidate = match.group(1)
+        key = _key(candidate)
+        if (
+            candidate not in taken
+            and (_is_successor(last, key) or _is_annex_entry(flat, last, key, match.start(), floor))
+            and not candidate.endswith(".0")
+            # nothing between the heading's title and the number: a paragraph, whatever the title ends in
+            and (not flat[start : match.start()].strip() or not _is_reference(flat, match.start()))
+            and not _is_pointer_by_follower(flat, match.end())
+        ):
+            marks.append((candidate, match.start(), match.end()))
+            last = key
+            floor = match.end()
+    return marks
+
+
+def _clauses_from_headings(flat: str, listed: list[tuple[str, str]]) -> tuple[Clause, ...]:
+    located = _locate(flat, listed)
+    found: list[Clause] = []
+    taken = frozenset(number for number, *_ in located)
+    for index, (number, title, _start, title_end) in enumerate(located):
+        region_end = located[index + 1][2] if index + 1 < len(located) else len(flat)
+        marks = _paragraph_marks(flat, number, title_end, region_end, taken)
+        first_stop = marks[0][1] if marks else region_end
+        found.append(
+            Clause(number=number, title=title, body=_body_after_title(flat, title_end, first_stop), heading=True)
+        )
+        for position, (candidate, _s, end) in enumerate(marks):
+            stop = marks[position + 1][1] if position + 1 < len(marks) else region_end
+            body = flat[end:stop].strip()
+            if len(body) >= 20 and not HISTORY_ROW.match(body):
+                found.append(Clause(number=candidate, title=first_sentence(body), body=body))
+    return tuple(found)
+
+
+def find(clauses_in: tuple[Clause, ...], number: str) -> Clause | None:
+    return next((clause for clause in clauses_in if clause.number == number), None)
+
+
+_STOPWORDS = frozenset(
+    [
+        "a",
+        "an",
+        "the",
+        "of",
+        "for",
+        "in",
+        "on",
+        "to",
+        "and",
+        "or",
+        "is",
+        "are",
+        "be",
+        "by",
+        "with",
+        "as",
+        "at",
+        "from",
+        "that",
+        "this",
+        "it",
+        "its",
+        "into",
+        "than",
+    ]
+)
+
+# Words that make a question a question and say nothing about the subject. Left in a query they are required terms
+# under all-terms matching and noise under any-term matching. Modal verbs ("must", "shall", "should") are NOT here:
+# in a normative document they are part of what is being asked. Measured on the blind retrieval set, dropping this
+# list took hit@1 from 38 to 45 of 82 and improved both halves of an even/odd split of the questions.
+_QUESTION_WORDS = frozenset(
+    [
+        "how",
+        "what",
+        "which",
+        "when",
+        "where",
+        "why",
+        "who",
+        "whom",
+        "do",
+        "does",
+        "did",
+        "i",
+        "my",
+        "me",
+        "we",
+        "you",
+        "your",
+        "there",
+        "these",
+        "those",
+        "if",
+        "any",
+        "some",
+        "say",
+        "says",
+        "said",
+        "get",
+        "gets",
+        "was",
+        "were",
+        "been",
+    ]
+)
+
+_DEFINITION_WORDS = frozenset({"definition", "definitions", "meaning", "means", "mean", "glossary", "term", "terms"})
+
+
+def query_tokens(raw: str) -> list[str]:
+    """The words of a query with punctuation stripped from their ends. Case is kept."""
+    return [token for token in (word.strip("?.,;:!()\"'") for word in raw.split()) if token]
+
+
+def asks_for_definition(raw: str) -> bool:
+    """Whether the query is looking up what a term means, which is when the definitions clause is the right answer."""
+    return any(token.lower() in _DEFINITION_WORDS for token in query_tokens(raw))
+
+
+def query_words(raw: str) -> list[str]:
+    """The words of a query that are search terms: filler and question words dropped, a query made only of them
+    kept as typed. Modal verbs stay: in a normative document they are part of what is being asked."""
+    tokens = [token for token in raw.split() if token]
+    # Filler words must not be required terms ("encoding for a value of a CHOICE type" matches no clause that
+    # contains every one of those words), but a query made only of them is still a query.
+    content = [token for token in tokens if token.strip("?.,;:!()\"'").lower() not in _STOPWORDS | _QUESTION_WORDS]
+    return content or tokens
+
+
+def fts_query(raw: str, *, any_term: bool = False) -> str:
+    """Turn a user query into something FTS5 will accept: `query_words`, each quoted as a phrase.
+
+    A raw query is not a valid MATCH expression: `set-of value order` parses as `set` NEAR `-of` (an
+    unsupported-unary-not) and raised OperationalError out of a search tool, which is exactly the wrong shape
+    of failure - the caller sees a crash where it should see either an answer or a stated error. Each token is
+    quoted, and internal double quotes are doubled, which is FTS5's own escape. The cost is that advanced
+    syntax (`AND`, `NEAR`, prefix `*`) stops working from the query box; that is the honest trade, and it is
+    said here rather than left for a user to discover. The index searches through the core's own quoting
+    (`primarysource.index.fts_query`), given these words.
+    """
+    tokens = query_words(raw)
+    if not tokens:
+        raise ValueError("empty query: there is nothing to search for")
+    joiner = " OR " if any_term else " "
+    return joiner.join(f'"{token.replace(chr(34), chr(34) * 2)}"' for token in tokens)
+
+
+# --- segmenting, normative statements and references -------------------------------------------------------
+
+_ABBREVIATIONS = ("Rec.", "e.g.", "i.e.", "No.", "Fig.", "Figs.", "Tab.", "etc.", "cf.", "vs.", "ITU-T.")
+_SENTENCE_BREAK = re.compile(r"(?<=[.;:])\s+(?=[A-Z(])")
+_NORMATIVE = re.compile(r"\b(shall(?: not)?|should(?: not)?|must(?: not)?|may)\b")
+_N = r"(?:\d{1,2}|[A-I])(?:\.\d{1,2})+"
+_INTERNAL = re.compile(rf"(?:clauses?|sub-?clauses?|§)\s*({_N}(?:\s*(?:,|and|or|to)\s*{_N})*)")
+_NUMBER_AT = re.compile(rf"(?<![\w.])({_N})(?![\d.]*\d)")
+_NUMBER = re.compile(_N)
+_REC = re.compile(r"\b(?:ITU-T\s+)?([A-Z]\.\d{1,4})(?:\s*\|\s*ISO/IEC\s+\d+(?:-\d+)?)?")
+_ISO = re.compile(r"\bISO/IEC\s+\d+(?:-\d+)?(?::\d{4})?")
+_RFC = re.compile(r"\bRFC\s*\d+")
+
+
+def sentences(body: str) -> tuple[str, ...]:
+    """Split a clause body into citable units.
+
+    The extracted text has no paragraph structure left (`tidy` flattens whitespace, and PDF line breaks carry
+    no information), so the unit is the sentence and `¶n` means the nth sentence of the extracted clause. That
+    numbering is stable for a digest-pinned edition and is NOT ITU's own paragraphing; a citation that uses it
+    should also give the clause number and edition, which `cite` does.
+    """
+    merged: list[str] = []
+    for piece in _SENTENCE_BREAK.split(body):
+        if merged and merged[-1].endswith(_ABBREVIATIONS):
+            merged[-1] = f"{merged[-1]} {piece}"
+        else:
+            merged.append(piece)
+    return tuple(merged)
+
+
+def normative(body: str) -> tuple[tuple[int, str], ...]:
+    """(¶ index, sentence) for every sentence stating an obligation, permission or recommendation."""
+    return tuple((n, s) for n, s in enumerate(sentences(body), start=1) if _NORMATIVE.search(s))
+
+
+@dataclass(frozen=True)
+class Refs:
+    clauses: tuple[str, ...]
+    bare: tuple[str, ...]  # dotted numbers after "see"/"in"/"of"...: pointers if, and only if, the clause exists
+    recommendations: tuple[str, ...]
+    standards: tuple[str, ...]
+    foreign: tuple[str, ...] = ()  # numbers qualified by another recommendation's designation: not this document
+
+
+_DESIGNATION = r"(?:ITU-T\s+[A-Z]\.\d+|ISO/IEC\s+\d+(?:-\d+)?)"
+# "... Rec. ITU-T X.680 | ISO/IEC 8824-1, clause 12.3": the designation directly before the pointer.
+_FOREIGN_BEFORE = re.compile(rf"{_DESIGNATION}(?:\s*\|\s*{_DESIGNATION})*\s*,?\s*$")
+# "clause 12.3 of Rec. ITU-T X.680": the designation directly after it.
+_FOREIGN_AFTER = re.compile(r"\s*,?\s*(?:of|in)\s+(?:Rec\.|ITU-T|ISO/IEC)")
+
+
+def _is_foreign(body: str, start: int, end: int) -> bool:
+    """True when a pointer is qualified by another recommendation's designation. Resolving "clause 12.3 of
+    Rec. ITU-T X.680" against this document's own clause 12.3 would silently cite the wrong text. A pointer
+    qualified by the document's own name is not recognised as such: this text says "this Recommendation" instead."""
+    return (
+        _FOREIGN_BEFORE.search(body[max(start - 60, 0) : start]) is not None
+        or _FOREIGN_AFTER.match(body, end) is not None
+    )
+
+
+def refs(body: str) -> Refs:
+    """What a clause points at: other clauses of the same document, other recommendations, ISO/IEC and RFCs.
+
+    Only references introduced by a word such as "clause" or the section sign count as clause references;
+    a bare dotted number is more often a version, an arc or an octet range than a pointer. Numbers after a
+    weaker introducer ("see 8.1.3", "as in 8.1.3.2") are returned separately as `bare`: callers should keep
+    one only if that clause exists, because nothing else distinguishes it from an arc or a version.
+    """
+    seen: dict[str, None] = {}
+    foreign: dict[str, None] = {}
+    for match in _INTERNAL.finditer(body):
+        target = foreign if _is_foreign(body, match.start(), match.end()) else seen
+        for number in _NUMBER.findall(match.group(1)):
+            target.setdefault(number)
+    bare = dict.fromkeys(
+        m.group(1)
+        for m in _NUMBER_AT.finditer(body)
+        if m.group(1) not in seen and _is_reference(body, m.start()) and not _is_foreign(body, m.start(), m.end())
+    )
+    recs = dict.fromkeys(m.group(1) for m in _REC.finditer(body))
+    standards = dict.fromkeys([m.group(0) for m in _ISO.finditer(body)] + [m.group(0) for m in _RFC.finditer(body)])
+    return Refs(tuple(seen), tuple(bare), tuple(recs), tuple(standards), tuple(foreign))
+
+
+_REF = re.compile(r"\s*(?:([A-Z]\.\d+(?:#[\d-]+)?)\s*)?§?\s*((?:\d{1,2}|[A-I])(?:\.\d+)*)(?:\s*¶\s*(\d+)(-)?)?\s*")
+
+
+@dataclass(frozen=True)
+class Target:
+    """A parsed ref. `onward`: the ref ends `¶n-`, the clause from sentence n on, which is the `next:` of a read."""
+
+    rec: str | None
+    number: str
+    para: int | None
+    onward: bool = False
+
+
+def parse_target(raw: str) -> Target:
+    """`X.690§11.6¶2`, `X.690#2021-02§11.6¶5-`, `X.690 11.6`, `§11.6`, `11.6¶2` -> `Target`."""
+    match = _REF.fullmatch(raw)
+    if not match:
+        raise ValueError(f"{raw!r} is not a clause reference; expected e.g. X.690§11.6 or 11.6¶2")
+    para = int(match.group(3)) if match.group(3) else None
+    if para == 0:
+        raise ValueError(f"{raw!r}: ¶ counts sentences from 1")
+    return Target(match.group(1), match.group(2), para, bool(match.group(4)))
+
+
+def parse_ref(raw: str) -> tuple[str | None, str, int | None]:
+    """`X.690§11.6¶2`, `X.690 11.6`, `§11.6`, `11.6¶2` -> (rec or None, clause number, ¶ or None).
+
+    The `¶n-` form is a read cursor and means more than one sentence, so it is refused here, where one ¶ is meant.
+    """
+    target = parse_target(raw)
+    if target.onward:
+        raise ValueError(f"{raw!r} is a read cursor (¶n-, the clause from ¶n on); name one ¶ or the clause")
+    return target.rec, target.number, target.para

primarysource/kconfig/__init__.py

@@ -0,0 +1,16 @@
+"""`kconfig`: the Kconfig language documents of the Linux kernel, a pinned release, read as reStructuredText.
+Served under `kconfig_`.
+
+Stdlib only. `mcp` is imported by `server.py` when a server is built; `primarysource kconfig ...` runs `cli.py`.
+`docs/kconfig.md` is the description; `catalogue.py` holds the pin, the licence and the build.
+"""
+
+from __future__ import annotations
+
+from primarysource.kconfig.catalogue import SPEC
+from primarysource.pkgdocs.runtime import make_collection
+
+COLLECTION = make_collection(SPEC, "primarysource.kconfig")
+"""Refs: `kconfig:kconfig-language§Menu entries`; see `pkgdocs.runtime`."""
+
+__all__ = ["COLLECTION", "SPEC"]