primarysource 0.1.0__py3-none-any.whl

primarysource/pkgdocs/index.py

@@ -0,0 +1,89 @@
+"""A per-edition full-text index: SQLite FTS5, built once at import, opened read-only.
+
+One row per section (its title and its blocks), ranked by BM25 with the title weighted over the body. This is the
+interim form of the core's `cl.index_fts5`; it keeps the rules the roadmap already names for that row (a sanitiser
+that quotes user text term by term so FTS5 operators cannot raise, strict AND then a relaxed OR that says so, a
+file that is built beside its destination and renamed into place, a read-only open).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import re
+import sqlite3
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from primarysource.errors import LibError
+from primarysource.pkgdocs.model import Doc
+
+SCHEMA_VERSION = 1
+MAX_TERMS = 12
+WEIGHTS = "0.0, 0.0, 8.0, 1.0"  # doc and sec are not searched; a title match outweighs a body match
+_TERM = re.compile(r"\w+", re.UNICODE)
+
+
+@dataclass(frozen=True, slots=True)
+class Hit:
+    doc: str
+    section: str
+    title: str
+    snippet: str
+    score: float
+
+
+def build(path: Path, docs: Sequence[Doc]) -> int:
+    """Write the index for `docs` at `path` (which must not exist). Returns the number of sections indexed."""
+    rows = 0
+    with contextlib.closing(sqlite3.connect(path)) as db:
+        db.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
+        db.execute(
+            "CREATE VIRTUAL TABLE sections USING fts5("
+            "doc UNINDEXED, sec UNINDEXED, title, body, tokenize='porter unicode61')"
+        )
+        for doc in docs:
+            for section in doc.sections:
+                db.execute(
+                    "INSERT INTO sections(doc, sec, title, body) VALUES (?, ?, ?, ?)",
+                    (doc.id, section.id, section.title, "\n".join(section.units)),
+                )
+                rows += 1
+        db.commit()
+    return rows
+
+
+def _match(terms: Sequence[str], joiner: str) -> str:
+    return f" {joiner} ".join('"' + t.replace('"', '""') + '"' for t in terms)
+
+
+def search(path: Path, query: str, limit: int = 10, offset: int = 0, doc_prefix: str = "") -> tuple[list[Hit], bool]:
+    """Sections matching `query`, best first, and whether the match was relaxed (OR instead of AND). An empty
+    list means nothing matched; a missing or unreadable index is a `LibError` that names the remedy."""
+    terms = _TERM.findall(query)[:MAX_TERMS]
+    if not terms:
+        raise LibError(f"nothing to search for in {query!r}", "give at least one word")
+    if not path.is_file():
+        raise LibError(f"there is no search index at {path}", "", "import the edition again")
+    try:
+        with contextlib.closing(sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)) as db:
+            if db.execute("PRAGMA user_version").fetchone()[0] != SCHEMA_VERSION:
+                raise LibError("the search index has a different schema version", "", "import the edition again")
+            for relaxed, joiner in ((False, "AND"), (True, "OR")):
+                if relaxed and len(terms) == 1:
+                    break
+                like = doc_prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
+                found = db.execute(
+                    "SELECT doc, sec, title, snippet(sections, 3, '[', ']', ' ... ', 24), "  # noqa: S608
+                    f"bm25(sections, {WEIGHTS}) "
+                    "FROM sections WHERE sections MATCH ? AND doc LIKE ? ESCAPE '\\' "
+                    f"ORDER BY bm25(sections, {WEIGHTS}) LIMIT ? OFFSET ?",
+                    (_match(terms, joiner), like, limit, offset),
+                ).fetchall()
+                if found:
+                    return [
+                        Hit(str(r[0]), str(r[1]), str(r[2]), " ".join(str(r[3]).split()), float(r[4])) for r in found
+                    ], relaxed
+    except sqlite3.DatabaseError as exc:
+        raise LibError(f"the search index is unreadable ({exc})", "", "import the edition again") from exc
+    return [], False

primarysource/pkgdocs/model.py

@@ -0,0 +1,76 @@
+"""The document a reader returns and the store keeps.
+
+Like `primarysource.model.Document` but each section carries an `anchor`, because Markdown, reStructuredText,
+AsciiDoc and LaTeX address sections by a label or a slug as well as by position. When the core's `Section` gains
+`anchor` (ROADMAP `cl.model_anchor`) this module reduces to a conversion.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from primarysource.errors import LibError
+from primarysource.model import Unit
+
+
+@dataclass(frozen=True, slots=True)
+class Sec:
+    """One addressable part of a document. `id` is an ordinal path (`2.1`); `units` are in reading order and
+    `¶1` is `units[0]`. `depth` is the nesting depth, 1 for a top-level section."""
+
+    id: str
+    title: str
+    depth: int
+    anchor: str
+    units: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class Doc:
+    """A document: an id, a title, the unit kind, its sections in order, free-form `meta` (source path, status,
+    licence note) and the `warnings` a reader raised about what it could not represent."""
+
+    id: str
+    title: str
+    unit: Unit
+    sections: tuple[Sec, ...] = ()
+    meta: Mapping[str, str] = field(default_factory=dict)
+    warnings: tuple[str, ...] = ()
+
+    def section(self, section_id: str) -> Sec | None:
+        for candidate in self.sections:
+            if candidate.id == section_id:
+                return candidate
+        return None
+
+    def with_meta(self, **more: str) -> Doc:
+        return Doc(self.id, self.title, self.unit, self.sections, {**self.meta, **more}, self.warnings)
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "title": self.title,
+            "unit": self.unit.value,
+            "meta": dict(self.meta),
+            "warnings": list(self.warnings),
+            "sections": [[s.id, s.title, s.depth, s.anchor, list(s.units)] for s in self.sections],
+        }
+
+    @staticmethod
+    def from_json(data: Mapping[str, Any]) -> Doc:
+        try:
+            sections = tuple(
+                Sec(str(r[0]), str(r[1]), int(r[2]), str(r[3]), tuple(str(u) for u in r[4])) for r in data["sections"]
+            )
+            return Doc(
+                str(data["id"]),
+                str(data["title"]),
+                Unit(data["unit"]),
+                sections,
+                {str(k): str(v) for k, v in dict(data["meta"]).items()},
+                tuple(str(w) for w in data["warnings"]),
+            )
+        except (KeyError, IndexError, TypeError, ValueError) as exc:
+            raise LibError(f"a stored document is damaged ({exc!r})", "", "import the edition again") from exc

primarysource/pkgdocs/readers/__init__.py

@@ -0,0 +1 @@
+"""Minimal, bounded readers: bytes in, a `Doc` out. See `primarysource.pkgdocs` for what "bounded" promises."""

primarysource/pkgdocs/readers/_common.py

@@ -0,0 +1,134 @@
+"""What every reader needs: strict decoding, slugs and the section builder."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+from primarysource.errors import LibError
+from primarysource.model import Unit
+from primarysource.pkgdocs.model import Doc, Sec
+
+_SLUG_DROP = re.compile(r"[^\w\- ]", re.UNICODE)
+
+
+def decode(data: bytes, what: str) -> str:
+    """UTF-8 text with line endings normalised to `\\n`. A NUL byte or invalid UTF-8 is an error, never a guess:
+    a reader that substituted characters would quietly change what a citation says."""
+    if b"\x00" in data:
+        raise LibError(f"{what} contains a NUL byte; it is not text", "check that the source is the expected file")
+    try:
+        text = data.decode("utf-8-sig")
+    except UnicodeDecodeError as exc:
+        raise LibError(
+            f"{what} is not valid UTF-8 ({exc.reason} at byte {exc.start})",
+            "the collection reads UTF-8 sources only; report the file rather than converting it",
+        ) from exc
+    if "\r" in text:
+        text = text.replace("\r\n", "\n").replace("\r", "\n")
+    return text
+
+
+_ROLE = re.compile(r":[A-Za-z][\w:+.-]*:`([^`]*)`")
+
+
+def clean_title(title: str) -> str:
+    """A heading as plain text: RST roles and inline-literal backticks removed, whitespace collapsed."""
+    return " ".join(_ROLE.sub(chr(92) + "1", title).replace("`", "").split())
+
+
+def slugify(title: str) -> str:
+    """GitHub's heading slug: lowercase, punctuation dropped, spaces to hyphens. This is the rule used here, not
+    a claim that any specification defines it."""
+    text = unicodedata.normalize("NFKC", title).strip().lower()
+    return _SLUG_DROP.sub("", text).replace(" ", "-")
+
+
+class Builder:
+    """Collects headings and units into sections with ordinal-path ids and unique anchors.
+
+    `heading(level, ...)` takes the format's own level; levels may skip (h1 to h3) and ids follow the nesting
+    actually seen, so a skipped level does not leave a gap in the path. Text before the first heading goes in
+    section `0` titled with the document title. With `base="5"` the document is one numbered unit of a larger
+    work (a chapter): `root()` opens section `5` and every later heading is `5.<n>`.
+    """
+
+    def __init__(self, base: str = "") -> None:
+        self._base = base
+        self._ids: list[str] = []
+        self._titles: list[str] = []
+        self._depths: list[int] = []
+        self._anchors_of: list[str] = []
+        self._units: list[list[str]] = []
+        self._stack: list[int] = []  # levels of the open sections
+        self._counts: list[int] = []
+        self._used: set[str] = set()
+        self._current: list[str] | None = None
+        self._preamble_title = ""
+        self._rooted = False
+
+    def set_preamble_title(self, title: str) -> None:
+        self._preamble_title = title
+
+    def _unique(self, anchor: str) -> str:
+        if not anchor:
+            anchor = "section"
+        candidate, n = anchor, 0
+        while candidate in self._used:
+            n += 1
+            candidate = f"{anchor}-{n}"
+        self._used.add(candidate)
+        return candidate
+
+    def _open(self, section_id: str, title: str, depth: int, anchor: str) -> None:
+        units: list[str] = []
+        self._ids.append(section_id)
+        self._titles.append(title)
+        self._depths.append(depth)
+        self._anchors_of.append(self._unique(anchor))
+        self._units.append(units)
+        self._current = units
+
+    def root(self, title: str, anchor: str = "") -> None:
+        """Open the numbered root section (`base`, or `0`) with the document's own title."""
+        self._rooted = True
+        self._stack = []
+        title = clean_title(title)
+        self._open(self._base or "0", title, 1, anchor or slugify(title))
+
+    def heading(self, level: int, title: str, anchor: str = "") -> None:
+        title = clean_title(title)
+        while self._stack and self._stack[-1] >= level:
+            self._stack.pop()
+        depth = len(self._stack) + 1
+        del self._counts[depth:]
+        while len(self._counts) < depth:
+            self._counts.append(0)
+        self._counts[depth - 1] += 1
+        path = ".".join(str(n) for n in self._counts[:depth])
+        section_id = f"{self._base}.{path}" if self._base else path
+        self._stack.append(level)
+        self._open(section_id, title, depth + (1 if self._rooted else 0), anchor or slugify(title))
+
+    def unit(self, text: str) -> None:
+        if not text.strip():
+            return
+        if self._current is None:
+            self._open(self._base or "0", self._preamble_title, 1, slugify(self._preamble_title) or "preamble")
+        if self._current is not None:
+            self._current.append(text)
+
+    def sections(self) -> tuple[Sec, ...]:
+        return tuple(
+            Sec(i, t, d, a, tuple(u))
+            for i, t, d, a, u in zip(
+                self._ids, self._titles, self._depths, self._anchors_of, self._units, strict=True
+            )
+        )
+
+    def doc(self, doc_id: str, title: str, *, unit: Unit = Unit.BLOCK, warnings: tuple[str, ...] = ()) -> Doc:
+        sections = self.sections()
+        if sections and not sections[0].title:
+            first = sections[0]
+            sections = (Sec(first.id, title or doc_id, first.depth, first.anchor, first.units), *sections[1:])
+        return Doc(doc_id, title or doc_id, unit, sections, {}, warnings)

primarysource/pkgdocs/readers/asciidoc.py

@@ -0,0 +1,160 @@
+"""A minimal AsciiDoc reader for git's Documentation: sections, blocks, anchors; nothing is executed or opened.
+
+What it reads:
+
+- the document title (`= Title`, or a line underlined with `=`) is the document's title, not a section;
+- sections by `==` ... `======` or by an underline of `-` (level 1), `~` (2), `^` (3), `+` (4) whose length is
+  within one of the title's, as asciidoc requires;
+- explicit anchors `[[id]]`, `[[id,text]]`, `[#id]` and `[id=...]` before a heading, else the GitHub-style slug;
+- blocks (the unit, so `¶n` counts blocks): a run of non-blank lines; a delimited block (`----`, `....`, `====`,
+  `****`, `____`, `++++`, a `|===` table) as one block with its delimiters dropped; and every labelled-list term
+  (`-a::`) or list marker (`* `, `. `, `1. `) starts a new block even without a blank line before it;
+- a block title (`.Title`) leads the block it titles.
+
+What it does not do, by design:
+
+- `include::path[]` is **never read**. It becomes the block `[include] path`, so the text says where the rest is;
+  the included file is its own document in the same collection;
+- `ifdef::`/`ifndef::`/`ifeval::`/`endif::` lines are dropped and their content kept unconditionally, and the
+  document carries a warning with the count, because the condition's result is the build's, not the text's;
+- attribute entries (`:name: value`), comments (`//`, `////` blocks), block attribute lines (`[verse]`) and the
+  `+` list continuation marker are dropped;
+- inline markup (`linkgit:`, backticks, `'emphasis'`) is left as written.
+"""
+
+from __future__ import annotations
+
+import re
+
+from primarysource.model import Unit
+from primarysource.pkgdocs.model import Doc
+from primarysource.pkgdocs.readers._common import Builder, decode
+
+_UNDERLINE_LEVEL = {"=": 0, "-": 1, "~": 2, "^": 3, "+": 4}
+_ATX = re.compile(r"(={1,6}) +(\S.*)")
+_DELIM = re.compile(r"(-{4,}|\.{4,}|={4,}|\*{4,}|_{4,}|\+{4,}|/{4,}|[|!,:]={3,})")
+_LIST = re.compile(r"\s*(?:\*{1,5}|-|\.{1,5}|\d{1,3}\.|[a-zA-Z]\.)\s+\S")
+_TERM = re.compile(r"\S.*?(?:::|;;|:::|::::)(?:\s|$)")
+_ATTR_ENTRY = re.compile(r":!?[\w][\w-]*!?:(?:\s.*)?")
+_COND_PREFIXES = ("ifdef::", "ifndef::", "ifeval::", "endif::")
+_ANCHOR = re.compile(r"\[\[([A-Za-z_][\w.:-]*)(?:,[^\]]*)?\]\]\s*")
+_ATTRLINE = re.compile(r"\[(?!\[)[^\]]*\]\s*")
+_BLOCKTITLE = re.compile(r"\.[^\s.].*")
+
+
+def _is_underline(line: str, title: str) -> int | None:
+    """The level an underline gives `title`, or None when `line` is not its underline."""
+    if len(line) < 2 or line[0] not in _UNDERLINE_LEVEL or line.count(line[0]) != len(line):
+        return None
+    if abs(len(line) - len(title)) > 1 or not title.strip() or title[0] in " \t[.*-+":
+        return None
+    return _UNDERLINE_LEVEL[line[0]]
+
+
+def read_asciidoc(data: bytes, doc_id: str, title: str | None = None) -> Doc:
+    """Read one AsciiDoc file. Raises `LibError` for NUL bytes or invalid UTF-8; everything else reads."""
+    text = decode(data, f"AsciiDoc file {doc_id!r}")
+    lines = text.expandtabs(8).split("\n")
+    builder = Builder()
+    doc_title = title or ""
+    title_set = bool(title)
+    pending_anchor = ""
+    pending_title = ""
+    conditionals = 0
+    para: list[str] = []
+
+    def flush() -> None:
+        nonlocal pending_title
+        if para:
+            block = "\n".join(para)
+            if pending_title:
+                block = pending_title + "\n" + block
+                pending_title = ""
+            builder.unit(block)
+            para.clear()
+
+    def heading(level: int, name: str) -> None:
+        nonlocal doc_title, title_set, pending_anchor
+        name = name.strip()
+        if level == 0 and not title_set and not doc_title:
+            doc_title = name
+            title_set = True
+            builder.set_preamble_title(name)
+            pending_anchor = ""
+            return
+        builder.heading(max(level, 1), name, pending_anchor)
+        pending_anchor = ""
+
+    i, count = 0, len(lines)
+    while i < count:
+        line = lines[i].rstrip()
+        i += 1
+        stripped = line.strip()
+        if not stripped:
+            flush()
+            continue
+        # an underlined title: exactly one paragraph line above, this line is its underline
+        if len(para) == 1:
+            level = _is_underline(line, para[0])
+            if level is not None and not _LIST.match(para[0]):
+                name = para.pop()
+                heading(level, name)
+                continue
+        if line.startswith(_COND_PREFIXES) and line.endswith("]") and "[" in line:
+            conditionals += 1
+            continue
+        if line.startswith("//") and not line.startswith("///"):
+            continue
+        if _ATTR_ENTRY.fullmatch(stripped) and not para:
+            continue
+        match = _DELIM.fullmatch(line)
+        if match and not (len(para) == 1 and _is_underline(line, para[0]) is not None):
+            flush()
+            body: list[str] = []
+            while i < count and lines[i].rstrip() != line:
+                body.append(lines[i].rstrip())
+                i += 1
+            i += 1  # the closing delimiter, or the end of the file for an unclosed block
+            if line.startswith("/"):
+                continue
+            block = "\n".join(body).strip("\n")
+            if pending_title:
+                block = pending_title + "\n" + block
+                pending_title = ""
+            builder.unit(block)
+            continue
+        if line == "--":
+            flush()
+            continue
+        if line == "+":
+            continue
+        atx = _ATX.fullmatch(line)
+        if atx and not para:
+            heading(len(atx.group(1)) - 1, atx.group(2))
+            continue
+        if line.startswith("include::") and line.endswith("]") and "[" in line:
+            flush()
+            builder.unit("[include] " + line[len("include::") : line.index("[")])
+            continue
+        anchor = _ANCHOR.fullmatch(line)
+        if anchor and not para:
+            pending_anchor = anchor.group(1)
+            continue
+        if _ATTRLINE.fullmatch(line) and not para:
+            found = re.search(r"#([A-Za-z_][\w.:-]*)|\bid=([A-Za-z_][\w.:-]*)", line)
+            if found:
+                pending_anchor = found.group(1) or found.group(2)
+            continue
+        if not para and _BLOCKTITLE.fullmatch(line):
+            pending_title = line
+            continue
+        if para and (_LIST.match(line) or _TERM.match(line) and not line.startswith(" ")):
+            flush()
+        para.append(line)
+    flush()
+    warnings: tuple[str, ...] = ()
+    if conditionals:
+        warnings = (f"{conditionals} ifdef/ifndef/ifeval/endif lines dropped; their content is kept unconditionally",)
+    if not doc_title:
+        doc_title = doc_id
+    return builder.doc(doc_id, doc_title, unit=Unit.BLOCK, warnings=warnings)