primarysource 0.1.0__py3-none-any.whl

primarysource/pkgdocs/readers/latex.py

@@ -0,0 +1,315 @@
+r"""A minimal LaTeX reader for the Gentoo Package Manager Specification, whose source is a set of `.tex` files.
+
+This is not a TeX engine. It reads the structure the specification uses and nothing else:
+
+- `\chapter`, `\section`, `\subsection`, `\subsubsection` (and the starred, unnumbered forms) are sections; the
+  numbers are LaTeX's own (chapters 1, 2, ...; after `\appendix`, chapters A, B, ...; then `8.2.4`), and a
+  `\label{...}` after a heading becomes the anchor, so `\ref{sec:slot-names}` in the text is replaced by the
+  number it names (`section~\ref{...}` reads `section 8.2.4`);
+- `\include{name}` and `\input{name}` are expanded **only from the mapping of files the caller passes in**
+  (a file of the same archive), to a depth of four, never twice in one chain and never from disk. A missing name
+  becomes the block `[include] name`;
+- blocks (the unit): a paragraph; each `\item` of `description`, `compactdesc`, `itemize`, `compactitem`,
+  `enumerate` and `compactenum` (a description item reads `TERM: text`); and each `verbatim`, `listing`,
+  `algorithm`, `tabular`, `centertable`, `longtable` environment as one block written as `[environment]` and the
+  raw text;
+- text between the preamble and the first heading (the title page's copyright and licence notice) is section `0`.
+
+Markup is reduced to its text: `\t{x}`, `\emph{x}`, `\textbf{x}` become `x`; `\cite{k}` becomes `[k]`; `~` is a
+space; `---` and `--` become dashes; escapes (`\_`, `\&`, `\%`, `\$`, `\#`) are resolved. Any other command with
+one argument keeps the argument, any other bare command is dropped, and the document carries one warning naming
+the commands that were dropped, so a reader knows the text was reduced. Math, floats and macros defined in the
+class file are not interpreted. Braces are scanned with a depth counter and a fixed number of passes, so
+unbalanced or deeply nested input costs time linear in its size.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+
+from primarysource.model import Unit
+from primarysource.pkgdocs.model import Doc, Sec
+from primarysource.pkgdocs.readers._common import decode, slugify
+
+MAX_DEPTH = 4
+MAX_LINES = 200_000
+_PASSES = 8
+_LEVELS = {"chapter": 0, "section": 1, "subsection": 2, "subsubsection": 3}
+_LIST_ENVS = frozenset({"description", "compactdesc", "itemize", "compactitem", "enumerate", "compactenum"})
+_RAW_ENVS = frozenset({"verbatim", "listing", "algorithm", "tabular", "centertable", "longtable"})
+_COMMENT = re.compile(r"(?<!\\)%.*$")
+_INCLUDE = re.compile(r"\s*\\(?:include|input)\{([^{}]*)\}\s*")
+_HEADING = re.compile(r"\s*\\(chapter|section|subsection|subsubsection)(\*?)\s*(?:\[[^\]]*\])?\s*\{")
+_BEGIN = re.compile(r"\s*\\begin\{([A-Za-z*]+)\}")
+_END = re.compile(r"\s*\\end\{([A-Za-z*]+)\}")
+_ITEM = re.compile(r"\s*\\item\b\s*(?:\[([^\]]*)\])?\s*(.*)")
+_LABEL = re.compile(r"\s*\\label\{([^{}]*)\}\s*")
+_ONE_ARG = re.compile(r"\\([A-Za-z]+)\*?(?:\[[^\]]*\])?\{([^{}]*)\}")
+_BARE = re.compile(r"\\([A-Za-z]+)\*?")
+_REF = re.compile(r"\\(?:ref|autoref|pageref)\{([^{}]*)\}")
+_LAYOUT = re.compile(
+    r"\s*\\(?:thispagestyle|vspace|tableofcontents|listof\w+|maketitle|bibliography|nobreakpar|clearpage|newpage"
+    r"|noindent|small|centering|vfill|hfill)\b.*"
+)
+_SPACE = re.compile(r"[ \t]+")
+_SYMBOLS = (
+    ("\\textasciicircum", "^"),
+    ("\\textbackslash", "\x01"),
+    ("\\textasciitilde", "~"),
+    ("\\slash", "/"),
+    ("\\allowbreak", ""),
+    ("\\dots", "..."),
+)
+_DROP_ARG = frozenset({"featurelabel", "ChangeWhenAddingAnEAPI", "hspace", "vspace", "index"})
+_KEEP_ARG = frozenset(
+    {"t", "emph", "textbf", "textit", "texttt", "path", "url", "code", "textsc", "textsf", "underline", "mbox"}
+)
+
+
+@dataclass(slots=True)
+class _Section:
+    id: str
+    title: str
+    depth: int
+    anchor: str
+    units: list[tuple[str, bool]] = field(default_factory=list)  # (text, already clean)
+
+
+def _braced(text: str, start: int) -> tuple[str, int]:
+    """The brace-balanced group whose `{` is just before `start`, and the index after its `}`."""
+    depth = 1
+    i = start
+    while i < len(text):
+        char = text[i]
+        if char == "\\":
+            i += 2
+            continue
+        if char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start:i], i + 1
+        i += 1
+    return text[start:], len(text)
+
+
+def _expand(
+    files: Mapping[str, bytes], name: str, chain: tuple[str, ...], out: list[str], missing: list[str]
+) -> None:
+    key = name if name in files else name + ".tex"
+    if key not in files or key in chain or len(chain) >= MAX_DEPTH:
+        missing.append(name)
+        out.append(f"[include] {name}")
+        return
+    for line in decode(files[key], key).split("\n"):
+        if len(out) >= MAX_LINES:
+            return
+        line = _COMMENT.sub("", line).rstrip()
+        found = _INCLUDE.fullmatch(line)
+        if found:
+            _expand(files, found.group(1).strip(), (*chain, key), out, missing)
+        else:
+            out.append(line)
+
+
+def _letter(n: int) -> str:
+    return chr(ord("A") + (n - 1) % 26)
+
+
+def read_latex(files: Mapping[str, bytes], root: str, doc_id: str, title: str) -> Doc:
+    """Read `root` (a key of `files`) with its includes into one document. Raises `LibError` for a NUL byte or
+    invalid UTF-8 in any file it reads; a `root` that is not in `files` is an empty document with a warning."""
+    lines: list[str] = []
+    missing: list[str] = []
+    _expand(files, root, (), lines, missing)
+    started = False
+    appendix = False
+    counters = [0, 0, 0, 0]
+    unnumbered = 0
+    sections: list[_Section] = [_Section("0", title, 1, slugify(title) or "preamble")]
+    labels: dict[str, str] = {}
+    para: list[str] = []
+    item: list[str] | None = None
+    list_depth = 0
+    i, count = 0, len(lines)
+
+    def emit(text: str, clean: bool = False) -> None:
+        if text.strip():
+            sections[-1].units.append((text, clean))
+
+    def flush() -> None:
+        if para:
+            emit(" ".join(para))
+            para.clear()
+
+    def flush_item() -> None:
+        nonlocal item
+        if item is not None:
+            emit(" ".join(item))
+            item = None
+
+    while i < count:
+        line = lines[i]
+        i += 1
+        if not started:
+            if line.strip().startswith("\\begin{document}"):
+                started = True
+            continue
+        if not line.strip():
+            flush()
+            continue
+        if line.strip().startswith("\\end{document}"):
+            break
+        heading = _HEADING.match(line)
+        if heading:
+            flush()
+            flush_item()
+            joined, extra = line, 0
+            while joined.count("{") > joined.count("}") and i < count and extra < 3:
+                joined += " " + lines[i].strip()
+                i += 1
+                extra += 1
+            start = _HEADING.match(joined)
+            if start is None:
+                continue
+            kind, star = start.group(1), start.group(2)
+            text, _ = _braced(joined, start.end())
+            level = _LEVELS[kind]
+            if star:
+                unnumbered += 1
+                section_id = f"u{unnumbered}"
+            else:
+                counters[level] += 1
+                for deeper in range(level + 1, 4):
+                    counters[deeper] = 0
+                first = _letter(counters[0]) if appendix else str(counters[0])
+                section_id = ".".join([first, *(str(n) for n in counters[1 : level + 1])])
+            sections.append(_Section(section_id, _plain(text, labels), level + 1, ""))
+            while i < count and not lines[i].strip():
+                i += 1
+            key = _LABEL.fullmatch(lines[i]) if i < count else None
+            if key:
+                sections[-1].anchor = key.group(1)
+                labels[key.group(1)] = section_id
+                i += 1
+            continue
+        if line.strip() == "\\appendix":
+            flush()
+            appendix = True
+            counters = [0, 0, 0, 0]
+            continue
+        begin = _BEGIN.match(line)
+        if begin and begin.group(1) in _RAW_ENVS:
+            flush()
+            flush_item()
+            env = begin.group(1)
+            raw = [line.strip()]
+            depth = 1
+            while i < count and depth:
+                nxt = lines[i]
+                i += 1
+                raw.append(nxt)
+                inner = _BEGIN.match(nxt)
+                if inner and inner.group(1) == env:
+                    depth += 1
+                closing = _END.match(nxt)
+                if closing and closing.group(1) == env:
+                    depth -= 1
+            emit(f"[{env}]\n" + "\n".join(raw), clean=True)
+            continue
+        if begin and begin.group(1) in _LIST_ENVS:
+            flush()
+            flush_item()
+            list_depth += 1
+            continue
+        end = _END.match(line)
+        if end and end.group(1) in _LIST_ENVS:
+            flush()
+            flush_item()
+            list_depth = max(0, list_depth - 1)
+            continue
+        entry = _ITEM.match(line) if list_depth else None
+        if entry:
+            flush()
+            flush_item()
+            item = ([f"{entry.group(1)}:"] if entry.group(1) is not None else []) + (
+                [entry.group(2)] if entry.group(2) else []
+            )
+            continue
+        if _LAYOUT.fullmatch(line):
+            continue
+        (item if item is not None else para).append(line.strip())
+    flush()
+    flush_item()
+    unknown: dict[str, int] = {}
+    built: list[Sec] = []
+    for section in sections:
+        units = tuple(t if clean else _plain(t, labels, unknown) for t, clean in section.units)
+        units = tuple(u for u in units if u.strip())
+        if section.id == "0" and not units:
+            continue
+        anchor = section.anchor or slugify(section.title) or section.id
+        built.append(Sec(section.id, section.title, section.depth, anchor, units))
+    warnings: list[str] = []
+    if unknown:
+        names = ", ".join(sorted(unknown, key=lambda k: (-unknown[k], k))[:12])
+        warnings.append(f"LaTeX commands dropped (their arguments kept): {names}")
+    if missing:
+        warnings.append(f"not in the archive selection: {', '.join(sorted(set(missing)))}")
+    return Doc(doc_id, title, Unit.BLOCK, _unique(built), {}, tuple(warnings))
+
+
+def _unique(sections: list[Sec]) -> tuple[Sec, ...]:
+    seen: set[str] = set()
+    out: list[Sec] = []
+    for section in sections:
+        anchor, n = section.anchor, 0
+        while anchor in seen:
+            n += 1
+            anchor = f"{section.anchor}-{n}"
+        seen.add(anchor)
+        out.append(Sec(section.id, section.title, section.depth, anchor, section.units))
+    return tuple(out)
+
+
+def _plain(text: str, labels: Mapping[str, str], unknown: dict[str, int] | None = None) -> str:
+    """Reduce LaTeX markup in `text` to its words (see the module docstring)."""
+    text = _REF.sub(lambda m: labels.get(m.group(1), m.group(1)), text)
+    text = re.sub(r"\\label\{[^{}]*\}", "", text)
+    text = re.sub(r"\\cite\{([^{}]*)\}", r"[\1]", text)
+    text = text.replace("\\\\", " ").replace("\\@", "").replace("\\,", " ").replace("\\ ", " ").replace("~", " ")
+    for command, plain in _SYMBOLS:
+        text = text.replace(command, plain)
+    for _ in range(_PASSES):
+
+        def one(match: re.Match[str]) -> str:
+            name, arg = match.group(1), match.group(2)
+            if name == "note":
+                return "Note: " + arg
+            if name in _DROP_ARG:
+                return ""
+            if name not in _KEEP_ARG and unknown is not None:
+                unknown[name] = unknown.get(name, 0) + 1
+            return arg
+
+        after = _ONE_ARG.sub(one, text)
+        if after == text:
+            break
+        text = after
+
+    def bare(match: re.Match[str]) -> str:
+        if unknown is not None:
+            unknown[match.group(1)] = unknown.get(match.group(1), 0) + 1
+        return ""
+
+    text = _BARE.sub(bare, text)
+    for escape in ("_", "&", "%", "$", "#"):
+        text = text.replace("\\" + escape, escape)
+    text = text.replace("\\{", "\x02").replace("\\}", "\x03").replace("{", "").replace("}", "")
+    text = text.replace("\x02", "{").replace("\x03", "}").replace("\x01", "\\")
+    text = text.replace("---", "\u2014").replace("--", "\u2013")
+    text = text.replace("``", '"').replace("''", '"')
+    return _SPACE.sub(" ", text).strip()

primarysource/pkgdocs/readers/markdown.py

@@ -0,0 +1,173 @@
+"""A minimal Markdown reader for rpm's manual and Debian's autopkgtest specification.
+
+Headings are ATX (`#` to `######`) or setext (`===` / `---` under a paragraph). Section ids are the ordinal path
+(`2`, `2.1`); the anchor is the GitHub-style slug of the title, duplicates suffixed `-1`, `-2`. That slug is the
+rule used here; this module does not claim CommonMark defines it. Text before the first heading is section `0`,
+titled with the document title. YAML front matter (a first line `---`) is not a section; its `title:` is used
+when no title is given.
+
+Blocks (the unit): a paragraph (a run of non-blank lines), a fenced code block as one block with its fences, an
+indented code block, a block quote, a table, and each **top-level** list item with everything nested inside it.
+HTML comments are dropped (a `<!-- -->` hiding a `#` line hides it from the outline too); an unclosed fence or
+comment runs to the end of the file, so no text is lost to a typo. Inline markup is left as written.
+"""
+
+from __future__ import annotations
+
+import re
+
+from primarysource.model import Unit
+from primarysource.pkgdocs.model import Doc
+from primarysource.pkgdocs.readers._common import Builder, decode
+
+_FENCE = re.compile(r" {0,3}(`{3,}|~{3,})")
+_ITEM = re.compile(r" {0,3}(?:[-*+]|\d{1,9}[.)])(?:[ \t]+\S|[ \t]*$)")
+_RULE = re.compile(r" {0,3}(?:(?:-[ \t]*){3,}|(?:\*[ \t]*){3,}|(?:_[ \t]*){3,})$")
+_SETEXT = re.compile(r" {0,3}(=+|-+)[ \t]*$")
+_TABLE_SEP = re.compile(r"[ \t|:-]+$")
+_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)")
+
+
+def _atx(line: str) -> tuple[int, str] | None:
+    stripped = line.lstrip(" ")
+    if len(line) - len(stripped) > 3 or not stripped.startswith("#"):
+        return None
+    body = stripped.lstrip("#")
+    level = len(stripped) - len(body)
+    if level > 6 or (body and body[0] not in " \t"):
+        return None
+    title = body.strip()
+    if title.endswith("#"):
+        shorter = title.rstrip("#")
+        if not shorter or shorter[-1] in " \t":
+            title = shorter.strip()
+    return level, title
+
+
+def _front_matter(lines: list[str]) -> tuple[int, str]:
+    """(first line after the front matter, its `title:`), or (0, "") when there is none."""
+    if not lines or lines[0].strip() != "---":
+        return 0, ""
+    for end in range(1, min(len(lines), 200)):
+        if lines[end].strip() in ("---", "..."):
+            title = ""
+            for line in lines[1:end]:
+                key, _, value = line.partition(":")
+                if key.strip() == "title" and value.strip():
+                    title = value.strip().strip("\"'")
+            return end + 1, title
+    return 0, ""
+
+
+def read_markdown(data: bytes, doc_id: str, title: str | None = None) -> Doc:
+    """Read one Markdown file. Raises `LibError` for NUL bytes or invalid UTF-8; everything else reads."""
+    text = decode(data, f"Markdown file {doc_id!r}")
+    lines = [ln.rstrip() for ln in text.expandtabs(4).split("\n")]
+    i, fm_title = _front_matter(lines)
+    doc_title = title or fm_title
+    builder = Builder()
+    first_h1 = ""
+    count = len(lines)
+    para: list[str] = []
+
+    def flush() -> None:
+        if para:
+            builder.unit("\n".join(para))
+            para.clear()
+
+    def heading(level: int, name: str) -> None:
+        nonlocal first_h1
+        name = _LINK.sub(chr(92) + "1", name)
+        first_h1 = first_h1 or (name if level == 1 else "")
+        builder.heading(level, name)
+
+    while i < count:
+        line = lines[i]
+        i += 1
+        if not line.strip():
+            flush()
+            continue
+        fence = _FENCE.match(line)
+        if fence:
+            flush()
+            marker = fence.group(1)
+            block = [line]
+            while i < count:
+                closing = lines[i].strip()
+                block.append(lines[i])
+                i += 1
+                if closing.startswith(marker[0] * len(marker)) and closing.strip(marker[0]) == "":
+                    break
+            builder.unit("\n".join(block).rstrip("\n"))
+            continue
+        if line.lstrip().startswith("<!--"):
+            flush()
+            i -= 1
+            while i < count and "-->" not in lines[i]:
+                i += 1
+            i += 1
+            continue
+        atx = _atx(line)
+        if atx:
+            flush()
+            if atx[1]:
+                heading(*atx)
+            continue
+        setext = _SETEXT.match(line)
+        if setext and para:
+            name = " ".join(para)
+            para.clear()
+            heading(1 if setext.group(1)[0] == "=" else 2, name)
+            continue
+        if _RULE.match(line) and not para:
+            continue
+        if para:
+            starts = _ITEM.match(line) or line.lstrip().startswith(">") or "|" in line
+            if not starts:
+                para.append(line)
+                continue
+            flush()
+        if _ITEM.match(line):
+            item = [line]
+            while i < count:
+                nxt = lines[i]
+                if not nxt.strip():
+                    j = i
+                    while j < count and not lines[j].strip():
+                        j += 1
+                    if j < count and (len(lines[j]) - len(lines[j].lstrip(" "))) >= 2:
+                        item.extend([""] * (j - i))
+                        i = j
+                        continue
+                    break
+                indented = (len(nxt) - len(nxt.lstrip(" "))) >= 2
+                if not indented and (_ITEM.match(nxt) or _atx(nxt) or _FENCE.match(nxt) or _RULE.match(nxt)):
+                    break
+                item.append(nxt)
+                i += 1
+            builder.unit("\n".join(item))
+            continue
+        if line.lstrip().startswith(">"):
+            block = [line]
+            while i < count and lines[i].strip() and (lines[i].lstrip().startswith(">") or not _atx(lines[i])):
+                block.append(lines[i])
+                i += 1
+            builder.unit("\n".join(block))
+            continue
+        if "|" in line and i < count and _TABLE_SEP.match(lines[i]) and "-" in lines[i] and "|" in lines[i]:
+            block = [line]
+            while i < count and lines[i].strip() and "|" in lines[i]:
+                block.append(lines[i])
+                i += 1
+            builder.unit("\n".join(block))
+            continue
+        if line.startswith("    ") and not para:
+            block = [line]
+            while i < count and (lines[i].startswith("    ") or not lines[i].strip()):
+                block.append(lines[i])
+                i += 1
+            builder.unit("\n".join(block).rstrip())
+            continue
+        para.append(line)
+    flush()
+    return builder.doc(doc_id, doc_title or first_h1 or doc_id, unit=Unit.BLOCK)

primarysource/pkgdocs/readers/pod.py

@@ -0,0 +1,132 @@
+"""A minimal POD reader for the dpkg manual pages, which are `.pod` files in dpkg's `man/` directory.
+
+`=head1`..`=head4` are sections (level 1 to 4). Blocks, the unit: an ordinary paragraph, a verbatim (indented)
+paragraph, and an `=item` with every paragraph up to the next item (so a tag and its description are one block,
+the way a man page's `.TP` is). `=over` and `=back` only delimit lists; nesting is flattened. `=begin`/`=end`
+regions and `=for` lines are dropped, as are `=encoding`, `=pod`, `=cut` and the `#` licence comment that dpkg's
+files carry before the first command.
+
+Formatting codes (`B<>`, `I<>`, `C<>`, `F<>`, `S<>`, `L<>`, `E<>`, `X<>`, `Z<>`) are resolved to their text,
+including the `C<< ... >>` form, in a fixed number of passes. A paragraph over `MAX_CODED` characters is left
+unresolved and the document carries a warning, so a hostile run of `C<<` costs time linear in the input.
+"""
+
+from __future__ import annotations
+
+import re
+
+from primarysource.model import Unit
+from primarysource.pkgdocs.model import Doc
+from primarysource.pkgdocs.readers._common import Builder, decode
+
+MAX_CODED = 20_000
+_PASSES = 12
+_ENTITIES = {"lt": "<", "gt": ">", "verbar": "|", "sol": "/", "amp": "&", "quot": '"', "apos": "'", "nbsp": " "}
+_WIDE = re.compile(r"([A-Z])(<{2,}) ([^<>]*?) (>{2,})")
+_NARROW = re.compile(r"([A-Z])<([^<>]*)>")
+_CMD = re.compile(r"=([a-z]+\d?)(?:[ \t]+(.*))?$", re.DOTALL)
+
+
+def _entity(name: str) -> str:
+    if name in _ENTITIES:
+        return _ENTITIES[name]
+    if name.isdigit():
+        return chr(int(name)) if 0 < int(name) < 0x110000 else ""
+    if name.lower().startswith("0x") and re.fullmatch(r"0[xX][0-9a-fA-F]{1,6}", name):
+        return chr(int(name, 16)) if int(name, 16) < 0x110000 else ""
+    return f"&{name};"
+
+
+def _code(match: re.Match[str]) -> str:
+    kind, inner = match.group(1), match.group(2) if match.re is _NARROW else match.group(3)
+    if kind == "E":
+        return _entity(inner.strip())
+    if kind in ("X", "Z"):
+        return ""
+    if kind == "L":
+        label = inner.split("|", 1)[0] if "|" in inner else inner
+        return label
+    if kind in ("B", "I", "C", "F", "S", "U"):
+        return inner
+    return match.group(0)
+
+
+def resolve_codes(text: str) -> str:
+    """Resolve POD formatting codes to their text. Unknown letters are left as written."""
+    for _ in range(_PASSES):
+        after = _NARROW.sub(_code, _WIDE.sub(_code, text))
+        if after == text:
+            break
+        text = after
+    return text
+
+
+def read_pod(data: bytes, doc_id: str, title: str | None = None) -> Doc:
+    """Read one POD file. Raises `LibError` for NUL bytes or invalid UTF-8."""
+    text = decode(data, f"POD file {doc_id!r}")
+    builder = Builder()
+    uncoded = 0
+    paragraphs = re.split(r"\n[ \t]*\n+", text.strip("\n"))
+    item: list[str] | None = None
+    skip_region = False
+    doc_title = title or ""
+    in_name = False
+    started = False  # the first `=` command seen
+
+    def emit(block: str) -> None:
+        nonlocal item
+        if item is not None:
+            item.append(block)
+        else:
+            builder.unit(block)
+
+    def close_item() -> None:
+        nonlocal item
+        if item is not None:
+            builder.unit("\n\n".join(item))
+            item = None
+
+    def coded(block: str) -> str:
+        nonlocal uncoded
+        if len(block) > MAX_CODED:
+            uncoded += 1
+            return block
+        return resolve_codes(block)
+
+    for paragraph in paragraphs:
+        if not started and not paragraph.startswith("="):
+            lines = [ln for ln in paragraph.split("\n") if not ln.startswith("#")]
+            if not lines:
+                continue  # the licence comment block
+            paragraph = "\n".join(lines)
+        command = _CMD.match(paragraph) if paragraph.startswith("=") else None
+        if skip_region:
+            skip_region = not (command and command.group(1) == "end")
+            continue
+        if command:
+            started = True
+            name, argument = command.group(1), (command.group(2) or "").strip()
+            if name.startswith("head") and name[4:].isdigit():
+                close_item()
+                title_text = coded(argument.replace("\n", " "))
+                in_name = title_text.strip().upper() == "NAME"
+                builder.heading(int(name[4:]), title_text)
+            elif name == "item":
+                close_item()
+                item = [coded(argument)]
+            elif name in ("back",):
+                close_item()
+            elif name == "begin":
+                skip_region = True
+            continue
+        if paragraph.lstrip().startswith("="):
+            continue
+        body = coded(paragraph.rstrip()) if paragraph[:1] not in " \t" else paragraph.rstrip()
+        if in_name and not doc_title:
+            doc_title = " ".join(body.split())
+        in_name = False
+        emit(body)
+    close_item()
+    warnings = (f"{uncoded} paragraphs over {MAX_CODED} characters left unresolved",) if uncoded else ()
+    builder.set_preamble_title(doc_title or doc_id)
+    return builder.doc(doc_id, doc_title or doc_id, unit=Unit.BLOCK, warnings=warnings)