doc-control 0.0.1.dev0__py3-none-any.whl

doc_control/model/index.py

@@ -0,0 +1,353 @@
+"""The paragraph index of ``word/document.xml``: visible text mapped back to runs.
+
+:func:`build_index` takes the paragraph spans from :func:`doc_control.ooxml.splice.scan`
+and, for each paragraph of the main story, computes its *visible text* and a list
+of :class:`Piece` records saying which source element produced which characters.
+
+Main story
+    Numbering follows the spec: paragraphs in document order, table-cell and
+    content-control paragraphs included. A paragraph inside another paragraph
+    (a text box) belongs to a different story: it is not numbered and its text
+    is not part of the enclosing paragraph.
+
+Visible text
+    Included: ``w:t`` text, ``w:tab`` as ``\\t``, ``w:br`` and ``w:cr`` as ``\\n``
+    (page and column breaks add nothing), ``w:noBreakHyphen`` as U+2011,
+    ``w:softHyphen`` as U+00AD; text inside ``w:ins``/``w:moveTo``, hyperlinks,
+    smart tags, content controls and field results.
+    Excluded: ``w:delText`` and everything in ``w:del``/``w:moveFrom``, field
+    instructions (between a field's ``begin`` and ``separate``, nested fields
+    included, and across paragraphs for a field that spans them), and runs whose
+    own ``w:rPr`` has ``w:vanish`` switched on. Hidden text that comes from a
+    character style is not resolved here (that needs ``styles.xml``).
+    Also shown: ``w:sym`` as its one ``w:char`` character, ``w:ptab`` as ``\\t``,
+    the base text of ``w:ruby`` (not the ``w:rt`` annotation) and, in a
+    paragraph-level ``mc:AlternateContent``, exactly one branch: the first
+    ``mc:Choice`` whose ``Requires`` prefixes resolve (in the paragraph's
+    namespace scope) to a Word namespace, otherwise the ``mc:Fallback``.
+    Not read: equations (``m:oMath``), which are not prose, and anything in a
+    text box, drawing or picture.
+
+Offsets
+    Offsets are indexes into the visible text as a Python ``str``, that is code
+    points, not user-perceived characters: a combining mark is its own offset, so
+    a range can split it from its base letter. What a write does about that is
+    decided by the edit operations (``doc_control.ops.tracked``), not here.
+"""
+
+from __future__ import annotations
+
+import bisect
+import xml.etree.ElementTree as ET
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+
+from doc_control.errors import DocControlError, ErrorCode
+from doc_control.model.address import Address, AddressRange, fingerprint, parse_address, parse_range
+from doc_control.model.segment import DEFAULT_LANGUAGE, sentence_span
+from doc_control.ooxml.splice import MC_NS, W_NS, DocumentSpans, scan
+from doc_control.ooxml.zipio import Package
+
+W = f"{{{W_NS}}}"
+MC = f"{{{MC_NS}}}"
+DOCUMENT_PART = "word/document.xml"
+
+# Containers whose children never carry visible body text.
+_SKIP = frozenset(
+    W + name
+    for name in ("pPr", "rPr", "sdtPr", "sdtEndPr", "del", "moveFrom", "txbxContent", "delText")
+)
+# Namespaces whose markup Word understands, for choosing an mc:Choice branch.
+_WORD_NAMESPACES = (W_NS, "http://schemas.microsoft.com/office/word/")
+_VANISH_OFF = frozenset({"0", "false", "off"})
+_BREAKS_WITHOUT_TEXT = frozenset({"page", "column"})
+_FIXED_TEXT = {
+    W + "tab": "\t",
+    W + "cr": "\n",
+    W + "noBreakHyphen": "\N{NON-BREAKING HYPHEN}",
+    W + "softHyphen": "\N{SOFT HYPHEN}",
+    W + "ptab": "\t",
+}
+
+_Emit = Callable[[str, tuple[int, ...]], None]
+
+
+@dataclass(frozen=True)
+class Piece:
+    """Visible characters ``[start, end)`` of a paragraph and where they come from.
+
+    ``path`` lists child indexes from the ``w:p`` element (as returned by
+    :meth:`DocumentSpans.parse_paragraph`, comments counted) down to the source
+    element: a ``w:t``, ``w:tab``, ``w:br``… ``path[:-1]`` is its run. For a
+    ``w:t``, character ``i`` of the paragraph is character ``i - start`` of the
+    element's text. ``in_field`` is true for text inside a field's result (also a
+    field that began in an earlier paragraph).
+    """
+
+    start: int
+    end: int
+    path: tuple[int, ...]
+    in_field: bool = False
+
+
+@dataclass(frozen=True)
+class Paragraph:
+    """One indexed paragraph of the main story."""
+
+    number: int  # 1-based
+    span_index: int  # position in DocumentSpans.paragraphs
+    text: str
+    fp: str
+    pieces: tuple[Piece, ...] = field(repr=False)
+
+    @property
+    def address(self) -> Address:
+        return Address(self.number, self.fp)
+
+    def pieces_in(self, start: int, end: int) -> list[Piece]:
+        """The pieces overlapping visible characters ``[start, end)``."""
+        if start >= end:
+            return []
+        first = max(bisect.bisect_right(self.pieces, start, key=lambda p: p.start) - 1, 0)
+        out: list[Piece] = []
+        for piece in self.pieces[first:]:
+            if piece.start >= end:
+                break
+            if piece.end > start:
+                out.append(piece)
+        return out
+
+
+@dataclass(frozen=True)
+class ResolvedRange:
+    """A range checked against the document, with offsets filled in.
+
+    A whole-paragraph range covers ``0`` to the end of the last paragraph. A sentence
+    range (``.s<k>``) gets the offsets of that sentence, found by segmentation.
+    """
+
+    range: AddressRange
+    first: Paragraph
+    last: Paragraph
+    start_offset: int | None
+    end_offset: int | None
+
+    @property
+    def sentence(self) -> int | None:
+        return self.range.sentence
+
+
+def _vanished(run: ET.Element) -> bool:
+    rpr = run.find(W + "rPr")
+    vanish = rpr.find(W + "vanish") if rpr is not None else None
+    return vanish is not None and vanish.get(W + "val", "1").lower() not in _VANISH_OFF
+
+
+class _Walker:
+    """Reads paragraph text in document order, carrying field state across paragraphs."""
+
+    def __init__(self) -> None:
+        # One entry per open field: True once its ``separate`` was seen (result part).
+        self.fields: list[bool] = []
+
+    def paragraph(
+        self, root: ET.Element, scope: Mapping[str, str]
+    ) -> tuple[str, tuple[Piece, ...]]:
+        parts: list[str] = []
+        pieces: list[Piece] = []
+        pos = 0
+
+        def emit(text: str, path: tuple[int, ...]) -> None:
+            nonlocal pos
+            if text:
+                parts.append(text)
+                pieces.append(Piece(pos, pos + len(text), path, bool(self.fields)))
+                pos += len(text)
+
+        self.walk(root, (), scope, emit)
+        return "".join(parts), tuple(pieces)
+
+    def walk(
+        self, elem: ET.Element, path: tuple[int, ...], scope: Mapping[str, str], emit: _Emit
+    ) -> None:
+        for i, child in enumerate(elem):
+            tag = child.tag
+            if not isinstance(tag, str) or tag in _SKIP:
+                continue
+            if tag == W + "r":
+                self.run(child, (*path, i), scope, emit)
+            elif tag == MC + "AlternateContent":
+                branch = _branch(child, scope)
+                if branch is not None:
+                    j, chosen = branch
+                    self.walk(chosen, (*path, i, j), scope, emit)
+            elif tag.startswith(W):
+                self.walk(child, (*path, i), scope, emit)
+
+    def run(
+        self, run: ET.Element, path: tuple[int, ...], scope: Mapping[str, str], emit: _Emit
+    ) -> None:
+        hidden = _vanished(run)
+        for i, child in enumerate(run):
+            tag = child.tag
+            if not isinstance(tag, str):
+                continue
+            if tag == W + "fldChar":
+                self.field_char(child.get(W + "fldCharType"))
+            elif hidden or not all(self.fields):
+                continue
+            elif tag == W + "t":
+                emit("".join(child.itertext()), (*path, i))
+            elif tag in _FIXED_TEXT:
+                emit(_FIXED_TEXT[tag], (*path, i))
+            elif tag == W + "br" and child.get(W + "type") not in _BREAKS_WITHOUT_TEXT:
+                emit("\n", (*path, i))
+            elif tag == W + "sym":
+                emit(_symbol(child), (*path, i))
+            elif tag == W + "ruby":
+                for j, part in enumerate(child):
+                    if part.tag == W + "rubyBase":  # the annotation (w:rt) is not body text
+                        self.walk(part, (*path, i, j), scope, emit)
+
+    def field_char(self, kind: str | None) -> None:
+        if kind == "begin":
+            self.fields.append(False)
+        elif kind == "separate" and self.fields:
+            self.fields[-1] = True
+        elif kind == "end" and self.fields:
+            self.fields.pop()
+
+
+def _symbol(sym: ET.Element) -> str:
+    """The one character of a ``w:sym`` (its ``w:char`` hex code); empty if malformed."""
+    try:
+        code = int(sym.get(W + "char", ""), 16)
+    except ValueError:
+        return ""
+    if code == 0 or 0xD800 <= code <= 0xDFFF or code > 0x10FFFF:
+        return ""
+    return chr(code)
+
+
+def _branch(alt: ET.Element, scope: Mapping[str, str]) -> tuple[int, ET.Element] | None:
+    """The single branch of an ``mc:AlternateContent`` to read: never both.
+
+    The first ``mc:Choice`` whose ``Requires`` prefixes all resolve, in the
+    paragraph's namespace scope, to a Word namespace; otherwise the Fallback.
+    """
+    fallback: tuple[int, ET.Element] | None = None
+    for j, child in enumerate(alt):
+        if child.tag == MC + "Choice":
+            prefixes = child.get("Requires", "").split()
+            if prefixes and all(_understood(scope.get(p)) for p in prefixes):
+                return j, child
+        elif child.tag == MC + "Fallback" and fallback is None:
+            fallback = j, child
+    return fallback
+
+
+def _understood(uri: str | None) -> bool:
+    return uri is not None and uri.startswith(_WORD_NAMESPACES)
+
+
+class DocumentIndex:
+    """Every main-story paragraph of one ``word/document.xml``, addressable."""
+
+    def __init__(self, spans: DocumentSpans, paragraphs: Sequence[Paragraph]) -> None:
+        self.spans = spans
+        self.paragraphs: tuple[Paragraph, ...] = tuple(paragraphs)
+
+    def __len__(self) -> int:
+        return len(self.paragraphs)
+
+    def paragraph(self, number: int) -> Paragraph:
+        """Paragraph ``number`` (1-based); ``STALE_ADDRESS`` when there is none."""
+        if not 1 <= number <= len(self.paragraphs):
+            raise DocControlError(
+                ErrorCode.STALE_ADDRESS,
+                f"p{number} is outside the document ({len(self.paragraphs)} paragraphs)",
+                number=number,
+                paragraphs=len(self.paragraphs),
+            )
+        return self.paragraphs[number - 1]
+
+    def resolve_address(self, address: Address | str) -> Paragraph:
+        """The paragraph at ``address``; ``STALE_ADDRESS`` unless position and fingerprint match."""
+        if isinstance(address, str):
+            address = parse_address(address)
+        para = self.paragraph(address.number)
+        if para.fp != address.fp:
+            raise DocControlError(
+                ErrorCode.STALE_ADDRESS,
+                f"{address} no longer matches the document",
+                address=str(address),
+                current=str(para.address),
+            )
+        return para
+
+    def resolve_range(
+        self, value: AddressRange | str, language: str = DEFAULT_LANGUAGE
+    ) -> ResolvedRange:
+        """Check a range against the document and fill in its offsets.
+
+        ``STALE_ADDRESS`` for a position or fingerprint that does not match;
+        ``ADDRESS_INVALID`` for an offset beyond its paragraph's text or a sentence
+        number beyond the paragraph's sentences (split per ``language``).
+        """
+        rng = parse_range(value) if isinstance(value, str) else value
+        first = self.resolve_address(rng.start)
+        last = first if rng.end == rng.start else self.resolve_address(rng.end)
+        start, end = rng.start_offset, rng.end_offset
+        if rng.sentence is not None:
+            span = sentence_span(first.text, rng.sentence, language)
+            if span is None:
+                raise DocControlError(
+                    ErrorCode.ADDRESS_INVALID,
+                    f"{rng} names a sentence the paragraph does not have",
+                    address=str(rng),
+                )
+            start, end = span.start, span.end
+        else:
+            if start is None or end is None:
+                start, end = 0, len(last.text)
+            elif start > len(first.text) or end > len(last.text) or (first is last and start > end):
+                raise DocControlError(
+                    ErrorCode.ADDRESS_INVALID,
+                    f"{rng} has an offset beyond its paragraph's text",
+                    address=str(rng),
+                )
+        return ResolvedRange(rng, first, last, start, end)
+
+    @classmethod
+    def from_package(cls, pkg: Package) -> DocumentIndex:
+        return build_index(pkg.read(DOCUMENT_PART))
+
+
+def build_index(data: bytes) -> DocumentIndex:
+    """Index the bytes of a ``word/document.xml``; tokenizer errors propagate.
+
+    Field state carries from one paragraph to the next so a field that spans
+    paragraphs (a table of contents) keeps its instruction hidden. If the body
+    ends with a field still open (a ``begin`` with no ``end``), that carry would
+    blank every later paragraph, so the document is indexed again with the field
+    state reset at each paragraph.
+    """
+    spans = scan(data)
+    paragraphs, open_fields = _paragraphs(spans, carry=True)
+    if open_fields:
+        paragraphs, _ = _paragraphs(spans, carry=False)
+    return DocumentIndex(spans, paragraphs)
+
+
+def _paragraphs(spans: DocumentSpans, *, carry: bool) -> tuple[list[Paragraph], bool]:
+    walker = _Walker()
+    paragraphs: list[Paragraph] = []
+    for span in spans.paragraphs:
+        if span.nested:
+            continue
+        if not carry:
+            walker.fields.clear()
+        text, pieces = walker.paragraph(spans.parse_paragraph(span.index), span.scope)
+        number = len(paragraphs) + 1
+        paragraphs.append(Paragraph(number, span.index, text, fingerprint(text), pieces))
+    return paragraphs, bool(walker.fields)

doc_control/model/layout.py

@@ -0,0 +1,143 @@
+"""Layout facts of a paragraph (D-18): what it looks like on the page.
+
+Plain functions of a :class:`~doc_control.model.document.Document` and a paragraph, so the
+read operations (T31, :mod:`doc_control.ops.layout`) and structure detection share one
+source of truth.
+
+``empty``
+    No visible text, or only whitespace (non-breaking spaces included), and no picture
+    (``w:drawing``, ``w:pict``, ``w:object``): an image-only paragraph is not empty.
+``blank_before``
+    How many empty paragraphs sit immediately before this one (0 when none), within
+    the same table cell or the same body level: counting stops at a table edge, at an
+    image-only paragraph and at a page-break-only paragraph (which is not blank).
+``heading_level``
+    1-9, from a direct outline level, else the style chain (``basedOn``), else a style
+    named ``heading N``; ``None`` for body text. (An outline entry's ``level``.)
+``style`` / ``style_name``
+    The paragraph's style id (explicit, else the default paragraph style) and its name;
+    ``para.get`` and ``outline`` already report them.
+``centred``
+    Centre alignment, direct (``w:jc``) or inherited through the style chain. Text
+    "centred" with tabs or spaces is not detected: that is a recipe (T25).
+``page_break_before``
+    The paragraph starts a page: ``w:pageBreakBefore`` (direct wins over the style
+    chain), a page break (``w:br w:type="page"``) before any visible content, the
+    previous paragraph (same cell or body level) ending in a page break or being only
+    one, or a section break of type nextPage, oddPage, evenPage or none (not
+    continuous or nextColumn) ending the previous paragraph.
+
+Alignment is resolved as direct, then style chain, then ``w:docDefaults``. Not covered:
+alignment from a table style or from numbering levels; ``w:lastRenderedPageBreak``.
+"""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from collections.abc import Iterator
+
+from doc_control.model.document import Document
+from doc_control.model.format import para_format
+from doc_control.model.index import Paragraph
+from doc_control.model.parts import W
+
+_NOT_CONTENT = frozenset(W + name for name in ("pPr", "del", "moveFrom", "delText"))
+_CONTENT = frozenset(W + name for name in ("t", "tab", "drawing", "pict", "object", "sym"))
+_IMAGES = frozenset(W + name for name in ("drawing", "pict", "object"))
+_PAGE_SECTIONS = frozenset({"nextPage", "oddPage", "evenPage"})  # not continuous / nextColumn
+
+
+def _events(elem: ET.Element) -> Iterator[bool]:
+    """Visible items of a paragraph in order: True for a page break, False for content."""
+    for child in elem:
+        tag = child.tag
+        if not isinstance(tag, str) or tag in _NOT_CONTENT:
+            continue
+        if tag == W + "br":
+            if child.get(W + "type") == "page":
+                yield True
+        elif tag in _CONTENT:
+            if tag != W + "t" or "".join(child.itertext()).strip():
+                yield False
+        else:
+            yield from _events(child)
+
+
+def _has_image(elem: ET.Element) -> bool:
+    for child in elem:
+        tag = child.tag
+        if not isinstance(tag, str) or tag in _NOT_CONTENT:
+            continue
+        if tag in _IMAGES or _has_image(child):
+            return True
+    return False
+
+
+def _elem(doc: Document, para: Paragraph) -> ET.Element:
+    return doc.index.spans.parse_paragraph(para.span_index)
+
+
+def is_empty(doc: Document, para: Paragraph) -> bool:
+    return not para.text.strip() and not _has_image(_elem(doc, para))
+
+
+def _break_only(doc: Document, para: Paragraph) -> bool:
+    items = list(_events(_elem(doc, para)))
+    return bool(items) and all(items) and not para.text.strip()
+
+
+def _previous(doc: Document, para: Paragraph) -> Paragraph | None:
+    """The paragraph before this one in the same table cell (or the same body level)."""
+    if para.number < 2 or doc.cells[para.number - 2] != doc.cells[para.number - 1]:
+        return None
+    return doc.index.paragraphs[para.number - 2]
+
+
+def blank_before(doc: Document, para: Paragraph) -> int:
+    count = 0
+    prev = _previous(doc, para)
+    while prev is not None and is_empty(doc, prev) and not _break_only(doc, prev):
+        count += 1
+        prev = _previous(doc, prev)
+    return count
+
+
+def centred(doc: Document, para: Paragraph) -> bool:
+    alignment = para_format(_elem(doc, para).find(W + "pPr")).get("alignment")
+    if alignment is None:
+        style = doc.style_id(para)
+        alignment = doc.styles.effective(style)[0].get("alignment") if style else None
+    if alignment is None:
+        alignment = doc.styles.defaults_ppr.get("alignment")
+    return alignment == "center"
+
+
+def _section_break_before(prev: ET.Element) -> bool:
+    sect = prev.find(f"{W}pPr/{W}sectPr")
+    if sect is None:
+        return False
+    kind = sect.find(W + "type")
+    return (kind.get(W + "val", "nextPage") if kind is not None else "nextPage") in _PAGE_SECTIONS
+
+
+def page_break_before(doc: Document, para: Paragraph) -> bool:
+    elem = _elem(doc, para)
+    direct = para_format(elem.find(W + "pPr")).get("page_break_before")
+    if direct is None:
+        style = doc.style_id(para)
+        direct = doc.styles.effective(style)[0].get("page_break_before") if style else None
+    if direct:
+        return True
+    first = next(_events(elem), False)
+    if first:
+        return True
+    prev = _previous(doc, para)
+    if prev is None:
+        return False
+    prev_elem = _elem(doc, prev)
+    if _section_break_before(prev_elem):
+        return True
+    last = None
+    for item in _events(prev_elem):
+        last = item
+    return last is True

doc_control/model/normalized.py

@@ -0,0 +1,160 @@
+"""Normalised view of visible paragraph text, with a two-way offset map.
+
+Offsets are code points of the visible text, as in :mod:`doc_control.model.index`; the
+end of a range is exclusive. :func:`normalize` applies, in this order:
+
+1. removal of soft hyphens (U+00AD) and zero-width characters (U+200B, U+200C, U+200D,
+   U+2060, U+FEFF);
+2. NFC;
+3. collapse of every run of whitespace (``str.isspace``: space, tab, newline, no-break
+   space U+00A0, U+2028...) into one space. Nothing is trimmed.
+
+Zero-width characters go even where they hold a sequence together: U+200D (ZWJ) is
+removed from an emoji sequence, which then reads as its separate emoji. NFC is applied to
+clusters of at most 32 code points, so a longer run of marks (not found in real text) is
+normalised in pieces.
+
+Every normalised character covers a half-open range of original characters (if nothing
+is left, the only boundary maps to 0). A removed
+character belongs to the character that follows it (the trailing ones to the last
+character), a collapsed run to its single space, and the characters NFC combines (``e`` +
+U+0301 -> ``e`` with acute) to the combined character.
+
+``to_original[i]`` is the original offset of normalised boundary ``i`` (``0..len(text)``)
+and ``to_normalized[o]`` the normalised offset of original boundary ``o``
+(``0..len(original)``). So ``text[a:b]`` comes from ``original[to_original[a]:to_original[b]]``.
+Boundaries that exist on both sides survive a round trip unchanged. A boundary with no
+counterpart (inside a collapsed whitespace run, between a letter and the accent NFC folds
+into it, or inside the two characters one original character expands to) maps to the
+nearest boundary before it, so both maps are monotonic and
+``to_original[to_normalized[to_original[i]]] == to_original[i]`` always holds.
+"""
+
+from __future__ import annotations
+
+import bisect
+import unicodedata
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+#: Longest run of code points normalised together. Real clusters are a few characters;
+#: Unicode's stream-safe text format allows 31 combining marks after a base. A longer run
+#: (U+0F73 repeated, say) is cut here, where NFC of the whole would cost quadratic time.
+MAX_CLUSTER = 32
+IGNORABLE = frozenset("\u00ad\u200b\u200c\u200d\u2060\ufeff")
+
+
+@dataclass(frozen=True)
+class NormalizedText:
+    """The normalised text and the offset maps between it and the original."""
+
+    text: str
+    original_length: int
+    to_original: tuple[int, ...]
+    to_normalized: tuple[int, ...]
+
+    def original_offset(self, offset: int) -> int:
+        """Original offset of normalised boundary ``offset``."""
+        if not 0 <= offset <= len(self.text):
+            raise IndexError(f"normalised offset {offset} outside 0..{len(self.text)}")
+        return self.to_original[offset]
+
+    def normalized_offset(self, offset: int) -> int:
+        """Normalised offset of original boundary ``offset``."""
+        if not 0 <= offset <= self.original_length:
+            raise IndexError(f"original offset {offset} outside 0..{self.original_length}")
+        return self.to_normalized[offset]
+
+
+def normalize(original: str) -> NormalizedText:
+    """Normalise ``original`` and map offsets both ways."""
+    kept = [i for i, ch in enumerate(original) if ch not in IGNORABLE]
+    # Output characters as [start, end) original ranges. A character's range starts
+    # where the previous one ended, so removed characters attach to the next one.
+    chars: list[str] = []
+    starts: list[int] = []
+    ends: list[int] = []
+    prev_end = 0
+    chunks = _chunks(original, kept)
+    for pos, chunk in enumerate(chunks):
+        out = unicodedata.normalize("NFC", "".join(original[i] for i in chunk))
+        # Removed characters at the very end belong to the last chunk.
+        end = len(original) if pos == len(chunks) - 1 else chunk[-1] + 1
+        for ch in out:
+            if ch.isspace() and chars and chars[-1] == " ":
+                ends[-1] = end  # collapse into the previous space
+                continue
+            chars.append(" " if ch.isspace() else ch)
+            starts.append(prev_end)
+            ends.append(end)
+        prev_end = end
+
+    # A chunk's characters all share its range, so an interior boundary sits at its start.
+    to_original = (*starts, len(original) if chars else 0)
+    to_normalized = tuple(bisect.bisect_right(ends, o) for o in range(len(original) + 1))
+    return NormalizedText("".join(chars), len(original), to_original, to_normalized)
+
+
+def _chunks(original: str, kept: list[int]) -> list[list[int]]:
+    """Split the kept indexes where NFC cannot combine across the cut.
+
+    A character joins the current chunk when it has a combining class, or when NFC of
+    the chunk plus the character differs from NFC of the chunk followed by the
+    character (a base letter and its accent, or Hangul L+V+T, which compose step by
+    step). Whitespace stands alone: nothing composes with it, and a collapsed space
+    must stay whole. A chunk stops growing at :data:`MAX_CLUSTER` code points.
+    """
+    chunks: list[list[int]] = []
+    text = ""  # the current chunk's characters
+    for i in kept:
+        ch = original[i]
+        if (
+            chunks
+            and len(text) < MAX_CLUSTER
+            and not text[-1].isspace()
+            and (
+                unicodedata.combining(ch) != 0
+                or (
+                    not (ch.isascii() and text.isascii())
+                    and unicodedata.normalize("NFC", text + ch)
+                    != unicodedata.normalize("NFC", text) + unicodedata.normalize("NFC", ch)
+                )
+            )
+        ):
+            chunks[-1].append(i)
+            text += ch
+        else:
+            chunks.append([i])
+            text = ch
+    return chunks
+
+
+def encode_runs(values: Sequence[int], lo: int, hi: int) -> list[list[int]]:
+    """Compact form of ``values[lo:hi]`` as runs ``[index, value, count]``.
+
+    Entry ``index + k`` is ``value + k`` for ``k < count``. A run whose values do not
+    move is written ``[index, value, count, 0]``: entry ``index + k`` is ``value``.
+    """
+    runs: list[list[int]] = []
+    i = lo
+    while i < hi:
+        step = values[i + 1] - values[i] if i + 1 < hi else 1
+        if step not in (0, 1):
+            step = 1
+        j = i + 1
+        while j < hi and values[j] - values[j - 1] == step:
+            j += 1
+        runs.append([i, values[i], j - i] if step else [i, values[i], j - i, 0])
+        i = j
+    return runs
+
+
+def decode_runs(runs: Sequence[Sequence[int]]) -> dict[int, int]:
+    """Inverse of :func:`encode_runs`: index to value."""
+    out: dict[int, int] = {}
+    for run in runs:
+        index, value, count = run[:3]
+        step = 0 if len(run) > 3 else 1
+        for k in range(count):
+            out[index + k] = value + k * step
+    return out