sugo-cli 0.1.0a1__py3-none-any.whl

sugo/chunkers/sections.py

@@ -0,0 +1,300 @@
+"""Split cleaned pages into chunks along the document's sections (ADR-047).
+
+Blocks are added in reading order, page after page, and grouped into chunks
+of at most ``max_tokens`` o200k_base tokens:
+
+- a heading closes the chunk before it, unless that chunk holds only
+  headings that the new one does not close: "# Part I" followed by
+  "## Chapter 1" and its text stay together;
+- a heading always travels with the text after it, never alone: when the
+  first block of a section does not fit beside its headings, that block is
+  split so its first piece does;
+- a block that does not fit closes the chunk and starts the next one;
+- a table is never split, even past the limit (ARCHITECTURE §4): half a
+  table cannot be read;
+- a longer paragraph is split after sentence ends or line breaks, code after
+  line breaks, then between words, and only as a last resort inside a word.
+  Pieces keep the text exactly, spaces and line breaks included, except at
+  their two ends. After the first piece of a list item, the rest reads as
+  plain text, so one bullet never turns into several.
+
+Every chunk carries the headings above it and the pages it comes from, so
+it can be traced back to its source (principle 5). Removed blocks are left
+out, exactly as in the Markdown output; nothing else is dropped.
+"""
+
+import hashlib
+import json
+import re
+from collections import Counter
+from collections.abc import Iterator
+
+from sugo.core.chunks import Chunk
+from sugo.core.constants import CHUNK_MAX_TOKENS, CHUNK_MIN_TOKENS, OUTPUT_ID_HEX_DIGITS
+from sugo.core.models import Block, BlockType, Page
+from sugo.metrics.tokenizer import count_tokens
+from sugo.serializers.escape import drop_controls
+from sugo.serializers.markdown import render, render_widest
+
+# Where a paragraph may be cut: after a sentence end and the spaces after
+# it ("3.14" is not one), after CJK sentence punctuation (ideographic full
+# stop, fullwidth ! and ?, written without a space), or after a line break.
+# The match stays with the text before it. A sentence end starts only at the
+# beginning of a run of punctuation and never gives characters back, so a
+# long run without a space after it costs linear time, not quadratic.
+_SENTENCE_BREAK = re.compile(r"(?<![.!?;:])[.!?;:]++\s+|[\u3002\uff01\uff1f]+\s*|\n+")
+# Where a run of text may be cut between words: after its spaces.
+_WORD_BREAK = re.compile(r"\s+")
+# Below this many tokens from the limit, the open chunk is counted exactly.
+_ESTIMATE_MARGIN = 16
+# Headings may take at most this share of the limit before the text that
+# follows them; beyond it they form a chunk of their own.
+_HEADINGS_MAX_SHARE = 0.5
+
+
+def document_id(source: str | None, source_sha256: str) -> str:
+    """The document's id: from its name when it has one, so editions share it."""
+    basis = f"source:{source}" if source is not None else f"sha256:{source_sha256}"
+    return "d_" + _hex(basis)
+
+
+class SectionChunker:
+    """Turns the cleaned pages of one document into chunks, streaming."""
+
+    def __init__(
+        self,
+        *,
+        source: str | None,
+        source_sha256: str,
+        sugo_version: str,
+        config_hash: str,
+        max_tokens: int = CHUNK_MAX_TOKENS,
+    ) -> None:
+        if max_tokens < CHUNK_MIN_TOKENS:
+            raise ValueError(f"max_tokens must be at least {CHUNK_MIN_TOKENS}")
+        self._max = max_tokens
+        self._source = source
+        self._source_sha256 = source_sha256
+        self._sugo_version = sugo_version
+        self._config_hash = config_hash
+        self._document_id = document_id(source, source_sha256)
+        # Blocks of the open chunk, each with its page number (from 1).
+        self._open: list[tuple[Block, int]] = []
+        # Tokens of the open chunk, estimated from its blocks (see _append).
+        self._estimate = 0
+        # Levels of the headings in the open chunk, while it holds nothing else.
+        self._open_levels: list[int] | None = []
+        self._headings: list[tuple[int, str]] = []
+        # Occurrences of each text so far: repeated text gets distinct ids.
+        self._seen: Counter[str] = Counter()
+
+    def add(self, page: Page) -> Iterator[Chunk]:
+        """Add the next page; yield the chunks it completes."""
+        for block in page.blocks:
+            if block.removed:
+                continue
+            yield from self._add_block(block, page.index + 1)
+
+    def finish(self) -> Iterator[Chunk]:
+        """Yield the last chunk, once every page has been added."""
+        yield from self._close()
+
+    def _add_block(self, block: Block, page_number: int) -> Iterator[Chunk]:
+        tokens = count_tokens(render_widest(block))
+        if block.type is BlockType.HEADING:
+            yield from self._add_heading(block, page_number, tokens)
+            return
+        headings_only = bool(self._open) and self._open_levels is not None
+        if block.type is BlockType.TABLE:
+            # Whole, with its section's headings if any, even past the limit.
+            if self._open and not headings_only and not self._fits(block, tokens):
+                yield from self._close()
+            self._append(block, page_number, tokens)
+            self._open_levels = None
+            return
+        if headings_only and self._fits(block, tokens):
+            self._append(block, page_number, tokens)
+            self._open_levels = None
+            return
+        if headings_only or tokens > self._max:
+            yield from self._add_in_pieces(block, page_number, headings_only)
+            return
+        if self._open and not self._fits(block, tokens):
+            yield from self._close()
+        self._append(block, page_number, tokens)
+        self._open_levels = None
+
+    def _add_heading(self, block: Block, page_number: int, tokens: int) -> Iterator[Chunk]:
+        level = block.heading_level or 1
+        # A heading at the level of one already in the chunk, or above it,
+        # ends that heading's (empty) section: it closes the chunk, so the
+        # chunk's heading path always names every heading it contains.
+        if self._open_levels is None or any(seen >= level for seen in self._open_levels):
+            yield from self._close()
+        levels = self._open_levels if self._open_levels is not None else []
+        while self._headings and self._headings[-1][0] >= level:
+            self._headings.pop()
+        # Controls dropped as in the Markdown: the path is shown to people too.
+        title = " ".join(drop_controls(block.text).split())
+        if title:
+            self._headings.append((level, title))
+        self._append(block, page_number, tokens)
+        levels.append(level)
+        self._open_levels = levels
+
+    def _add_in_pieces(
+        self, block: Block, page_number: int, headings_only: bool
+    ) -> Iterator[Chunk]:
+        """Split a block too big for the room left, keeping a section's headings with it."""
+        room = self._max
+        if headings_only:
+            headings = count_tokens(render(b for b, _ in self._open))
+            if headings + 1 <= self._max * _HEADINGS_MAX_SHARE:
+                room = self._max - headings - 1
+            else:
+                yield from self._close()
+                headings_only = False
+        elif self._open:
+            yield from self._close()
+        for index, piece in enumerate(_pieces(block, self._max, first_max=room)):
+            if index == 0 and headings_only:
+                self._open.append((piece, page_number))
+            else:
+                self._open = [(piece, page_number)]
+            self._open_levels = None
+            yield from self._close()
+
+    def _fits(self, block: Block, tokens: int) -> bool:
+        """Whether the block fits in the open chunk; exact near the limit."""
+        estimate = self._estimate + 1 + tokens
+        if estimate <= self._max - _ESTIMATE_MARGIN:
+            return True
+        return count_tokens(render([*(b for b, _ in self._open), block])) <= self._max
+
+    def _append(self, block: Block, page_number: int, tokens: int) -> None:
+        # Blocks are joined by a line break, one token, and o200k_base never
+        # merges tokens across a line break, so the sum is a close estimate.
+        self._estimate = tokens if not self._open else self._estimate + 1 + tokens
+        self._open.append((block, page_number))
+
+    def _close(self) -> Iterator[Chunk]:
+        blocks, self._open, self._open_levels = self._open, [], []
+        self._estimate = 0
+        text = render(block for block, _ in blocks)
+        if not text:
+            return
+        occurrence = self._seen[text]
+        self._seen[text] += 1
+        yield Chunk(
+            id="c_" + _hex(self._document_id, text, occurrence),
+            document_id=self._document_id,
+            text=text,
+            tokens=count_tokens(text),
+            heading_path=tuple(title for _, title in self._headings),
+            pages=tuple(sorted({page for _, page in blocks})),
+            source=self._source,
+            source_sha256=self._source_sha256,
+            sugo_version=self._sugo_version,
+            config_hash=self._config_hash,
+        )
+
+
+def _pieces(block: Block, max_tokens: int, *, first_max: int) -> Iterator[Block]:
+    """Split one long block into blocks that each render within the limit."""
+    code = block.type is BlockType.CODE
+    texts = _split(block.text, block, max_tokens, first_max=first_max, lines_only=code)
+    for index, text in enumerate(texts):
+        # The rest of a list item is plain text: a second bullet would be a
+        # second item. Chat messages keep their speaker on every piece.
+        continued = index > 0 and block.type is BlockType.LIST_ITEM
+        yield _with_text(block, text, index, as_paragraph=continued)
+
+
+def _split(
+    text: str, block: Block, max_tokens: int, *, first_max: int, lines_only: bool
+) -> list[str]:
+    pieces: list[str] = []
+    current = ""
+    # Sum of each unit rendered alone: never below the real count, since a
+    # unit alone pays its own prefix and escapes; counted exactly near the limit.
+    estimate = 0
+    for unit in _units(text, lines_only):
+        limit = first_max if not pieces else max_tokens
+        current += unit
+        estimate += _cost(unit, block, len(pieces))
+        if estimate <= limit - _ESTIMATE_MARGIN:
+            continue
+        estimate = _cost(current, block, len(pieces))
+        if estimate <= limit:
+            continue
+        previous = current[: len(current) - len(unit)]
+        if previous.strip():
+            pieces.append(previous)
+            current = unit
+            limit = max_tokens
+            estimate = _cost(current, block, len(pieces))
+        while estimate > limit:
+            # One unit alone is too long: cut it until the head fits.
+            head, current = _cut(current, limit, block, len(pieces))
+            pieces.append(head)
+            limit = max_tokens
+            estimate = _cost(current, block, len(pieces))
+    if current.strip():
+        pieces.append(current)
+    edges = "\n" if lines_only else None
+    return [piece.strip(edges) for piece in pieces if piece.strip()]
+
+
+def _units(text: str, lines_only: bool) -> Iterator[str]:
+    """Consecutive slices of ``text`` ending at a break; they join back to it exactly."""
+    if lines_only:
+        yield from text.splitlines(keepends=True)
+        return
+    start = 0
+    for match in _SENTENCE_BREAK.finditer(text):
+        if match.end() > start:
+            yield text[start : match.end()]
+            start = match.end()
+    if start < len(text):
+        yield text[start:]
+
+
+def _cut(text: str, max_tokens: int, block: Block, index: int) -> tuple[str, str]:
+    """Split ``text`` into a head within the limit and the rest; the head is never empty."""
+    breaks = [match.end() for match in _WORD_BREAK.finditer(text) if match.end() < len(text)]
+    # The last word break whose head fits, found by bisection: a longer head
+    # never costs fewer tokens.
+    low, high = 0, len(breaks)
+    while low < high:
+        middle = (low + high) // 2
+        if _cost(text[: breaks[middle]], block, index) <= max_tokens:
+            low = middle + 1
+        else:
+            high = middle
+    if low:
+        cut = breaks[low - 1]
+        return text[:cut], text[cut:]
+    # No word fits whole (a URL, an encoded blob): cut inside it, by halving.
+    cut = len(text)
+    while cut > 1 and _cost(text[:cut], block, index) > max_tokens:
+        cut //= 2
+    return text[:cut], text[cut:]
+
+
+def _cost(text: str, block: Block, index: int) -> int:
+    continued = index > 0 and block.type is BlockType.LIST_ITEM
+    return count_tokens(render_widest(_with_text(block, text, index, as_paragraph=continued)))
+
+
+def _with_text(block: Block, text: str, index: int, *, as_paragraph: bool = False) -> Block:
+    # Only blocks with text are split: an illegible block has none.
+    fields = {**block.model_dump(), "id": f"{block.id}#{index}", "text": text}
+    if as_paragraph:
+        fields["type"] = BlockType.PARAGRAPH
+        fields["list_level"] = None
+    return Block.model_validate(fields)
+
+
+def _hex(*parts: str | int) -> str:
+    encoded = json.dumps(parts, ensure_ascii=False).encode()
+    return hashlib.sha256(encoded).hexdigest()[:OUTPUT_ID_HEX_DIGITS]

sugo/cleaners/__init__.py

@@ -0,0 +1 @@
+"""Cleaners: propose removals of noise from IR pages; the core applies them (ADR-028)."""

sugo/cleaners/decorative_images.py

@@ -0,0 +1,31 @@
+"""Remove decorative pictures: bullets, rules and icons too small to hold content.
+
+The Office reader flags a picture under 32 x 32 pixels ``decorative``
+(ADR-054); this cleaner only proposes to remove those blocks, and the core
+applies the removal as a soft deletion with the reason below (principle 3).
+"""
+
+from typing import ClassVar
+
+from sugo.core.decisions import Decision
+from sugo.core.models import BlockFlag, Page
+
+REASON = "decorative picture under 32 x 32 pixels, too small to hold content"
+
+
+class DecorativeImageCleaner:
+    """Proposes the removal of every decorative picture still present on a page."""
+
+    name: ClassVar[str] = "decorative_images"
+
+    def propose(self, page: Page, stats: object = None) -> list[Decision]:
+        """Return one removal decision per decorative block not yet removed.
+
+        ``stats`` is part of the cleaner contract and unused here: this
+        cleaner needs no document-wide context.
+        """
+        return [
+            Decision(block_id=block.id, reason=REASON)
+            for block in page.blocks
+            if BlockFlag.DECORATIVE in block.flags and not block.removed
+        ]

sugo/cleaners/headers_footers.py

@@ -0,0 +1,56 @@
+"""Remove running headers and footers: text repeated at the same page edge.
+
+The edge analyzer finds which edge texts repeat across the document; this
+cleaner proposes to remove each copy on a page, and the core applies the
+removal as a soft deletion with a reason (principle 3, ADR-041).
+
+Short boilerplate ("Annual Report 2026", "Confidential") goes from every
+page. Longer text may be a clause someone repeated on every page so that it
+would be dropped as a footer (THREAT_MODEL C): its first copy is kept, and
+only the later identical copies go, so no content leaves the output.
+
+Headers and footers a document declares itself, such as Word's, have no
+place on the page and appear once per section: the first copy stays, even a
+short one, which may say what the document is ("DRAFT"), and the identical
+copies of later sections go (ADR-054).
+"""
+
+from typing import ClassVar
+
+from sugo.analyzers.edges import Band, EdgeStats, declared_texts, edge_texts
+from sugo.core.decisions import Decision
+from sugo.core.models import BlockType, Page
+
+_NAMES = {Band.TOP: "header", Band.BOTTOM: "footer"}
+_DECLARED = {
+    BlockType.HEADER: "running header declared by the document",
+    BlockType.FOOTER: "running footer declared by the document",
+}
+
+
+class HeaderFooterCleaner:
+    """Proposes the removal of running headers and footers on a page."""
+
+    name: ClassVar[str] = "headers_footers"
+
+    def propose(self, page: Page, stats: EdgeStats) -> list[Decision]:
+        """Return one removal decision per copy of a running header or footer."""
+        decisions = []
+        for text in edge_texts(page):
+            if text.page_number is not None:
+                continue  # Left to the page number cleaner.
+            repeated = stats.repeated_edge(text, page.index)
+            if repeated is None:
+                continue
+            reason = f"running {_NAMES[text.band]} repeated on {repeated.pages} pages"
+            if repeated.substantive:
+                if page.index == repeated.first_page:
+                    continue
+                reason += f"; first copy kept on page {repeated.first_page + 1}"
+            decisions.append(Decision(block_id=text.block_id, reason=reason))
+        for declared in declared_texts(page):
+            first = stats.first_declared(declared)
+            if declared.type in _DECLARED and first is not None and first < page.index:
+                reason = f"{_DECLARED[declared.type]}; first copy kept on page {first + 1}"
+                decisions.append(Decision(block_id=declared.block_id, reason=reason))
+        return decisions

sugo/cleaners/hidden_text.py

@@ -0,0 +1,35 @@
+"""Remove text a reader cannot see on the page: white on white, microscopic,
+off the page, drawn invisible, or the unverified text layer of a scan.
+
+Hidden text is a classic way to slip instructions to an LLM that no person
+reviewing the document would notice. The PDF extractor flags such text
+``hidden`` and keeps it apart from visible text (ADR-043); this cleaner only
+proposes to remove those blocks, and the core applies the removal as a soft
+deletion with the reason below (principle 3). The text stays in the IR, so
+preview and explain can still show what was hidden.
+"""
+
+from typing import ClassVar
+
+from sugo.core.decisions import Decision
+from sugo.core.models import BlockFlag, Page
+
+REASON = "text not visible on the rendered page (hidden, invisible or an unverified text layer)"
+
+
+class HiddenTextCleaner:
+    """Proposes the removal of every hidden block still present on a page."""
+
+    name: ClassVar[str] = "hidden_text"
+
+    def propose(self, page: Page, stats: object = None) -> list[Decision]:
+        """Return one removal decision per hidden block not yet removed.
+
+        ``stats`` is part of the cleaner contract and unused here: this
+        cleaner needs no document-wide context.
+        """
+        return [
+            Decision(block_id=block.id, reason=REASON)
+            for block in page.blocks
+            if BlockFlag.HIDDEN in block.flags and not block.removed
+        ]

sugo/cleaners/page_numbers.py

@@ -0,0 +1,39 @@
+"""Remove page numbers that follow the page sequence of the document.
+
+A lone "7" at the bottom of a page is a page number only if the pages
+around it carry 6 and 8 in the same place; the edge analyzer checks that.
+This cleaner proposes to remove those numbers, and the core applies the
+removal as a soft deletion with the reason below (principle 3, ADR-041).
+"""
+
+from typing import ClassVar
+
+from sugo.analyzers.edges import EdgeStats, declared_texts, edge_texts
+from sugo.core.decisions import Decision
+from sugo.core.models import BlockType, Page
+
+REASON = "page number"
+
+
+class PageNumberCleaner:
+    """Proposes the removal of the page number printed on a page."""
+
+    name: ClassVar[str] = "page_numbers"
+
+    def propose(self, page: Page, stats: EdgeStats) -> list[Decision]:
+        """Return one removal decision per page number in the document's sequence.
+
+        A page number the document declares itself, with no place on the page
+        (a slide number placeholder), goes when its text is a page number.
+        """
+        decisions = [
+            Decision(block_id=text.block_id, reason=REASON)
+            for text in edge_texts(page)
+            if stats.is_page_number(text, page.index)
+        ]
+        decisions.extend(
+            Decision(block_id=declared.block_id, reason=REASON)
+            for declared in declared_texts(page)
+            if declared.type is BlockType.PAGE_NUMBER and declared.page_number
+        )
+        return decisions

sugo/cleaners/text_normalization.py

@@ -0,0 +1,177 @@
+"""Normalise text that costs tokens without saying anything (ADR-050).
+
+PDFs and OCR leave text that reads the same but tokenizes worse: ligatures
+("fi"), no-break and fixed-width spaces, zero-width characters, soft hyphens,
+runs of spaces, trailing spaces, accents stored as two characters, words cut
+by a hyphen at the end of a line, and in Italian and French a stray space
+after an elision ("dell' albero"). This cleaner proposes one edit per such
+spot; the core checks that each edit is of the kind it claims and applies it,
+keeping the original on the block (principle 3).
+
+Every rule keeps the letters, digits and signs a reader sees, with one
+exception the core checks separately: a line-break hyphen inside a word.
+"infor-" + "mation" becomes "information" only if the document writes
+"information" elsewhere; "well-" + "known" becomes "well-known" if it writes
+that; otherwise the hyphen stays (ADR-025: when in doubt, leave it). A soft
+hyphen is always a word break, so it always joins.
+
+Code, tables and formulas are left alone: their spacing can matter.
+"""
+
+import re
+import unicodedata
+from typing import ClassVar
+
+from sugo.analyzers.lexicon import Lexicon
+from sugo.core.constants import INVISIBLE_CHARS, LIGATURES, MAX_EDITS_PER_BLOCK, SPECIAL_SPACES
+from sugo.core.decisions import EditDecision, within_limits
+from sugo.core.models import BlockType, EditKind, Page
+from sugo.core.text import APOSTROPHES, HYPHEN_BREAK, is_letter
+from sugo.languages.packs import elisions
+
+# Block types whose text is prose.
+TEXT_TYPES = frozenset(
+    {
+        BlockType.HEADING,
+        BlockType.PARAGRAPH,
+        BlockType.LIST_ITEM,
+        BlockType.CHAT_MESSAGE,
+        BlockType.CAPTION,
+        BlockType.FOOTNOTE,
+    }
+)
+
+REASON_HYPHEN = "word split by a hyphen at the end of a line"
+REASON_COMPOUND = "hyphenated word split at the end of a line"
+REASON_SPACES = "spacing normalised"
+REASON_INVISIBLE = "invisible character"
+REASON_LIGATURE = "typographic ligature"
+REASON_COMPOSED = "accented letter stored as two characters"
+REASON_ELISION = "space after an elision"
+
+_LETTER = r"[^\W\d_]"
+_SPACE = f"[ {SPECIAL_SPACES}]"
+# One pass over the text; alternatives never overlap, earlier ones win.
+_RULES = re.compile(
+    "|".join(
+        (
+            f"(?P<hyphen>{HYPHEN_BREAK})",
+            rf"(?P<elision>(?<=[{APOSTROPHES}]){_SPACE}+(?={_LETTER}))",
+            rf"(?P<trailing>(?<=\S){_SPACE}+(?=\n|\Z))",
+            rf"(?P<spaces>(?<=\S)(?:{_SPACE}{{2,}}|[{SPECIAL_SPACES}]))",
+            f"(?P<invisible>[{INVISIBLE_CHARS}])",
+            f"(?P<ligature>[{''.join(LIGATURES)}])",
+            r"(?P<composed>.[\u0300-\u036f]+)",
+        )
+    )
+)
+# Longer "words" are not words; capping the scan keeps it linear on hostile text.
+_MAX_WORD = 64
+
+
+class TextNormalizationCleaner:
+    """Proposes normalisation edits for the prose of a page."""
+
+    name: ClassVar[str] = "text_normalization"
+
+    def __init__(self, languages: tuple[str, ...] = ("en",)) -> None:
+        """
+        Raises:
+            ValueError: If a language has no pack.
+        """
+        self._elisions = elisions(languages)
+
+    def propose(self, page: Page, stats: Lexicon) -> list[EditDecision]:
+        """Return one edit per spot to normalise in the page's prose."""
+        return within_limits(
+            self._block_edits(block.id, block.text, stats)
+            for block in page.blocks
+            if not block.removed and block.type in TEXT_TYPES
+        )
+
+    def _block_edits(self, block_id: str, text: str, stats: Lexicon) -> list[EditDecision]:
+        decisions: list[EditDecision] = []
+        for match in _RULES.finditer(text):
+            edit = self._edit(text, match, stats)
+            if edit is not None:
+                replacement, kind, reason = edit
+                decisions.append(
+                    EditDecision(
+                        block_id=block_id,
+                        start=match.start(),
+                        end=match.end(),
+                        replacement=replacement,
+                        kind=kind,
+                        reason=reason,
+                    )
+                )
+                if len(decisions) > MAX_EDITS_PER_BLOCK:
+                    # Left as it is anyway (within_limits): stop early.
+                    break
+        return decisions
+
+    def _edit(
+        self, text: str, match: re.Match[str], lexicon: Lexicon
+    ) -> tuple[str, EditKind, str] | None:
+        rule, found = match.lastgroup, match.group()
+        if rule == "hyphen":
+            return _hyphen(text, match, lexicon)
+        if rule == "elision":
+            apostrophe = match.start() - 1
+            before = _word_before(text, apostrophe)
+            if before is None or before.casefold() not in self._elisions:
+                return None
+            # "'all' keyword" and "rock 'n' roll" close a quote: no elision.
+            opening = apostrophe - len(before) - 1
+            if opening >= 0 and text[opening] in APOSTROPHES:
+                return None
+            return "", EditKind.NORMALIZE, REASON_ELISION
+        if rule == "trailing":
+            return "", EditKind.NORMALIZE, REASON_SPACES
+        if rule == "spaces":
+            return " ", EditKind.NORMALIZE, REASON_SPACES
+        if rule == "invisible":
+            return "", EditKind.NORMALIZE, REASON_INVISIBLE
+        if rule == "ligature":
+            return LIGATURES[found], EditKind.NORMALIZE, REASON_LIGATURE
+        composed = unicodedata.normalize("NFC", found)
+        return None if composed == found else (composed, EditKind.NORMALIZE, REASON_COMPOSED)
+
+
+def _hyphen(text: str, match: re.Match[str], lexicon: Lexicon) -> tuple[str, EditKind, str] | None:
+    """Join a word split at a line end, keep a compound's hyphen, or leave it."""
+    head = _word_before(text, match.start())
+    tail = _word_after(text, match.end())
+    if head is None or tail is None:
+        # Not inside a word: "2024-\n2025", "x²-\n", a dash at a line end.
+        return None
+    # The last hyphen before the line break; soft hyphens may come before it.
+    hyphen = match.group().rstrip(" \t\n")[-1]
+    if hyphen == "\u00ad":
+        # A soft hyphen marks where a word may break: always one word.
+        return "", EditKind.HYPHEN_JOIN, REASON_HYPHEN
+    if head + tail in lexicon:
+        return "", EditKind.HYPHEN_JOIN, REASON_HYPHEN
+    if f"{head}-{tail}" in lexicon:
+        return hyphen, EditKind.NORMALIZE, REASON_COMPOUND
+    return None
+
+
+def _word_before(text: str, end: int) -> str | None:
+    """The whole word ending at ``end``, or None if there is none or it is too long."""
+    start = end
+    while start > 0 and end - start <= _MAX_WORD and is_letter(text[start - 1]):
+        start -= 1
+    if start == end or end - start > _MAX_WORD:
+        return None
+    return text[start:end]
+
+
+def _word_after(text: str, start: int) -> str | None:
+    """The whole word starting at ``start``, or None if there is none or it is too long."""
+    end = start
+    while end < len(text) and end - start <= _MAX_WORD and is_letter(text[end]):
+        end += 1
+    if end == start or end - start > _MAX_WORD:
+        return None
+    return text[start:end]