latexwalker 0.1.0__py3-none-any.whl

latexwalker/__init__.py

@@ -0,0 +1,72 @@
+"""Public surface of latexwalker.
+
+Symbols re-exported here form the supported API; deeper modules remain
+importable for advanced use but may change without notice.
+"""
+
+from ._version import __version__
+from .errors import LatexWalkerError, ParseError, RulesError
+from .lexer import tokenize
+from .nodes import (
+    BraceGroup,
+    BracketGroup,
+    Comment,
+    Content,
+    Document,
+    Environment,
+    Macro,
+    MacroArgument,
+    Math,
+    MathMode,
+    Node,
+    ParBreak,
+    SpecialChar,
+    Text,
+    Whitespace,
+)
+from .normalize import normalize
+from .parser import parse_document
+from .rewrite import CompiledRuleSet
+from .rules import Rule, RuleScope, RuleSet, decode_rules, load_rules
+from .source import SourceMap, SourcePoint, SourceSpan, covering_span
+from .tokens import Token, TokenType
+from .traverse import iter_children, walk
+
+__all__ = [
+    "BraceGroup",
+    "BracketGroup",
+    "Comment",
+    "CompiledRuleSet",
+    "Content",
+    "Document",
+    "Environment",
+    "LatexWalkerError",
+    "Macro",
+    "MacroArgument",
+    "Math",
+    "MathMode",
+    "Node",
+    "ParBreak",
+    "ParseError",
+    "Rule",
+    "RuleScope",
+    "RuleSet",
+    "RulesError",
+    "SourceMap",
+    "SourcePoint",
+    "SourceSpan",
+    "SpecialChar",
+    "Text",
+    "Token",
+    "TokenType",
+    "Whitespace",
+    "__version__",
+    "covering_span",
+    "decode_rules",
+    "iter_children",
+    "load_rules",
+    "normalize",
+    "parse_document",
+    "tokenize",
+    "walk",
+]

latexwalker/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

latexwalker/errors.py

@@ -0,0 +1,24 @@
+"""Typed failure surface for lexing and parsing operations."""
+
+from .source import SourceSpan
+
+
+class LatexWalkerError(Exception):
+    """Base class for every error raised by latexwalker."""
+
+
+class ParseError(LatexWalkerError):
+    """Raised when source text cannot be tokenized or parsed."""
+
+    message: str
+    span: SourceSpan | None
+
+    def __init__(self, message: str, *, span: SourceSpan | None = None) -> None:
+        rendered = f"{message} ({span.describe()})" if span is not None else message
+        super().__init__(rendered)
+        self.message = message
+        self.span = span
+
+
+class RulesError(LatexWalkerError):
+    """Raised when a normalization rule set cannot be read or compiled."""

latexwalker/lexer.py

@@ -0,0 +1,129 @@
+"""Streaming lexer converting raw LaTeX text into a token sequence."""
+
+from collections.abc import Iterator
+
+from .errors import ParseError
+from .source import SourceMap, SourceSpan
+from .tokens import Token, TokenType
+
+_SINGLE_CHAR_TYPES: dict[str, TokenType] = {
+    "{": TokenType.GROUP_OPEN,
+    "}": TokenType.GROUP_CLOSE,
+    "[": TokenType.BRACKET_OPEN,
+    "]": TokenType.BRACKET_CLOSE,
+    "&": TokenType.ALIGNMENT,
+    "#": TokenType.PARAMETER,
+    "_": TokenType.SUBSCRIPT,
+    "^": TokenType.SUPERSCRIPT,
+    "~": TokenType.TIE,
+}
+
+_WORD_STOPPERS: frozenset[str] = frozenset(_SINGLE_CHAR_TYPES) | {"\\", "%", "$"}
+_WHITESPACE: frozenset[str] = frozenset(" \t\n\r")
+
+
+def tokenize(source: SourceMap) -> Iterator[Token]:
+    """Yield every token in ``source``, ending with a sentinel end-of-input."""
+    text = source.text
+    length = len(text)
+    index = 0
+    while index < length:
+        char = text[index]
+        if char == "\\":
+            token, index = _scan_macro(source, index)
+        elif char == "%":
+            token, index = _scan_comment(source, index)
+        elif char == "$":
+            token, index = _scan_math_shift(source, index)
+        elif char in _WHITESPACE:
+            token, index = _scan_whitespace_run(source, index)
+        elif char in _SINGLE_CHAR_TYPES:
+            token = _make_token(
+                source, index, index + 1, _SINGLE_CHAR_TYPES[char], char
+            )
+            index += 1
+        else:
+            token, index = _scan_word(source, index)
+        yield token
+    yield _make_token(source, length, length, TokenType.END_OF_INPUT, "")
+
+
+def _make_token(
+    source: SourceMap, start: int, end: int, token_type: TokenType, value: str
+) -> Token:
+    return Token(
+        type=token_type,
+        value=value,
+        span=SourceSpan(start=source.point_at(start), end=source.point_at(end)),
+    )
+
+
+def _scan_word(source: SourceMap, start: int) -> tuple[Token, int]:
+    text = source.text
+    end = start + 1
+    while (
+        end < len(text)
+        and text[end] not in _WORD_STOPPERS
+        and text[end] not in _WHITESPACE
+    ):
+        end += 1
+    return _make_token(source, start, end, TokenType.WORD, text[start:end]), end
+
+
+def _scan_macro(source: SourceMap, start: int) -> tuple[Token, int]:
+    """Scan one control sequence; the stored value excludes the backslash."""
+    text = source.text
+    name_start = start + 1
+    if name_start >= len(text):
+        raise ParseError(
+            "dangling backslash at end of input",
+            span=SourceSpan.empty_at(source.point_at(start)),
+        )
+    lead = text[name_start]
+    if lead == "\n":
+        raise ParseError(
+            "backslash cannot be continued by a newline",
+            span=SourceSpan.empty_at(source.point_at(start)),
+        )
+    if lead.isalpha():
+        name_end = name_start + 1
+        while name_end < len(text) and text[name_end].isalpha():
+            name_end += 1
+        return _make_token(
+            source, start, name_end, TokenType.MACRO, text[name_start:name_end]
+        ), name_end
+    return _make_token(
+        source, start, name_start + 1, TokenType.MACRO, lead
+    ), name_start + 1
+
+
+def _scan_comment(source: SourceMap, start: int) -> tuple[Token, int]:
+    text = source.text
+    newline = text.find("\n", start)
+    end = newline if newline != -1 else len(text)
+    return _make_token(source, start, end, TokenType.COMMENT, text[start:end]), end
+
+
+def _scan_math_shift(source: SourceMap, start: int) -> tuple[Token, int]:
+    display = source.text.startswith("$$", start)
+    if display:
+        return (
+            _make_token(source, start, start + 2, TokenType.MATH_SHIFT_DISPLAY, "$$"),
+            start + 2,
+        )
+    return (
+        _make_token(source, start, start + 1, TokenType.MATH_SHIFT_INLINE, "$"),
+        start + 1,
+    )
+
+
+def _scan_whitespace_run(source: SourceMap, start: int) -> tuple[Token, int]:
+    text = source.text
+    end = start + 1
+    while end < len(text) and text[end] in _WHITESPACE:
+        end += 1
+    raw = text[start:end]
+    token_type = (
+        TokenType.PARAGRAPH_BREAK if raw.count("\n") >= 2 else TokenType.WHITESPACE
+    )
+    return _make_token(source, start, end, token_type, raw), end

latexwalker/nodes.py

@@ -0,0 +1,123 @@
+"""Immutable syntax tree schemas produced by the parser.
+
+Every node kind is a frozen ``msgspec.Struct`` carrying the source extent it
+was built from, so trees stay lossless and cheap to share across stages.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+import msgspec
+
+from .source import SourceMap, SourceSpan
+
+
+class MathMode(StrEnum):
+    """Whether a math region is inline or display."""
+
+    INLINE = "inline"
+    DISPLAY = "display"
+
+
+class Node(msgspec.Struct, frozen=True, kw_only=True):
+    """Base schema shared by every syntax node; anchors it in the source."""
+
+    span: SourceSpan
+
+
+class Text(Node, frozen=True):
+    """A run of ordinary characters, including interior spacing."""
+
+    value: str
+
+
+class Whitespace(Node, frozen=True):
+    """A run of spaces or newlines that does not separate paragraphs."""
+
+    value: str
+
+
+class ParBreak(Node, frozen=True):
+    """One or more blank lines separating paragraphs."""
+
+
+class Comment(Node, frozen=True):
+    """A percent-introduced comment up to but excluding its newline."""
+
+    text: str
+
+
+class SpecialChar(Node, frozen=True):
+    """A category-code character such as ``&``, ``#``, ``_``, or ``^``."""
+
+    char: str
+
+
+class BraceGroup(Node, frozen=True):
+    """A brace-delimited ``{body}``."""
+
+    body: tuple[Content, ...]
+
+
+class BracketGroup(Node, frozen=True):
+    """A bracket-delimited ``[body]``."""
+
+    body: tuple[Content, ...]
+
+
+type MacroArgument = BraceGroup | BracketGroup
+
+
+class Macro(Node, frozen=True):
+    """A control sequence with its immediately attached group arguments."""
+
+    name: str
+    arguments: tuple[MacroArgument, ...]
+
+
+class Environment(Node, frozen=True):
+    """A ``\\begin{name}...\\end{name}`` region with its begin-side arguments."""
+
+    name: str
+    arguments: tuple[MacroArgument, ...]
+    body: tuple[Content, ...]
+
+
+class Math(Node, frozen=True):
+    """An inline or display math region delimited by shifts or ``\\[``/``\\]``."""
+
+    mode: MathMode
+    body: tuple[Content, ...]
+
+
+class Document(Node, frozen=True):
+    """Root of a parsed file, owning the offset index over its exact source."""
+
+    children: tuple[Content, ...]
+    source: SourceMap
+
+    @property
+    def text(self) -> str:
+        return self.source.text
+
+    def __str__(self) -> str:
+        return self.source.text
+
+    def source_of(self, node: Node) -> str:
+        """Return the exact original text the given node was built from."""
+        return self.source.extract(node.span)
+
+
+type Content = (
+    Text
+    | Whitespace
+    | ParBreak
+    | Comment
+    | SpecialChar
+    | BraceGroup
+    | BracketGroup
+    | Macro
+    | Environment
+    | Math
+)

latexwalker/normalize.py

@@ -0,0 +1,46 @@
+"""Public entry point reducing LaTeX source toward its canonical form."""
+
+from functools import lru_cache
+from importlib import resources
+
+from .errors import RulesError
+from .rewrite import CompiledRuleSet
+from .rules import RuleScope, RuleSet, decode_rules
+from .zones import ZoneKind, classify
+
+_ALLOWED_SCOPES: dict[ZoneKind, frozenset[RuleScope]] = {
+    ZoneKind.PRESERVE: frozenset(),
+    ZoneKind.MATH: frozenset((RuleScope.ANY, RuleScope.MATH)),
+    ZoneKind.TEXT: frozenset((RuleScope.ANY, RuleScope.TEXT)),
+}
+
+
+@lru_cache(maxsize=1)
+def default_compiled() -> CompiledRuleSet:
+    resource = resources.files(__package__).joinpath("resources", "rules.yaml")
+    try:
+        raw_yaml = resource.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise RulesError("packaged default rule set is unreadable") from exc
+    return CompiledRuleSet(decode_rules(raw_yaml))
+
+
+def normalize(
+    source_text: str,
+    *,
+    rules: RuleSet | CompiledRuleSet | None = None,
+) -> str:
+    """Rewrite ``source_text`` with canonicalization rules over safe regions.
+
+    ``rules`` replaces the packaged defaults entirely; pass a pre-built
+    :class:`CompiledRuleSet` to amortize compilation across many snippets.
+    """
+    compiled = (
+        default_compiled()
+        if rules is None
+        else (rules if isinstance(rules, CompiledRuleSet) else CompiledRuleSet(rules))
+    )
+    return "".join(
+        compiled.rewrite(zone.text, allowed=_ALLOWED_SCOPES[zone.kind])
+        for zone in classify(source_text)
+    )