texdiff 0.2.0__py3-none-any.whl

texdiff/flatten.py

@@ -0,0 +1,185 @@
+"""\\input/\\include expansion: multi-file document → single source.
+
+The equivalent of ``latexdiff --flatten`` / ``latexpand``, inside the
+tool. texdiff parses per file, then expands references so the aligner
+sees one document tree.
+
+Design:
+
+* textual (linear) expansion, matching latexpand's contract: no macro
+  semantics, just file inclusion - what ``\\input`` does in TeX;
+* files are resolved relative to the INCLUDING file's directory
+  (TeX's kpathsea semantics simplified: the main file's dir, then the
+  including chain);
+* cycle protection: a file already on the expansion stack is kept
+  verbatim instead of being recursed into (TeX would loop forever);
+* missing files are kept verbatim rather than fatal - a broken
+  ``\\input`` must not block the diff, and keeping the reference makes
+  the output still compilable;
+* comment lines and verbatim environments are skipped: a commented
+  ``% \\input{x}`` is not an inclusion, and verbatim bodies must never
+  be expanded.
+
+The emitter later re-uses the skip logic for the same purpose (never
+mark up inside verbatim).
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+# \input{name} / \include{name}; name may lack the .tex extension
+# (TeX appends it), may contain letters, /, -, _, . and spaces
+_INPUT_RE = re.compile(r"\\(input|include)\*?\s*\{([^{}]+)\}")
+
+# block of lines which must never be expanded: verbatim-like envs
+_VERBATIM_ENVS = (
+    "verbatim",
+    "verbatim*",
+    "lstlisting",
+    "minted",
+    "alltt",
+    "filecontents",
+    "filecontents*",
+)
+
+_MARKER_PREFIX = "% texdiff-flatten:"
+
+
+class Flattener:
+    """Expand ``\\input``/``\\include`` in a source string.
+
+    Attributes:
+        base_dir: directory files are resolved against.
+        expanded: number of references successfully expanded.
+        missing: number of references whose file did not exist
+            (kept verbatim).
+        markers: insert ``% texdiff-flatten:`` annotation comments
+            around expanded content (for debugging / provenance).
+    """
+
+    def __init__(self, base_dir: Path | str | None = None, markers: bool = False):
+        self.base_dir = Path(base_dir) if base_dir is not None else Path.cwd()
+        self.markers = markers
+        self.expanded = 0
+        self.missing = 0
+
+    def flatten(self, source: str) -> str:
+        """Expand all \\input/\\include references in ``source``."""
+        return self._expand(source, self.base_dir, stack=frozenset())
+
+    def _expand(self, source: str, current_dir: Path, stack: frozenset[Path]) -> str:
+        out: list[str] = []
+        pos = 0
+        for m in self._iter_skipping(source):
+            out.append(source[pos : m.start()])
+            cmd, name = m.group(1), m.group(2)
+            target = self._resolve(name, current_dir)
+            if target is None:
+                self.missing += 1
+                out.append(m.group(0))
+            else:
+                resolved = target.resolve()
+                if resolved in stack:
+                    # cycle: keep verbatim, do not recurse
+                    out.append(m.group(0))
+                    self.expanded += 1
+                    pos = m.end()
+                    continue
+                content = target.read_text(encoding="utf-8", errors="replace")
+                beginning = len(out)
+                if self.markers:
+                    out.append(f"{_MARKER_PREFIX} begin {target.name}\n")
+                out.append(self._expand(content, resolved.parent, stack | {resolved}))
+                if self.markers:
+                    out.append(f"\n{_MARKER_PREFIX} end {target.name}\n")
+                self.expanded += 1
+                del beginning
+            pos = m.end()
+        out.append(source[pos:])
+        return "".join(out)
+
+    def _resolve(self, name: str, current_dir: Path) -> Path | None:
+        """Resolve an \\input name to a file, TeX-style (with .tex)."""
+        candidate = name.strip()
+        if candidate.startswith("/"):
+            # absolute: as-is (with .tex fallback)
+            p = Path(candidate)
+            return p if p.is_file() else self._with_tex(p)
+
+        for base in (current_dir, self.base_dir):
+            p = base / candidate
+            if p.is_file():
+                return p
+            p = p.with_suffix(".tex")
+            if p.is_file():
+                with_tex = base / f"{candidate}.tex"
+                if with_tex.is_file():
+                    return with_tex
+        return None
+
+    def _with_tex(self, p: Path) -> Path | None:
+        q = p.with_suffix(".tex")
+        return p if p.is_file() else (q if q.is_file() else None)
+
+    def _iter_skipping(self, source: str) -> list[re.Match]:
+        """Matches of _INPUT_RE outside comments and verbatim bodies."""
+        results: list[re.Match] = []
+        i = 0
+        n = len(source)
+        while i < n:
+            # comment: up to end of line (comment includes it)
+            if source[i] == "%":
+                nl = source.find("\n", i)
+                i = n if nl < 0 else nl + 1
+                continue
+            # verbatim-like environment: skip whole body
+            m_verb = re.compile(
+                rf"\\begin\{{({'|'.join(re.escape(e) for e in _VERBATIM_ENVS)})\}}"
+            ).match(source, i)
+            if m_verb:
+                end = re.compile(rf"\\end\{{{m_verb.group(1)}\}}")
+                m_end = end.search(source, m_verb.end())
+                skip_to = m_end.end() if m_end else n
+                results_adjacent = _INPUT_RE.finditer(source, m_verb.end(), skip_to)
+                # inside verbatim: do not expand - skip entirely
+                i = skip_to
+                continue
+            m = _INPUT_RE.match(source, i)
+            if m:
+                results.append(m)
+                i = m.end()
+                continue
+            i += 1
+        return results
+
+
+def flatten_source(
+    source: str,
+    base_dir: Path | str | None = None,
+    markers: bool = False,
+) -> str:
+    """Expand \\input/\\include in a source string (base_dir=cwd)."""
+    return Flattener(base_dir=base_dir, markers=markers).flatten(source)
+
+
+def flatten_file(
+    source_path: Path | str,
+    base_dir: Path | str | None = None,
+    markers: bool = False,
+) -> str:
+    """Read a file and expand all its \\input/\\include references.
+
+    Args:
+        source_path: the main .tex file.
+        base_dir: fallback directory for unresolvable-relative names
+            (default: the main file's directory).
+        markers: insert provenance comments around expansions.
+
+    Returns:
+        The flattened source text.
+    """
+    sp = Path(source_path)
+    f = Flattener(base_dir=base_dir if base_dir is not None else sp.parent, markers=markers)
+    return f.flatten(sp.read_text(encoding="utf-8", errors="replace"))

texdiff/nodes.py

@@ -0,0 +1,147 @@
+"""Nodes - the intermediate representation produced by parsing.
+
+texdiff never operates on pylatexenc node classes directly: we convert
+them to our own lightweight :class:`Node` records first. Each node
+
+* carries ``text`` - its EXACT source span, so unchanged nodes round
+  trip byte-identically (critical for listings, attachments and
+  generated tables);
+* carries ``children`` - a list of sub-nodes (only for group-like
+  nodes: environments, braced groups, macro arguments);
+* can be *atomic* (math, verbatim, unknown macros): no children, never
+  diffed internally, compared on exact source equality;
+* knows whether it may carry change markup inside it (``atom`` False
+  only for text-ish nodes; markup is NEVER injected into atomic
+  nodes - that injector is what breaks markup in latexdiff).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Iterator, Optional
+
+# sectioning macros whose {\title} becomes part of the signature
+_SECTIONING_RE = re.compile(
+    r"\s*\\(?:chapter|section|subsection|subsubsection|paragraph|subparagraph)\*?\s*\{"
+)
+
+
+@dataclass(frozen=True)
+class Node:
+    """One node of the concrete syntax tree.
+
+    Attributes:
+        kind: node type discriminator, e.g. ``"env"`` (environment),
+            ``"group"`` (braced group / macro argument), ``"macro"``,
+            ``"text"`` (plain characters run), ``"math"`` (inline or
+            display math), ``"comment"``, ``"specials"``, ``"verb"``
+            (verbatim-like environment, atomic).
+        text: exact source text of the node INCLUDING delimiters
+            (``\\begin{...}...\\end{...}`` for environments, the braces
+            for groups). Unchanged nodes are emitted using this text
+            verbatim, so formatting survives the diff round trip.
+        name: environment or macro name (``None`` for text).
+        children: sub-nodes for group-like nodes; empty otherwise.
+        atom: if True the node is atomic - never diffed internally and
+            never receives inline markup (math, verbatim, unknown
+            macros, specials).
+    """
+
+    kind: str
+    text: str
+    name: Optional[str] = None
+    children: list[Node] = field(default_factory=list)
+    atom: bool = True
+
+    def walk(self) -> Iterator["Node"]:
+        """Yield this node and all descendants, depth first."""
+        yield self
+        for child in self.children:
+            yield from child.walk()
+
+    def signature(self) -> str:
+        """Return a canonical key used to align two nodes.
+
+        Two nodes are aligned candidate-wise when their signatures are
+        equal and their exact source differs (then we diff inside);
+        nodes with equal signature AND equal text are identical.
+
+        For group-like nodes the signature carries a *content anchor*:
+        the key of the first table row found inside (typically the
+        ``\\caption`` row of a ``longtable``). Without it every
+        ``{\\scriptsize \\begin{longtable}...}`` group would share the
+        bare signature ``group`` and a shifted positional match could
+        pair two *different* generated tables - whose column counts
+        then disagree and the marked-up document no longer compiles.
+        The anchor ties each table to its true counterpart as long as
+        its caption (or first row) is stable, which holds even when
+        every data row changed.
+        """
+        if self.name and self.kind not in ("group", "env"):
+            if self.kind == "macro" and _SECTIONING_RE.match(self.text or ""):
+                # sectioning macros: carry the title. Without it every
+                # \subsubsection* shares one signature and a heading
+                # inserted mid-document shifts every later heading by
+                # one - the aligner then retires the old titles just
+                # to re-add them as insertions further down.
+                title = self._heading_title()
+                if title:
+                    return f"{self.kind}:{self.name}:{title}"
+            return f"{self.kind}:{self.name}"
+        if self.kind in ("group", "env"):
+            base = f"{self.kind}:{self.name}" if self.name else self.kind
+            anchor = self._row_anchor()
+            if anchor:
+                return f"{base}:{anchor}"
+            return base
+        return self.kind
+
+    def _heading_title(self) -> str | None:
+        """Normalised title argument of a sectioning macro, if any."""
+        m = re.match(r"\\[a-zA-Z]+\*?\s*(\[[^\]]*\])?\s*\{", self.text or "")
+        if not m:
+            return None
+        title = (self.text or "")[m.end() :]  # after the opening brace
+        depth = 1
+        for i, ch in enumerate(title):
+            if ch == "{" and (i == 0 or title[i - 1] != "\\"):
+                depth += 1
+            elif ch == "}" and (i == 0 or title[i - 1] != "\\"):
+                depth -= 1
+                if depth == 0:
+                    return re.sub(r"\s+", " ", title[:i]).strip() or None
+        return None
+
+    def _row_anchor(self) -> str | None:
+        """Key of the first table row in this subtree, if any."""
+        for node in self.walk():
+            if node.kind == "row" and node.name:
+                return node.name[:64]
+        return None
+
+
+def text_node(s: str) -> Node:
+    """Build a plain text run node."""
+    return Node(kind="text", text=s, atom=True)
+
+
+def env(
+    name: str,
+    *children: Node,
+    text: Optional[str] = None,
+    atom: bool = False,
+) -> Node:
+    """Build an environment node (test helper / prototyping)."""
+    if text is None:
+        body = "".join(c.text for c in children)
+        text = f"\\begin{{{name}}}{body}\\end{{{name}}}"
+    return Node(kind="env", text=text, name=name, children=list(children), atom=atom)
+
+
+def group(*children: Node, name: Optional[str] = None, text: Optional[str] = None) -> Node:
+    """Build a braced group node (test helper / prototyping)."""
+    if text is None:
+        body = "".join(c.text for c in children)
+        text = "{" + body + "}"
+    return Node(kind="group", text=text, name=name, children=list(children), atom=False)

texdiff/oldlines.py

@@ -0,0 +1,49 @@
+"""CROSS-REVISION CONTEXT FOR MARKUP EMISSION.
+
+refine-diff semantics: an added line is only "genuinely new" - and
+rendered blue - when its normalised content did not already exist in
+the OLD revision of the document. Re-emitted table cells (makecell
+bodies that moved between rows, reformatted config listings) carry
+lines that existed verbatim on the old side; marking those blue makes
+the diff noisy and wrong. This module provides global access to a set
+of all old-revision line norms for consult during rendering, sparing
+us a large refactor to thread it through the render call chain.
+"""
+
+from __future__ import annotations
+
+import re
+
+# module-level mutable state (populated from api.diff_documents):
+# the old revision as one blob of concatenated line norms - refine-diff
+# matches by SUBSTRING containment, so a re-emitted row line whose old
+# counterpart carries extra trailing structure (``... \\ \\hline``)
+# still counts as existing
+old_blob: str = ""
+old_lines: set[str] = set()
+
+
+def norm_line(line: str) -> str:
+    """Normalise a source line the way the reference build does."""
+    clean = re.sub(r"%DIF.*$", "", line)
+    clean = re.sub(r"[%\\{}\[\]&]", "", clean)
+    return re.sub(r"\s+", "", clean)
+
+
+def mark_old_lines(source: str) -> None:
+    """Record all content line norms of the OLD revision."""
+    global old_blob, old_lines
+    norms = [norm_line(line) for line in source.split("\n")]
+    old_lines = {n for n in norms if len(n) >= 10}
+    old_blob = "".join(n + "\n" for n in norms if len(n) >= 10)
+
+
+def in_old(line: str) -> bool:
+    """True when a normalised version of `line` existed in the old revision."""
+    norm = norm_line(line)
+    return len(norm) >= 10 and norm in old_blob
+
+
+def is_struct_line(line: str) -> bool:
+    """Bare structure lines never take a colour declaration."""
+    return bool(re.match(r"^\s*\\(?:rowcolor|hline|begin|end|caption)\b", line))