texdiff 0.2.0__py3-none-any.whl

texdiff/__init__.py

@@ -0,0 +1,45 @@
+"""texdiff - AST-driven semantic diff for LaTeX documents.
+
+The pipeline IS::
+
+    old.tex ─┐
+             ├─> parse → align trees → emit markup → diff.tex
+    new.tex ─┘
+
+ latexdiff diffs characters and breaks structure;
+ texdiff diffs structure and never breaks characters.
+"""
+
+from .align import Delete, Edit, Insert, Match, Modify, align
+from .api import DiffResult, DiffStats, diff_documents, diff_files
+from .emit import LatexdiffMarkup, render
+from .flatten import Flattener, flatten_file, flatten_source
+from .nodes import Node
+from .parse import ParseError, parse, parse_file
+from .textdiff import Chunk, word_diff
+
+__all__ = [
+    "align",
+    "Chunk",
+    "Delete",
+    "diff_documents",
+    "diff_files",
+    "DiffResult",
+    "DiffStats",
+    "Edit",
+    "Flattener",
+    "flatten_file",
+    "flatten_source",
+    "Insert",
+    "LatexdiffMarkup",
+    "Match",
+    "Modify",
+    "Node",
+    "parse",
+    "ParseError",
+    "parse_file",
+    "render",
+    "word_diff",
+]
+
+__version__ = "0.1.0"

texdiff/align.py

@@ -0,0 +1,202 @@
+"""Tree alignment: match two node lists into aligned pairs and edits.
+
+The aligner is where latexdiff's heuristics become data. It produces,
+for a pair of node lists, an edit script of three shapes:
+
+* ``match(old, new)`` - same signature, identical text → keep verbatim
+* ``modify(old, new)`` - same signature, different text → recurse if
+  both sides have children (block diff), else whole-block replacement
+* ``insert(new)`` / ``delete(old)`` - no counterpart
+
+Uses :class:`difflib.SequenceMatcher` over node signatures; runs of
+equal signature with different text become ``modify`` pairs aligned
+positionally within the run.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from difflib import SequenceMatcher
+
+from .nodes import Node
+
+
+@dataclass(frozen=True)
+class Match:
+    """A node kept verbatim (identical on both sides)."""
+
+    node: Node
+
+
+@dataclass(frozen=True)
+class Modify:
+    """A node pair to be diffed further (recursed) - or replaced.
+
+    Attributes:
+        old: the node from the old revision.
+        new: the node from the new revision.
+        inner: the recursive edit script of the children, when both
+            sides are recursable; ``None`` means whole-block replace.
+    """
+
+    old: Node
+    new: Node
+    inner: "list[Edit] | None" = None
+
+
+@dataclass(frozen=True)
+class Insert:
+    """A node present only in the new version."""
+
+    new: Node
+
+
+@dataclass(frozen=True)
+class Delete:
+    """A node present only in the old version."""
+
+    old: Node
+
+
+Edit = Match | Modify | Insert | Delete
+
+
+def align(old: list[Node], new: list[Node]) -> list[Edit]:
+    """Align two node lists into an edit script.
+
+    The returned script is in document order, interleaving old and new
+    positions like a unified diff.
+
+    Algorithm:
+
+    1. an equal *prefix* and *suffix* of signatures anchors the
+       alignment first: ``SequenceMatcher`` finds the globally
+       longest matching block, which can let a later identical run
+       steal the match from the earliest identical nodes - an
+       inserted chapter between two near-identical passages then
+       drags unchanged leading lines into the insertion (they
+       render as added);
+    2. ``SequenceMatcher`` over the remaining node *signatures*
+       yields runs of same-signature positions deemed "matching";
+    3. within a run, nodes are paired positionally: identical source
+       text becomes :class:`Match`, differing text :class:`Modify`;
+    4. gap regions become ``Delete`` (old-only) then ``Insert``
+       (new-only) sequences, preserving document order.
+    """
+
+    def _pair(o: Node, n: Node) -> Edit:
+        if o.text == n.text:
+            return Match(node=o)
+        if o.kind == "row" and n.kind == "row" and o.name == n.name:
+            # equal row signature (content key) with differing text
+            # means the difference is purely structural: \hline on
+            # the other side of the row, whitespace, comments.
+            # Duplicating such a pair (del + add) would emit the
+            # table header material (\endfirsthead, \endhead ...)
+            # twice inside one longtable, which can throw longtable
+            # into an infinite loop. Keep the old row verbatim.
+            return Match(node=o)
+        return Modify(old=o, new=n)
+
+    # 1. common prefix, paired positionally (identical to what a
+    # matching-block run does - only the anchoring is stronger)
+    pre = 0
+    while (
+        pre < len(old)
+        and pre < len(new)
+        and old[pre].signature() == new[pre].signature()
+    ):
+        pre += 1
+    # common suffix (may not overlap the prefix)
+    suf = 0
+    while (
+        suf < len(old) - pre
+        and suf < len(new) - pre
+        and old[len(old) - 1 - suf].signature() == new[len(new) - 1 - suf].signature()
+    ):
+        suf += 1
+    mid_old = old[pre : len(old) - suf]
+    mid_new = new[pre : len(new) - suf]
+
+    sm = SequenceMatcher(
+        a=[n.signature() for n in mid_old],
+        b=[n.signature() for n in mid_new],
+        autojunk=False,
+    )
+
+    edits: list[Edit] = [_pair(o, n) for o, n in zip(old[:pre], new[:pre])]
+    prev_a = prev_b = 0
+    for block in sm.get_matching_blocks():
+        # gap before this matching block: deletions then insertions.
+        # Block-head refinement: ``SequenceMatcher`` extends a matching
+        # block as far as equal *signatures* reach - with the wildcard
+        # ``text`` signature, a run may begin by pairing two text nodes
+        # whose contents share nothing, while the node that should pair
+        # with the old head sits at the START of the insertion gap just
+        # before the block. When that pairing is clearly better, consume
+        # the gap head into a Modify pair, emit the rest of the gap as
+        # inserts, and let the block start one position later on both
+        # sides (its first new-side node then lands at the gap end).
+        new_gap = mid_new[prev_b : block.b]
+        head_a, head_b = block.a, block.b
+        if (
+            block.size
+            and new_gap
+            and mid_old[block.a].kind == "text"
+            and new_gap[0].kind == "text"
+            and mid_new[block.b].kind == "text"
+        ):
+            o_head = mid_old[block.a]
+            best = max(
+                range(min(len(new_gap), 4)),
+                key=lambda k: _text_similarity(o_head.text, new_gap[k].text),
+            )
+            n_head = new_gap[best]
+            if _text_similarity(o_head.text, n_head.text) > (
+                _text_similarity(o_head.text, mid_new[block.b].text) + 0.2
+            ):
+                edits.extend(Delete(old=n) for n in mid_old[prev_a : block.a])
+                edits.extend(Insert(new=n) for n in new_gap[:best])
+                edits.append(_pair(o_head, n_head))
+                edits.extend(Insert(new=n) for n in new_gap[best + 1 :])
+                edits.append(Insert(new=mid_new[block.b]))
+                head_a, head_b = block.a + 1, block.b + 1
+                for i in range(1, block.size):
+                    edits.append(
+                        _pair(mid_old[block.a + i], mid_new[block.b + i])
+                    )
+                prev_a, prev_b = block.a + block.size, block.b + block.size
+                continue
+        edits.extend(Delete(old=n) for n in mid_old[prev_a : block.a])
+        edits.extend(Insert(new=n) for n in new_gap)
+        # the matching run: pair positionally, decide Match vs Modify
+        for i in range(head_a, head_a + block.size):
+            edits.append(
+                _pair(mid_old[i], mid_new[head_b + (i - head_a)])
+            )
+        prev_a, prev_b = head_a + block.size, head_b + block.size
+    edits.extend(Delete(old=n) for n in mid_old[prev_a:])
+    edits.extend(Insert(new=n) for n in mid_new[prev_b:])
+    # common suffix, in document order
+    edits.extend(
+        _pair(o, n)
+        for o, n in zip(
+            old[len(old) - suf :], new[len(new) - suf :]
+        )
+    )
+    return edits
+
+
+def _text_similarity(a: str, b: str) -> float:
+    """Quick content similarity of two text runs, in ``[0, 1]``.
+
+    Used only to compare pairing candidates at matching-block
+    boundaries, so a cheap ``SequenceMatcher.ratio`` over word
+    tokens (not raw characters) is enough - and stays stable for
+    long runs where character noise would dominate.
+    """
+    wa = [t for t in a.split() if t]
+    wb = [t for t in b.split() if t]
+    if not wa or not wb:
+        return 0.0
+    return SequenceMatcher(a=wa, b=wb, autojunk=False).ratio()

texdiff/api.py

@@ -0,0 +1,326 @@
+"""Public API of texdiff.
+
+Three functions make up the pipeline::
+
+    from texdiff import parse, align, render
+    from texdiff import diff_documents
+
+    result = diff_documents(old_source, new_source)
+    print(result.marked_up)   # the diff.tex source
+    print(result.stats)       # counts of edits
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .align import Delete, Edit, Insert, Match, Modify, align
+
+# word-similarity floor below which a paired text run is retired and
+# re-added wholesale instead of word-marked: two sentences sharing
+# less than half their words are a rewrite, not an edit
+_REPLACE_WORD_SIMILARITY = 0.5
+from .emit import LatexdiffMarkup, render
+from .flatten import Flattener, flatten_file, flatten_source
+from .nodes import Node, text_node
+from .parse import parse
+from .preamble import (
+    count_preamble_changes,
+    mark_preamble_macro_changes,
+    split_preamble,
+)
+from .textdiff import DELETE, EQUAL, Chunk, word_diff
+
+
+@dataclass(frozen=True)
+class DiffStats:
+    """Aggregated change counts of one diff."""
+
+    matches: int = 0
+    modifications: int = 0
+    insertions: int = 0
+    deletions: int = 0
+    preamble_changes: int = 0
+
+    @property
+    def changed(self) -> int:
+        """Number of changed blocks (modify+insert+delete)."""
+        return self.modifications + self.insertions + self.deletions
+
+    def __add__(self, other: "DiffStats") -> "DiffStats":
+        """Sum two stats (used to layer preamble counts onto body counts)."""
+        return DiffStats(
+            matches=self.matches + other.matches,
+            modifications=self.modifications + other.modifications,
+            insertions=self.insertions + other.insertions,
+            deletions=self.deletions + other.deletions,
+            preamble_changes=self.preamble_changes + other.preamble_changes,
+        )
+
+    def __str__(self) -> str:  # pragma: no cover - cosmetic
+        pre = (
+            f", {self.preamble_changes} preamble lines changed"
+            if self.preamble_changes
+            else ""
+        )
+        return (
+            f"{self.matches} unchanged, {self.modifications} modified, "
+            f"{self.insertions} added, {self.deletions} deleted{pre}"
+        )
+
+
+@dataclass(frozen=True)
+class DiffResult:
+    """Outcome of diffing two documents.
+
+    Attributes:
+        marked_up: the LaTeX source with \\DIFadd/\\DIFdel markup.
+        stats: change counts.
+        edits: the full edit script (for tools/UIs on top).
+    """
+
+    marked_up: str
+    stats: DiffStats
+    edits: list[Edit]
+
+
+def diff_documents(
+    old_source: str,
+    new_source: str,
+    markup: LatexdiffMarkup = LatexdiffMarkup(),
+    inject_preamble: bool = True,
+) -> DiffResult:
+    """Diff two LaTeX documents (flattened sources).
+
+    Args:
+        old_source: old revision LaTeX source.
+        new_source: new revision LaTeX source.
+        markup: markup style for additions/deletions.
+        inject_preamble: insert the markup macro definitions
+            (``\\DIFadd`` etc.) before ``\\begin{document}`` so the
+            output compiles standalone; skipped when the document
+            defines them already or has no document body.
+
+    Returns:
+        :class:`DiffResult` with the marked-up document.
+    """
+    # preamble policy: the output uses the NEW preamble; preamble
+    # differences are counted, never marked up (see texdiff.preamble)
+    pre_old, body_old, post_old = split_preamble(old_source)
+    pre_new, body_new, post_new = split_preamble(new_source)
+    preamble_changes = count_preamble_changes(old_source, new_source)
+
+    # cross-revision context: added lines that already existed in the
+    # old revision render black, not blue (refine-diff semantics)
+    from . import oldlines
+
+    oldlines.mark_old_lines(old_source)
+
+    edits = _diff_nodes(parse(body_old), parse(body_new))
+    body_markup = render(edits, markup)
+    stats = _count(edits) + DiffStats(preamble_changes=preamble_changes)
+    marked_up = pre_new + body_markup + post_new if pre_new else body_markup
+    marked_up = _fix_listing_languages(marked_up)
+    if pre_new:
+        # change-record and other body-invoked preamble macros:
+        # their typeset content would otherwise silently swallow
+        # additions (preamble policy keeps the new revision as-is)
+        marked_up = mark_preamble_macro_changes(marked_up, old_source, new_source)
+    if inject_preamble and stats.changed:
+        marked_up = _inject_preamble(marked_up, new_source)
+    return DiffResult(marked_up=marked_up, stats=stats, edits=edits)
+
+
+def _fix_listing_languages(marked_up: str) -> str:
+    """Drop ``language=<lang>,`` from document-level ``\\lstset`` calls.
+
+    The marked-up listings carry ``[alsolanguage=DIFcode]`` so the
+    ``%DIF`` line markers render as strikeout/blue instead of literal
+    text. But a primary language with C-style *block comments*
+    (``/** ... */``) swallows the markers inside comment regions -
+    listings stops processing delimiters once a block comment opens.
+    The reference pipeline strips ``language=C++`` for exactly this
+    reason; keyword colouring is a visual nicety, marker visibility
+    is the point of a diff document.
+    """
+    import re
+
+    return re.sub(r"(\\lstset\{)language=[A-Za-z0-9+#]*,", r"\1", marked_up)
+
+
+def _inject_preamble(marked_up: str, source: str) -> str:
+    """Insert PREAMBLE_TEMPLATE before ``\\begin{document}``.
+
+    Follows latexdiff's convention; skips injection when the source
+    already defines ``\\DIFadd`` or has no ``\\begin{document}```
+    (a fragment - the caller provides definitions).
+    """
+    if "\\DIFadd" in source:
+        return marked_up
+    idx = marked_up.find("\\begin{document}")
+    if idx < 0:
+        return marked_up
+    from .emit import PREAMBLE_TEMPLATE
+
+    return marked_up[:idx] + PREAMBLE_TEMPLATE + marked_up[idx:]
+
+
+def diff_files(
+    old_path: str,
+    new_path: str,
+    flatten: bool = True,
+    **kwargs,
+) -> DiffResult:
+    """Diff two LaTeX files (UTF-8).
+
+    By default the sources are flattened first (``\\input``/``\\include``
+    expanded), like ``latexdiff --flatten``; pass ``flatten=False`` to
+    compare already-flattened sources.
+    """
+    from pathlib import Path
+
+    def read(p):
+        return Path(p).read_text(encoding="utf-8")
+
+    old_source = flatten_file(old_path) if flatten else read(old_path)
+    new_source = flatten_file(new_path) if flatten else read(new_path)
+    return diff_documents(old_source, new_source, **kwargs)
+
+
+def _diff_nodes(old: list[Node], new: list[Node]) -> list[Edit]:
+    """Align, then recurse into modified pairable nodes.
+
+    A ``Modify`` pair whose both sides have children is re-aligned at
+    the next level down (e.g. a changed paragraph lives inside the
+    ``document`` environment); the recursion result is carried on the
+    ``Modify`` (``inner``) and rendered in place of the whole-block
+    del/add replacement.
+
+    A ``Modify`` pair of two *text* nodes is refined to a word-level
+    diff so individual words - not whole paragraphs - get marked up.
+    """
+    edits = align(old, new)
+    result: list[Edit] = []
+    for edit in edits:
+        if (
+            isinstance(edit, Modify)
+            and edit.old.children
+            and edit.new.children
+        ):
+            inner = _diff_nodes(edit.old.children, edit.new.children)
+            result.append(Modify(old=edit.old, new=edit.new, inner=inner))
+        elif (
+            isinstance(edit, Modify)
+            and edit.old.kind == "text"
+            and edit.new.kind == "text"
+        ):
+            # heavily rewritten paragraphs: interleaving word marks
+            # between two sentences sharing less than half their words
+            # reads as word salad. Such paragraphs retire wholesale -
+            # delete + re-add, the track-changes convention - while
+            # well-matched paragraphs of the same run keep their
+            # word-level treatment.
+            para_edits = _paragraph_edits(edit.old.text, edit.new.text)
+            if para_edits is None:
+                result.append(edit)
+            else:
+                result.extend(para_edits)
+        else:
+            result.append(edit)
+    return result
+
+
+def _run_similarity(a: str, b: str) -> float:
+    """Word-level similarity of two text runs, in ``[0, 1]``.
+
+    Tokens are compared with edge punctuation stripped, so
+    ``text.`` matches ``text`` - the word differ itself works at
+    that granularity and the retire/re-add decision must agree
+    with it, or trivially reworded sentences (``Body text.`` to
+    ``Body text changed.``) would look like rewrites.
+    """
+    from difflib import SequenceMatcher
+
+    wa = [w for w in (t.strip(".,;:!?()\"'`") for t in a.split()) if w]
+    wb = [w for w in (t.strip(".,;:!?()\"'`") for t in b.split()) if w]
+    if not wa or not wb:
+        return 0.0
+    return SequenceMatcher(a=wa, b=wb, autojunk=False).ratio()
+
+
+def _paragraph_edits(old: str, new: str) -> list[Edit] | None:
+    """Word-refine a text run paragraph by paragraph.
+
+    A text run spans everything up to the next macro/environment -
+    often several paragraphs. Blank-line-separated paragraphs pair
+    positionally; each pair either takes the word-level diff or, when
+    the two paragraphs share less than half their words, the
+    whole-paragraph delete + re-add. Whitespace around and between
+    paragraphs keeps the NEW bytes - separator differences are
+    invisible and must not produce marks.
+
+    Returns ``None`` when paragraph counts do not match or the run
+    is a wholesale rewrite: the caller then keeps the block-level
+    ``Modify`` (whole-run retire + re-add).
+    """
+
+    def split_run(s: str) -> tuple[str, list[str], str]:
+        lead = s[: len(s) - len(s.lstrip())]
+        trail = s[len(s.rstrip()) :]
+        core = s[len(lead) : len(s) - len(trail)]
+        return lead, re.split(r"\n\s*\n", core), trail
+
+    lead_a, paras_a, trail_a = split_run(old)
+    lead_b, paras_b, trail_b = split_run(new)
+    if len(paras_a) != len(paras_b) or any(
+        not p.strip() for p in paras_a + paras_b
+    ):
+        # mismatched paragraph structure: whole-run judgement
+        return (
+            None
+            if _run_similarity(old, new) < _REPLACE_WORD_SIMILARITY
+            else _chunks_to_edits(word_diff(old, new))
+        )
+
+    def sep(text: str) -> Edit:
+        # paragraph separator, kept from the NEW side when it exists
+        return Match(node=text_node(text if text.strip() else "\n\n"))
+
+    out: list[Edit] = [sep(lead_b or lead_a)]
+    for k, (ca, cb) in enumerate(zip(paras_a, paras_b)):
+        if _run_similarity(ca, cb) < _REPLACE_WORD_SIMILARITY:
+            out.append(Delete(old=text_node(ca)))
+            out.append(Insert(new=text_node(cb)))
+        else:
+            out.extend(_chunks_to_edits(word_diff(ca, cb)))
+        if k + 1 < len(paras_a):
+            out.append(sep("\n\n"))
+    out.extend(_chunks_to_edits(word_diff(trail_a, trail_b)))
+    return out
+
+
+def _chunks_to_edits(chunks: list[Chunk]) -> list[Edit]:
+    """Convert word-diff chunks to a flat list of atomic edits."""
+    out: list[Edit] = []
+    for chunk in chunks:
+        if chunk.op == EQUAL:
+            out.append(Match(node=text_node(chunk.text)))
+        elif chunk.op == DELETE:
+            out.append(Delete(old=text_node(chunk.text)))
+        else:  # insert
+            out.append(Insert(new=text_node(chunk.text)))
+    return out
+
+def _count(edits: list[Edit]) -> DiffStats:
+    m = i = d = n = 0
+    for e in edits:
+        if isinstance(e, Match):
+            m += 1
+        elif isinstance(e, Modify):
+            n += 1
+        elif isinstance(e, Insert):
+            i += 1
+        elif isinstance(e, Delete):
+            d += 1
+    return DiffStats(matches=m, modifications=n, insertions=i, deletions=d)

texdiff/check.py

@@ -0,0 +1,61 @@
+"""--check: compile the marked-up diff once, report/fail loudly.
+
+The whole point of texdiff is diffs that compile. ``--check`` makes
+that property *verified* rather than trusted: it writes the marked-up
+document into a scratch directory, runs one ``pdflatex`` pass, and
+returns the verdict with the log tail on failure.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import tempfile
+from pathlib import Path
+
+from .api import diff_files
+
+_PDFLATEX_TIMEOUT = 120
+
+
+def run_pdflatex(tex_path: Path, work_dir: Path) -> subprocess.CompletedProcess:
+    """Run one nonstop pdflatex pass in ``work_dir``."""
+    return subprocess.run(
+        [
+            "pdflatex",
+            "-interaction=nonstopmode",
+            "-halt-on-error",
+            "-output-directory",
+            str(work_dir),
+            str(tex_path),
+        ],
+        cwd=work_dir,
+        capture_output=True,
+        text=True,
+        timeout=_PDFLATEX_TIMEOUT,
+    )
+
+
+def check_compiles(old_path, new_path, work_dir: Path | None = None) -> tuple[bool, str]:
+    """Diff two files and verify the marked-up result compiles.
+
+    Returns:
+        (ok, log): ok is True when pdflatex exited 0; log is the
+        pdflatex output (empty string when pdflatex is unavailable).
+    """
+    if work_dir is None:
+        work_dir = Path(tempfile.mkdtemp(prefix="texdiff-check-"))
+    else:
+        work_dir = Path(work_dir)
+        work_dir.mkdir(parents=True, exist_ok=True)
+
+    result = diff_files(str(old_path), str(new_path))
+    tex = work_dir / "texdiff-check.tex"
+    tex.write_text(result.marked_up, encoding="utf-8")
+
+    from shutil import which
+
+    if which("pdflatex") is None:  # pragma: no cover - environment
+        return True, ""
+
+    cp = run_pdflatex(tex, work_dir)
+    return cp.returncode == 0, cp.stdout + cp.stderr