pyDiffTools 0.1.38__tar.gz → 0.1.40__tar.gz

{pydifftools-0.1.38/pyDiffTools.egg-info → pydifftools-0.1.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyDiffTools
-Version: 0.1.38
+Version: 0.1.40
 Summary: Diff tools
 Author: J M Franck
 License: Copyright (c) 2015, jmfranck

{pydifftools-0.1.38 → pydifftools-0.1.40/pyDiffTools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyDiffTools
-Version: 0.1.38
+Version: 0.1.40
 Summary: Diff tools
 Author: J M Franck
 License: Copyright (c) 2015, jmfranck

{pydifftools-0.1.38 → pydifftools-0.1.40}/pydifftools/wrap_sentences.py

@@ -16,6 +16,16 @@ PARAGRAPH_SPLIT = re.compile(r"(\n(?:[ \t]*\n)+)")
 SENTENCE_END = r"[^.!?]{3}[.!?]"
 SENTENCE_SPLIT = re.compile("(" + SENTENCE_END + r")[ \n]")
 SENTENCE_BOUNDARY = re.compile("(" + SENTENCE_END + r")([ \n])")
+# A word is a run of non-space characters, except that an attribute block
+# attached to a figure's closing ")" (or any "{#id ...}" block) stays inside
+# the word, so a break never lands inside a pandoc-crossref marker.
+WORD = re.compile(r"(?:(?<=\))\{[^}]*\}|\{#[^}]*\}|\S)+")
+# A pandoc-crossref marker must directly follow the closing "$$" of its
+# equation or the closing ")" of its figure, and must stay on one line.
+DETACHED_CROSSREF = re.compile(
+    r"(?:(?<!\\)\$\$|\))(?P<gap>[ \t]*\n?[ \t]*)(?=\{#(?:eq|fig):)"
+    r"|(?P<marker>\{#(?:eq|fig):[^}\n]*\n[^}]*\})"
+)
 LATEX_MACRO_SPLIT = re.compile(
     r"(\\(?:begin|end|usepackage|newcommand|section"
     r"|subsection|subsubsection|paragraph|input){[^}]*})"
@@ -584,6 +594,11 @@ def classify_lines(
                     if not equation.endswith("\n"):
                         equation += "\n"
                     equation += "$$"
+                    # keep a pandoc-crossref marker on its closing "$$"
+                    marker = re.match(r"\{#eq:[^}]*\}", content[stop:])
+                    if marker:
+                        equation += marker[0]
+                        stop += marker.end()
                 if stop < len(content):
                     equation += "\n"
                     if content[stop] == "\n":
@@ -686,7 +701,9 @@ def wrap_prose(
         lines = []
         indentation = 0
         for sentence in sentences:
-            words = [word for word in re.split("[ \n]+", sentence) if word]
+            words = [
+                re.sub(r"\s+", " ", word) for word in WORD.findall(sentence)
+            ]
             boundaries = _math_boundaries(words)
             if filetype == "latex":
                 indentation = 0
@@ -794,6 +811,15 @@ def _prepare(filename, cleanoo=False, i=-1):
 def wr(filename, wrapnumber=45, punctuation_slop=20, cleanoo=False, i=-1):
     alltext, filetype, indent_amount, _ = _prepare(filename, cleanoo, i)
     if filetype == "markdown":
+        # reattach detached pandoc-crossref markers before wrapping
+        alltext = DETACHED_CROSSREF.sub(
+            lambda found: (
+                re.sub(r"\s+", " ", found["marker"])
+                if found["marker"]
+                else found[0][: len(found[0]) - len(found["gap"])]
+            ),
+            alltext,
+        )
         result = wrap_blocks(
             markdown_blocks(alltext),
             wrapnumber,
@@ -865,7 +891,7 @@ def check_prose(content, wrapnumber, punctuation_slop, line_start=1):
                 if fragment:
                     # {{{ check line lengths within this sentence fragment
                     words, positions = [], []
-                    for match in re.finditer(r"\S+", fragment):
+                    for match in WORD.finditer(fragment):
                         words.append(match.group())
                         positions.append(frag_offset + match.start())
                     if words:
@@ -942,7 +968,9 @@ def _iter_classified_spans(text, filetype, line_start=1):
     leaf prose blocks -- both just need the same protected-region
     exclusions wr itself uses.
     """
-    classified = classify_lines(text, filetype, normalize_math=True)
+    # normalizing math can add lines (e.g. splitting "$${#eq:x}"), which
+    # would shift every reported line number after it
+    classified = classify_lines(text, filetype)
     line = line_start
     for wrappable, group in itertools.groupby(classified, key=lambda x: x[0]):
         content = "".join(l for _, l in group)
@@ -985,6 +1013,9 @@ def markdown_lint_issues_from_text(
     prose_content = re.sub(
         r"<!--.*?(?:-->|\Z)", mask_comment, content, flags=re.DOTALL
     )
+    # trailing spaces are reported separately below, and a double space
+    # (a hard line break) must not look like a sentence ending mid-line
+    prose_content = re.sub(r"[ \t]+$", "", prose_content, flags=re.M)
     issues = [
         issue
         for line, width, span in _iter_block_spans(
@@ -996,6 +1027,27 @@ def markdown_lint_issues_from_text(
         (issue["line"], issue["message"])
         for issue in unclosed_markdown_spans(content)
     )
+    # {{{ report single trailing spaces and detached crossref markers
+    fence = None
+    for number, line in enumerate(content.splitlines(), start=1):
+        opening = re.match(r"[ \t]*(\x60{3,}|~{3,})", line)
+        if opening and (fence is None or opening[1].startswith(fence)):
+            fence = None if fence else opening[1]
+        elif fence is None and re.search(r"(?<! ) $", line):
+            issues.append((
+                number,
+                "trailing space: remove the single space at the end of this "
+                "line (two spaces are a deliberate hard line break)",
+            ))
+    for found in DETACHED_CROSSREF.finditer(prose_content):
+        if found["gap"] == "":
+            continue
+        issues.append((
+            prose_content.count("\n", 0, found.start()) + 1,
+            "detached crossref marker: keep the {#eq:...} or {#fig:...} "
+            "marker on one line, directly after the closing $$ or )",
+        ))
+    # }}}
     return sorted(issues)
 
 
@@ -1099,6 +1151,33 @@ def apply_markdown_issue_fix(content, line_number, message):
         raise ValueError(
             f"Markdown issue refers to missing line {line_number}"
         )
+    if message.startswith("detached crossref marker"):
+        # {{{ join the marker onto its "$$" or ")" (this can remove a line)
+        start = sum(len(line) for line in lines[:index])
+        found = next(
+            found
+            for found in DETACHED_CROSSREF.finditer(content, start)
+            if found["gap"] != ""
+        )
+        if found["gap"] is None:
+            span = found.span("marker")
+            joined = re.sub(r"\s+", " ", found["marker"])
+        else:
+            span = found.span("gap")
+            joined = ""
+        before = content[found.start() : found.end()]
+        updated = content[: span[0]] + joined + content[span[1] :]
+        after = updated[
+            found.start() : span[0] + len(joined) + found.end() - span[1]
+        ]
+        return (
+            updated,
+            before,
+            after,
+            "A crossref marker was separated from its equation or figure. "
+            "I joined it back on.",
+        )
+        # }}}
     old_line = lines[index]
     ending = ""
     if old_line.endswith("\r\n"):
@@ -1121,20 +1200,33 @@ def apply_markdown_issue_fix(content, line_number, message):
     else:
         prefix = re.match(r"^[ \t]*", old_line).group()
     # }}}
+    if message.startswith("trailing space:"):
+        lines[index] = old_line[:-1] + ending
+        return (
+            "".join(lines),
+            old_line,
+            old_line[:-1],
+            "A line ended with a single stray space. I removed it.",
+        )
     if message.startswith("line too long:"):
-        match = re.search(r"before '([^']+)'", message)
+        match = re.search(r"after '(.+?)' \(before '(.+?)'\); move", message)
         if match is None:
             raise ValueError(
                 f"Cannot find the suggested word in {message!r}"
             )
-        word = re.escape(match.group(1))
-        position = re.search(r"(?<!\S)" + word + r"(?!\S)", old_line)
+        # split at the suggested word that follows wr's break word, since the
+        # same word can appear earlier on the line (e.g. "for ... ($3x$ for")
+        position = re.search(
+            r"(?<!\S)" + re.escape(match.group(1)) + r"[ \t]+("
+            + re.escape(match.group(2)) + r")(?!\S)",
+            old_line,
+        )
         if position is None:
             raise ValueError(
-                f"Cannot find {match.group(1)!r} on Markdown line "
-                f"{line_number}"
+                f"Cannot find {match.group(2)!r} after {match.group(1)!r} on "
+                f"Markdown line {line_number}"
             )
-        split_at = position.start()
+        split_at = position.start(1)
         first = old_line[:split_at].rstrip()
         second = old_line[split_at:].lstrip(" \t")
         reason = (
@@ -1157,8 +1249,12 @@ def apply_markdown_issue_fix(content, line_number, message):
     else:
         raise ValueError(f"No automatic fix is available for {message!r}")
 
-    fixed_line = first + ending + prefix + second + ending
-    lines[index] = fixed_line
+    if not first.strip() or not second.strip():
+        # a break here would insert a blank line, i.e. a paragraph break
+        raise ValueError(
+            f"Automatic fix would leave an empty line at {line_number}"
+        )
+    lines[index] = first + ending + prefix + second + ending
     after = first + "\n" + prefix + second
     return "".join(lines), old_line, after, reason
 
@@ -1168,39 +1264,44 @@ def autofix_markdown_file(filename, wrapnumber=55, punctuation_slop=20):
     with open(filename, encoding="utf-8", newline="") as fp:
         content = fp.read()
     fixes = []
-    for _ in range(1000):
+    # Lint once, fix every flagged line, then lint again, since fixing one
+    # issue at a time re-lints the whole file for every single fix.
+    for _ in range(100):
         issues = markdown_lint_issues_from_text(
             content, wrapnumber=wrapnumber, punctuation_slop=punctuation_slop
         )
-        fixable = next(
-            (
-                issue
-                for issue in issues
-                if not issue[1].startswith(
-                    ("unclosed math:", "unclosed HTML comment:")
-                )
-            ),
-            None,
-        )
-        if fixable is None:
+        fixable = {}
+        for line_number, message in issues:
+            if not message.startswith(
+                ("unclosed math:", "unclosed HTML comment:")
+            ):
+                # a fix only rewrites its own line, so fix one issue per line
+                # per pass and let the next lint catch whatever remains
+                fixable.setdefault(line_number, message)
+        if not fixable:
             break
-        line_number, message = fixable
-        updated, before, after, reason = apply_markdown_issue_fix(
-            content, line_number, message
-        )
-        if updated == content:
-            raise RuntimeError(
-                f"Automatic Markdown fix made no change at line {line_number}"
+        # work from the bottom up so each inserted line break leaves the
+        # line numbers of the remaining issues unchanged
+        pass_fixes = []
+        for line_number in sorted(fixable, reverse=True):
+            updated, before, after, reason = apply_markdown_issue_fix(
+                content, line_number, fixable[line_number]
             )
-        fixes.append(
-            {
-                "line": line_number,
-                "reason": reason,
-                "before": before,
-                "after": after,
-            }
-        )
-        content = updated
+            if updated == content:
+                raise RuntimeError(
+                    "Automatic Markdown fix made no change at line "
+                    f"{line_number}"
+                )
+            pass_fixes.append(
+                {
+                    "line": line_number,
+                    "reason": reason,
+                    "before": before,
+                    "after": after,
+                }
+            )
+            content = updated
+        fixes.extend(reversed(pass_fixes))
     else:
         raise RuntimeError("Too many automatic Markdown fixes were needed.")
 

{pydifftools-0.1.38 → pydifftools-0.1.40}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pyDiffTools"
-version = "0.1.38"
+version = "0.1.40"
 authors = [{ name = "J M Franck" }]
 description = "Diff tools"
 readme = "README.rst"

{pydifftools-0.1.38 → pydifftools-0.1.40}/tests/test_wrap_check.py

@@ -1,3 +1,4 @@
+import re
 import subprocess
 
 import pytest
@@ -120,6 +121,138 @@ def test_one_reported_long_line_is_fixed_at_the_suggested_word():
     assert "hard to read" in reason
 
 
+def test_lint_line_numbers_survive_labeled_display_math():
+    # normalizing "$${#eq:x}" for wrapping splits it onto two lines, which
+    # must not shift the line numbers reported for later prose
+    source = (
+        "Intro.\n$$\nx = y\n$${#eq:x}\nwhere this sentence ends. "
+        "Another starts.\n"
+    )
+
+    ((line, message),) = markdown_lint_issues_from_text(source, 55)
+
+    assert line == 5
+    assert message.startswith("sentence ends mid-line")
+
+
+def test_long_line_fix_splits_at_the_word_after_the_break():
+    # "for" also starts the line, but wr breaks before the second one
+    source = (
+        "for $30\\;\\text{s}$ ($3\\times$ for AOT) with some more words "
+        "added.\n"
+    )
+    message = (
+        "line too long: wr would break after '($3\\times$' (before 'for'); "
+        "move 'for' onward to the next line, or shorten this sentence"
+    )
+
+    fixed, _, _, _ = apply_markdown_issue_fix(source, 1, message)
+
+    assert fixed.startswith("for $30\\;\\text{s}$ ($3\\times$\nfor AOT)")
+
+
+def test_single_trailing_space_is_flagged_and_removed(tmp_path):
+    # from RM_ESR: a sentence ending in a stray space
+    path = tmp_path / "source.md"
+    path.write_text("It melts near 238 K. \nIn contrast, it stays.\n")
+
+    issues = markdown_lint_issues_from_text(path.read_text(), wrapnumber=55)
+    autofix_markdown_file(path)
+
+    assert (1, "trailing space") in [
+        (line, message.split(":")[0]) for line, message in issues
+    ]
+    fixed = path.read_text()
+    assert fixed.startswith("It melts near 238 K.\nIn contrast,")
+    assert " \n" not in fixed
+
+
+def test_double_trailing_space_hard_break_is_allowed():
+    source = "The first line ends here with a break  \nand continues.\n"
+
+    assert markdown_lint_issues_from_text(source, wrapnumber=55) == []
+
+
+@pytest.mark.parametrize(
+    "detached",
+    [
+        # equation markers must sit directly on the closing $$
+        "Intro.\n$$\nx = y\n$$ {#eq:x}\nwhere this holds.\n",
+        "Intro.\n$$\nx = y\n$$\n{#eq:x}\nwhere this holds.\n",
+        # figure markers must sit directly on the closing ) (from eigenmode)
+        "![A caption.](./media/BothProbes_Bfields.png)\n"
+        '{#fig:ProbeMagFieldsDrawing width="4in"}\n',
+        # and must not be split across lines (from RM_ESR)
+        "![A caption for\nspectroscopy.](Figures/simple_oned_CAT16.png)"
+        "{#fig:justSpectraCat16\nwidth=4in}\n",
+    ],
+)
+def test_detached_crossref_marker_is_flagged_and_joined(tmp_path, detached):
+    path = tmp_path / "source.md"
+    path.write_text(detached)
+
+    issues = markdown_lint_issues_from_text(detached, wrapnumber=55)
+    report = autofix_markdown_file(path)
+
+    fixed = path.read_text()
+    assert [message.split(":")[0] for _, message in issues] == [
+        "detached crossref marker"
+    ]
+    assert report["fixes"]
+    assert markdown_lint_issues_from_text(fixed, wrapnumber=55) == []
+    assert re.search(r"(\$\$|\))\{#(eq|fig):[^}\n]*\}", fixed)
+
+
+def test_figure_crossref_marker_is_never_split_from_its_figure(tmp_path):
+    # from RM_ESR: a caption spanning lines, ending in a marker with
+    # attributes; wr and the linter must treat "...){#fig:... }" as one word
+    source = (
+        "![Room temperature ESR of 100 mM TEMPO-SO~4~ RMs with water "
+        "loading $w_0=30$ in isooctane dispersant (blue), single HE "
+        "simulation (orange), multi HE simulation "
+        "(green)](Figures/HE_Easyspin_Simul_100mM.png)\n"
+        "{#fig:HESimul width=5.5in}\n"
+    )
+    path = tmp_path / "source.md"
+    path.write_text(source)
+
+    wr(str(path), 55)
+
+    wrapped = path.read_text()
+    assert (
+        "(green)](Figures/HE_Easyspin_Simul_100mM.png)"
+        "{#fig:HESimul width=5.5in}\n"
+    ) in wrapped
+    assert markdown_lint_issues_from_text(wrapped, wrapnumber=55) == []
+
+
+def test_wr_keeps_equation_marker_on_closing_dollars(tmp_path):
+    path = tmp_path / "source.md"
+    path.write_text(
+        "Intro.\n$$\nx = y\n$$ {#eq:tauFromLinewidth}\nwhere this holds.\n"
+    )
+
+    wr(str(path), 55)
+
+    wrapped = path.read_text()
+    assert "\n$${#eq:tauFromLinewidth}\n" in wrapped
+    assert markdown_lint_issues_from_text(wrapped, wrapnumber=55) == []
+
+
+def test_autofix_fixes_every_line_in_one_pass_from_the_end(tmp_path):
+    path = tmp_path / "source.md"
+    paragraph = "First sentence ends here. A second sentence begins.\n"
+    path.write_text("\n".join([paragraph] * 30))
+
+    report = autofix_markdown_file(path)
+
+    fixed = path.read_text()
+    assert len(report["fixes"]) == 30
+    assert [fix["line"] for fix in report["fixes"]] == list(range(1, 60, 2))
+    assert fixed.count("\n\n") == 29
+    assert markdown_lint_issues_from_text(fixed, wrapnumber=55) == []
+
+
 def test_unclosed_math_is_reported_after_fifty_lines():
     short_math = "$$\n" + ("x = y\n" * 48)
     long_math = "$$\n" + ("x = y\n" * 49)