pyDiffTools 0.1.39__tar.gz → 0.1.41__tar.gz

{pydifftools-0.1.39/pyDiffTools.egg-info → pydifftools-0.1.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyDiffTools
-Version: 0.1.39
+Version: 0.1.41
 Summary: Diff tools
 Author: J M Franck
 License: Copyright (c) 2015, jmfranck

{pydifftools-0.1.39 → pydifftools-0.1.41/pyDiffTools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyDiffTools
-Version: 0.1.39
+Version: 0.1.41
 Summary: Diff tools
 Author: J M Franck
 License: Copyright (c) 2015, jmfranck

{pydifftools-0.1.39 → pydifftools-0.1.41}/pydifftools/continuous.py

@@ -310,7 +310,7 @@ def run_pandoc(
 
     automatic_fixes_were_made = False
     while True:
-        report = autofix_markdown_file(filename, wrapnumber=55)
+        report = autofix_markdown_file(filename, max_line_length=79)
         automatic_fixes_were_made |= bool(report["fixes"])
         if report["fixes"] or report["warnings"]:
             user_says_fixed = show_markdown_fix_dialog(report)

{pydifftools-0.1.39 → pydifftools-0.1.41}/pydifftools/wrap_sentences.py

@@ -16,6 +16,16 @@ PARAGRAPH_SPLIT = re.compile(r"(\n(?:[ \t]*\n)+)")
 SENTENCE_END = r"[^.!?]{3}[.!?]"
 SENTENCE_SPLIT = re.compile("(" + SENTENCE_END + r")[ \n]")
 SENTENCE_BOUNDARY = re.compile("(" + SENTENCE_END + r")([ \n])")
+# A word is a run of non-space characters, except that an attribute block
+# attached to a figure's closing ")" (or any "{#id ...}" block) stays inside
+# the word, so a break never lands inside a pandoc-crossref marker.
+WORD = re.compile(r"(?:(?<=\))\{[^}]*\}|\{#[^}]*\}|\S)+")
+# A pandoc-crossref marker must directly follow the closing "$$" of its
+# equation or the closing ")" of its figure, and must stay on one line.
+DETACHED_CROSSREF = re.compile(
+    r"(?:(?<!\\)\$\$|\))(?P<gap>[ \t]*\n?[ \t]*)(?=\{#(?:eq|fig):)"
+    r"|(?P<marker>\{#(?:eq|fig):[^}\n]*\n[^}]*\})"
+)
 LATEX_MACRO_SPLIT = re.compile(
     r"(\\(?:begin|end|usepackage|newcommand|section"
     r"|subsection|subsubsection|paragraph|input){[^}]*})"
@@ -584,6 +594,11 @@ def classify_lines(
                     if not equation.endswith("\n"):
                         equation += "\n"
                     equation += "$$"
+                    # keep a pandoc-crossref marker on its closing "$$"
+                    marker = re.match(r"\{#eq:[^}]*\}", content[stop:])
+                    if marker:
+                        equation += marker[0]
+                        stop += marker.end()
                 if stop < len(content):
                     equation += "\n"
                     if content[stop] == "\n":
@@ -686,7 +701,9 @@ def wrap_prose(
         lines = []
         indentation = 0
         for sentence in sentences:
-            words = [word for word in re.split("[ \n]+", sentence) if word]
+            words = [
+                re.sub(r"\s+", " ", word) for word in WORD.findall(sentence)
+            ]
             boundaries = _math_boundaries(words)
             if filetype == "latex":
                 indentation = 0
@@ -794,6 +811,15 @@ def _prepare(filename, cleanoo=False, i=-1):
 def wr(filename, wrapnumber=45, punctuation_slop=20, cleanoo=False, i=-1):
     alltext, filetype, indent_amount, _ = _prepare(filename, cleanoo, i)
     if filetype == "markdown":
+        # reattach detached pandoc-crossref markers before wrapping
+        alltext = DETACHED_CROSSREF.sub(
+            lambda found: (
+                re.sub(r"\s+", " ", found["marker"])
+                if found["marker"]
+                else found[0][: len(found[0]) - len(found["gap"])]
+            ),
+            alltext,
+        )
         result = wrap_blocks(
             markdown_blocks(alltext),
             wrapnumber,
@@ -865,7 +891,7 @@ def check_prose(content, wrapnumber, punctuation_slop, line_start=1):
                 if fragment:
                     # {{{ check line lengths within this sentence fragment
                     words, positions = [], []
-                    for match in re.finditer(r"\S+", fragment):
+                    for match in WORD.finditer(fragment):
                         words.append(match.group())
                         positions.append(frag_offset + match.start())
                     if words:
@@ -978,26 +1004,71 @@ def _iter_block_spans(blocks, wrapnumber, line_start=1):
 
 
 def markdown_lint_issues_from_text(
-    content, wrapnumber=45, punctuation_slop=20
+    content, wrapnumber=45, punctuation_slop=20, max_line_length=None
 ):
-    """Return Markdown writing issues, including unfinished source spans."""
+    """Return Markdown writing issues, including unfinished source spans.
+
+    With `max_line_length`, a prose line is only too long when it holds
+    more than that many characters, instead of wr's greedy line choice.
+    """
     def mask_comment(match):
         return "".join("\n" if char == "\n" else " " for char in match[0])
 
     prose_content = re.sub(
         r"<!--.*?(?:-->|\Z)", mask_comment, content, flags=re.DOTALL
     )
-    issues = [
-        issue
-        for line, width, span in _iter_block_spans(
-            markdown_blocks(prose_content), wrapnumber
-        )
-        for issue in check_prose(span, width, punctuation_slop, line)
-    ]
+    # trailing spaces are reported separately below, and a double space
+    # (a hard line break) must not look like a sentence ending mid-line
+    prose_content = re.sub(r"[ \t]+$", "", prose_content, flags=re.M)
+    issues = []
+    prose_lines = prose_content.splitlines()
+    for line, width, span in _iter_block_spans(
+        markdown_blocks(prose_content), wrapnumber
+    ):
+        for issue in check_prose(span, width, punctuation_slop, line):
+            if max_line_length is None or not issue[1].startswith(
+                "line too long:"
+            ):
+                issues.append(issue)
+        if max_line_length is not None:
+            # {{{ flag prose lines longer than the character limit
+            for number in range(line, line + len(span.splitlines())):
+                text = prose_lines[number - 1]
+                # a single unbreakable word cannot be shortened
+                if len(text) > max_line_length and len(
+                    WORD.findall(text)
+                ) > 1:
+                    issues.append((
+                        number,
+                        f"line over {max_line_length} characters: break "
+                        "it at a word boundary",
+                    ))
+            # }}}
     issues.extend(
         (issue["line"], issue["message"])
         for issue in unclosed_markdown_spans(content)
     )
+    # {{{ report single trailing spaces and detached crossref markers
+    fence = None
+    for number, line in enumerate(content.splitlines(), start=1):
+        opening = re.match(r"[ \t]*(\x60{3,}|~{3,})", line)
+        if opening and (fence is None or opening[1].startswith(fence)):
+            fence = None if fence else opening[1]
+        elif fence is None and re.search(r"(?<! ) $", line):
+            issues.append((
+                number,
+                "trailing space: remove the single space at the end of this "
+                "line (two spaces are a deliberate hard line break)",
+            ))
+    for found in DETACHED_CROSSREF.finditer(prose_content):
+        if found["gap"] == "":
+            continue
+        issues.append((
+            prose_content.count("\n", 0, found.start()) + 1,
+            "detached crossref marker: keep the {#eq:...} or {#fig:...} "
+            "marker on one line, directly after the closing $$ or )",
+        ))
+    # }}}
     return sorted(issues)
 
 
@@ -1101,6 +1172,33 @@ def apply_markdown_issue_fix(content, line_number, message):
         raise ValueError(
             f"Markdown issue refers to missing line {line_number}"
         )
+    if message.startswith("detached crossref marker"):
+        # {{{ join the marker onto its "$$" or ")" (this can remove a line)
+        start = sum(len(line) for line in lines[:index])
+        found = next(
+            found
+            for found in DETACHED_CROSSREF.finditer(content, start)
+            if found["gap"] != ""
+        )
+        if found["gap"] is None:
+            span = found.span("marker")
+            joined = re.sub(r"\s+", " ", found["marker"])
+        else:
+            span = found.span("gap")
+            joined = ""
+        before = content[found.start() : found.end()]
+        updated = content[: span[0]] + joined + content[span[1] :]
+        after = updated[
+            found.start() : span[0] + len(joined) + found.end() - span[1]
+        ]
+        return (
+            updated,
+            before,
+            after,
+            "A crossref marker was separated from its equation or figure. "
+            "I joined it back on.",
+        )
+        # }}}
     old_line = lines[index]
     ending = ""
     if old_line.endswith("\r\n"):
@@ -1123,7 +1221,33 @@ def apply_markdown_issue_fix(content, line_number, message):
     else:
         prefix = re.match(r"^[ \t]*", old_line).group()
     # }}}
-    if message.startswith("line too long:"):
+    if message.startswith("trailing space:"):
+        lines[index] = old_line[:-1] + ending
+        return (
+            "".join(lines),
+            old_line,
+            old_line[:-1],
+            "A line ended with a single stray space. I removed it.",
+        )
+    if message.startswith("line over "):
+        # {{{ break at the last word boundary that fits the limit
+        limit = int(re.match(r"line over (\d+)", message)[1])
+        gaps = [
+            found.end()
+            for found in WORD.finditer(old_line)
+            if container.end() < found.end() < len(old_line.rstrip())
+        ]
+        fitting = [gap for gap in gaps if gap <= limit]
+        # when even the first word is too long, break right after it
+        split_at = fitting[-1] if fitting else gaps[0]
+        first = old_line[:split_at].rstrip()
+        second = old_line[split_at:].lstrip(" \t")
+        reason = (
+            f"This line was longer than {limit} characters. "
+            "I moved the last words onto a new line."
+        )
+        # }}}
+    elif message.startswith("line too long:"):
         match = re.search(r"after '(.+?)' \(before '(.+?)'\); move", message)
         if match is None:
             raise ValueError(
@@ -1164,26 +1288,19 @@ def apply_markdown_issue_fix(content, line_number, message):
     else:
         raise ValueError(f"No automatic fix is available for {message!r}")
 
-    if not first.strip():
+    if not first.strip() or not second.strip():
         # a break here would insert a blank line, i.e. a paragraph break
         raise ValueError(
             f"Automatic fix would leave an empty line at {line_number}"
         )
-    if second.strip():
-        lines[index] = first + ending + prefix + second + ending
-        after = first + "\n" + prefix + second
-    else:
-        # only trailing whitespace followed the break, so drop it rather
-        # than adding a blank line
-        lines[index] = first + ending
-        after = first
-        reason = (
-            "A sentence ended with a trailing space. I removed the space."
-        )
+    lines[index] = first + ending + prefix + second + ending
+    after = first + "\n" + prefix + second
     return "".join(lines), old_line, after, reason
 
 
-def autofix_markdown_file(filename, wrapnumber=55, punctuation_slop=20):
+def autofix_markdown_file(
+    filename, wrapnumber=55, punctuation_slop=20, max_line_length=None
+):
     """Apply safe source line breaks and report unfinished source spans."""
     with open(filename, encoding="utf-8", newline="") as fp:
         content = fp.read()
@@ -1192,7 +1309,10 @@ def autofix_markdown_file(filename, wrapnumber=55, punctuation_slop=20):
     # issue at a time re-lints the whole file for every single fix.
     for _ in range(100):
         issues = markdown_lint_issues_from_text(
-            content, wrapnumber=wrapnumber, punctuation_slop=punctuation_slop
+            content,
+            wrapnumber=wrapnumber,
+            punctuation_slop=punctuation_slop,
+            max_line_length=max_line_length,
         )
         fixable = {}
         for line_number, message in issues:

{pydifftools-0.1.39 → pydifftools-0.1.41}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pyDiffTools"
-version = "0.1.39"
+version = "0.1.41"
 authors = [{ name = "J M Franck" }]
 description = "Diff tools"
 readme = "README.rst"

{pydifftools-0.1.39 → pydifftools-0.1.41}/tests/test_wrap_check.py

@@ -1,3 +1,4 @@
+import re
 import subprocess
 
 import pytest
@@ -150,15 +151,92 @@ def test_long_line_fix_splits_at_the_word_after_the_break():
     assert fixed.startswith("for $30\\;\\text{s}$ ($3\\times$\nfor AOT)")
 
 
-def test_trailing_space_after_sentence_is_removed_not_a_blank_line(tmp_path):
+def test_single_trailing_space_is_flagged_and_removed(tmp_path):
+    # from RM_ESR: a sentence ending in a stray space
     path = tmp_path / "source.md"
     path.write_text("It melts near 238 K. \nIn contrast, it stays.\n")
 
+    issues = markdown_lint_issues_from_text(path.read_text(), wrapnumber=55)
     autofix_markdown_file(path)
 
+    assert (1, "trailing space") in [
+        (line, message.split(":")[0]) for line, message in issues
+    ]
     fixed = path.read_text()
     assert fixed.startswith("It melts near 238 K.\nIn contrast,")
-    assert "\n\n" not in fixed
+    assert " \n" not in fixed
+
+
+def test_double_trailing_space_hard_break_is_allowed():
+    source = "The first line ends here with a break  \nand continues.\n"
+
+    assert markdown_lint_issues_from_text(source, wrapnumber=55) == []
+
+
+@pytest.mark.parametrize(
+    "detached",
+    [
+        # equation markers must sit directly on the closing $$
+        "Intro.\n$$\nx = y\n$$ {#eq:x}\nwhere this holds.\n",
+        "Intro.\n$$\nx = y\n$$\n{#eq:x}\nwhere this holds.\n",
+        # figure markers must sit directly on the closing ) (from eigenmode)
+        "![A caption.](./media/BothProbes_Bfields.png)\n"
+        '{#fig:ProbeMagFieldsDrawing width="4in"}\n',
+        # and must not be split across lines (from RM_ESR)
+        "![A caption for\nspectroscopy.](Figures/simple_oned_CAT16.png)"
+        "{#fig:justSpectraCat16\nwidth=4in}\n",
+    ],
+)
+def test_detached_crossref_marker_is_flagged_and_joined(tmp_path, detached):
+    path = tmp_path / "source.md"
+    path.write_text(detached)
+
+    issues = markdown_lint_issues_from_text(detached, wrapnumber=55)
+    report = autofix_markdown_file(path)
+
+    fixed = path.read_text()
+    assert [message.split(":")[0] for _, message in issues] == [
+        "detached crossref marker"
+    ]
+    assert report["fixes"]
+    assert markdown_lint_issues_from_text(fixed, wrapnumber=55) == []
+    assert re.search(r"(\$\$|\))\{#(eq|fig):[^}\n]*\}", fixed)
+
+
+def test_figure_crossref_marker_is_never_split_from_its_figure(tmp_path):
+    # from RM_ESR: a caption spanning lines, ending in a marker with
+    # attributes; wr and the linter must treat "...){#fig:... }" as one word
+    source = (
+        "![Room temperature ESR of 100 mM TEMPO-SO~4~ RMs with water "
+        "loading $w_0=30$ in isooctane dispersant (blue), single HE "
+        "simulation (orange), multi HE simulation "
+        "(green)](Figures/HE_Easyspin_Simul_100mM.png)\n"
+        "{#fig:HESimul width=5.5in}\n"
+    )
+    path = tmp_path / "source.md"
+    path.write_text(source)
+
+    wr(str(path), 55)
+
+    wrapped = path.read_text()
+    assert (
+        "(green)](Figures/HE_Easyspin_Simul_100mM.png)"
+        "{#fig:HESimul width=5.5in}\n"
+    ) in wrapped
+    assert markdown_lint_issues_from_text(wrapped, wrapnumber=55) == []
+
+
+def test_wr_keeps_equation_marker_on_closing_dollars(tmp_path):
+    path = tmp_path / "source.md"
+    path.write_text(
+        "Intro.\n$$\nx = y\n$$ {#eq:tauFromLinewidth}\nwhere this holds.\n"
+    )
+
+    wr(str(path), 55)
+
+    wrapped = path.read_text()
+    assert "\n$${#eq:tauFromLinewidth}\n" in wrapped
+    assert markdown_lint_issues_from_text(wrapped, wrapnumber=55) == []
 
 
 def test_autofix_fixes_every_line_in_one_pass_from_the_end(tmp_path):
@@ -175,6 +253,51 @@ def test_autofix_fixes_every_line_in_one_pass_from_the_end(tmp_path):
     assert markdown_lint_issues_from_text(fixed, wrapnumber=55) == []
 
 
+def test_character_limit_ignores_lines_wr_would_break_sooner():
+    # from RM_ESR: 74 characters, which wr's width-55 rule would break
+    source = (
+        "temperatures not amenable to other methods, as in "
+        "[@Banerjee2009ESREviCoe].\n"
+    )
+
+    assert markdown_lint_issues_from_text(source, 55) != []
+    assert markdown_lint_issues_from_text(source, max_line_length=79) == []
+
+
+@pytest.mark.parametrize("prefix", ["", "- ", "> "])
+def test_character_limit_breaks_at_last_fitting_word(tmp_path, prefix):
+    # from RM_ESR: a sentence that grew past 79 characters
+    line = (
+        prefix + "This script determines the field positions and amplitudes "
+        "of the spectral lines for an entire variable-temperature "
+        "experiment.\n"
+    )
+    path = tmp_path / "source.md"
+    path.write_text(line)
+
+    issues = markdown_lint_issues_from_text(line, max_line_length=79)
+    report = autofix_markdown_file(path, max_line_length=79)
+
+    fixed = path.read_text()
+    assert [message.split(":")[0] for _, message in issues] == [
+        "line over 79 characters"
+    ]
+    assert "79 characters" in report["fixes"][0]["reason"]
+    assert all(len(text) <= 79 for text in fixed.splitlines())
+    assert fixed.replace("\n> ", "\n").split() == line.split()
+    # the first line is filled as far as the limit allows
+    first = fixed.splitlines()[0]
+    next_word = line.split()[len(first.split())]
+    assert len(first) + 1 + len(next_word) > 79
+    assert markdown_lint_issues_from_text(fixed, max_line_length=79) == []
+
+
+def test_character_limit_skips_a_single_unbreakable_word():
+    source = "https://example.org/" + "x" * 90 + "\n"
+
+    assert markdown_lint_issues_from_text(source, max_line_length=79) == []
+
+
 def test_unclosed_math_is_reported_after_fifty_lines():
     short_math = "$$\n" + ("x = y\n" * 48)
     long_math = "$$\n" + ("x = y\n" * 49)