prose-preflight 0.1.0__py3-none-any.whl

prose_preflight/checks/__init__.py

@@ -0,0 +1,25 @@
+"""Checker registry. Adding a checker is adding a module and one line here."""
+
+from prose_preflight.checks import (
+    acronym,
+    claim,
+    readability,
+    sentence_length,
+    structure,
+    terminology,
+    units,
+    vale,
+)
+
+CHECKERS = {
+    "acronym": acronym.check,
+    "claim": claim.check,
+    "readability": readability.check,
+    "sentence_length": sentence_length.check,
+    "structure": structure.check,
+    "terminology": terminology.check,
+    "units": units.check,
+    "vale": vale.check,
+}
+
+__all__ = ["CHECKERS"]

prose_preflight/checks/_text.py

@@ -0,0 +1,60 @@
+"""Shared paragraph and sentence helpers. Not a checker, not registered."""
+
+import re
+
+from prose_preflight.extract import Document
+
+# ponytail: regex sentence split -- misses "e.g. Foo" and "Fig. 3 shows". No spaCy for
+# this; upgrade the lookbehind when a fixture shows a real miss.
+SENTENCE_END = re.compile(r"(?<=[.!?])[\"')\]]*\s+(?=[\"'(\[]*[A-Z0-9])")
+WORD = re.compile(r"\S+")
+
+
+def alternation(words, ignore_case: bool = True):
+    """Compile `words` into one word-bounded alternation. Longest first, so a variant
+    that starts another one still matches in full."""
+    return re.compile(
+        r"\b(?:{})\b".format(
+            "|".join(re.escape(w) for w in sorted(words, key=len, reverse=True))
+        ),
+        re.IGNORECASE if ignore_case else 0,
+    )
+
+
+def paragraphs(doc: Document):
+    """Yield (text, positions) per paragraph: masked text joined with single spaces,
+    plus a (line, col) per character, so a match offset maps back to the source."""
+    for first, last in doc.paragraphs:
+        chunks, positions = [], []
+        for n in range(first, last + 1):
+            line = doc.masked[n - 1]
+            if positions:
+                chunks.append(" ")
+                positions.append(positions[-1])
+            chunks.append(line)
+            positions += [(n, col) for col in range(1, len(line) + 1)]
+        yield "".join(chunks), positions
+
+
+def sentences(text: str):
+    """Yield (offset, sentence) over `text`."""
+    start = 0
+    for split in SENTENCE_END.finditer(text):
+        yield start, text[start : split.start()]
+        start = split.end()
+    if text[start:].strip():
+        yield start, text[start:]
+
+
+def truncate(text: str, limit: int) -> str:
+    """First `limit` characters, with an ellipsis when something was cut."""
+    return text[:limit].strip() + ("\u2026" if len(text) > limit else "")
+
+
+def excerpt(line: str, start: int, end: int, pad: int = 30) -> str:
+    left, right = max(0, start - pad), min(len(line), end + pad)
+    return (
+        ("…" if left else "")
+        + line[left:right].strip()
+        + ("…" if right < len(line) else "")
+    )

prose_preflight/checks/acronym.py

@@ -0,0 +1,46 @@
+"""First-use expansion tracking. An acronym is defined by a parenthesized pairing."""
+
+import re
+
+from prose_preflight.checks._text import excerpt
+from prose_preflight.extract import Document
+from prose_preflight.finding import Finding
+
+ACRONYM = re.compile(r"\b([A-Z]{2,})s?\b")
+# "Deep Neural Network (DNN)" -- the words before the parenthesis define it.
+DEFINED_AFTER = re.compile(r"(?:\b[\w-]+\s+){1,6}\(([A-Z]{2,})s?\)")
+# "DNN (deep neural network)" -- the parenthesis expands it.
+DEFINED_BEFORE = re.compile(r"\b([A-Z]{2,})s?\s*\((?=[^)]*[a-z])[^)]{4,}\)")
+
+
+def check(doc: Document, rule: dict) -> list[Finding]:
+    allow = {form.upper() for form in rule.get("allowlist", [])}
+    minimum = rule.get("min_uses", 1)
+    severity = rule.get("severity", "warning")
+    uses, defined = {}, set()
+    for n, line in doc.prose():
+        defined |= {m.group(1) for m in DEFINED_AFTER.finditer(line)}
+        defined |= {m.group(1) for m in DEFINED_BEFORE.finditer(line)}
+        for m in ACRONYM.finditer(line):
+            form = m.group(1)
+            if form in allow or form in defined:
+                continue
+            uses.setdefault(form, []).append((n, m))
+    findings = []
+    for form, hits in uses.items():
+        if len(hits) < minimum:
+            continue
+        n, m = hits[0]
+        findings.append(
+            Finding(
+                check="acronym.undefined",
+                category="acronym",
+                severity=severity,
+                line=n,
+                col=m.start() + 1,
+                excerpt=excerpt(doc.lines[n - 1], m.start(), m.end()),
+                message=f"{form!r} is used {len(hits)}× but never expanded; spell it out on first use.",
+                section=doc.section_at(n),
+            )
+        )
+    return findings

prose_preflight/checks/claim.py

@@ -0,0 +1,34 @@
+"""Hedges and boosters. Emits `review` only -- never auto-applied, by design."""
+
+from prose_preflight.checks._text import alternation, excerpt
+from prose_preflight.extract import Document
+from prose_preflight.finding import Finding
+
+KINDS = {
+    "hedge": "Hedge -- check the claim is not weaker than the evidence.",
+    "booster": "Booster -- check the evidence supports a claim this strong.",
+}
+
+
+def check(doc: Document, rule: dict) -> list[Finding]:
+    findings = []
+    for kind, message in KINDS.items():
+        words = rule.get(f"{kind}s", [])
+        if not words:
+            continue
+        pattern = alternation(words)
+        for n, line in doc.prose():
+            for m in pattern.finditer(line):
+                findings.append(
+                    Finding(
+                        check=f"claim.{kind}",
+                        category="claim",
+                        severity="review",  # load-bearing: never auto-applied
+                        line=n,
+                        col=m.start() + 1,
+                        excerpt=excerpt(doc.lines[n - 1], m.start(), m.end()),
+                        message=f"{message} ({m.group()!r})",
+                        section=doc.section_at(n),
+                    )
+                )
+    return findings

prose_preflight/checks/readability.py

@@ -0,0 +1,60 @@
+"""Per-section readability scores from textstat, against YAML thresholds."""
+
+from bisect import bisect_right
+
+import textstat
+
+from prose_preflight.extract import Document
+from prose_preflight.finding import Finding
+
+METRICS = {  # rule suffix: (textstat function, comparison, threshold key)
+    "flesch": (textstat.flesch_reading_ease, "min", "min_flesch"),
+    "fog": (textstat.gunning_fog, "max", "max_fog"),
+}
+
+
+def check(doc: Document, rule: dict) -> list[Finding]:
+    severity = rule.get("severity", "warning")
+    min_words = rule.get("min_words", 50)
+    findings = []
+    for line, title, text, words in _sections(doc):
+        if words < min_words:  # short sections score as noise
+            continue
+        for name, (score_of, direction, key) in METRICS.items():
+            limit = rule.get(key)
+            score = round(score_of(text), 1)
+            if limit is None or (
+                score >= limit if direction == "min" else score <= limit
+            ):
+                continue
+            findings.append(
+                Finding(
+                    check=f"readability.{name}",
+                    category="readability",
+                    severity=severity,
+                    line=line,
+                    col=1,
+                    excerpt=title or str(doc.path.name),
+                    message=(
+                        f"{name} score {score} is "
+                        f"{'below' if direction == 'min' else 'above'} the {limit} "
+                        f"threshold ({words} words); shorten sentences or simplify wording."
+                    ),
+                    section=title,
+                )
+            )
+    return findings
+
+
+def _sections(doc: Document):
+    """Yield (line, title, text, word count) per section; the whole file if unheaded."""
+    bounds = [(line, title) for line, _, title in doc.headings] or [(1, None)]
+    starts = [line for line, _ in bounds]
+    chunks = [[] for _ in bounds]
+    for first, last in doc.paragraphs:
+        owner = bisect_right(starts, first) - 1
+        if owner >= 0:  # paragraphs above the first heading belong to no section
+            chunks[owner] += [doc.masked[n - 1].strip() for n in range(first, last + 1)]
+    for (start, title), parts in zip(bounds, chunks):
+        text = " ".join(parts)
+        yield start, title, text, len(text.split())

prose_preflight/checks/sentence_length.py

@@ -0,0 +1,30 @@
+"""Sentence-length outliers. Regex segmentation, word count against a YAML maximum."""
+
+from prose_preflight.checks._text import WORD, paragraphs, sentences, truncate
+from prose_preflight.extract import Document
+from prose_preflight.finding import Finding
+
+
+def check(doc: Document, rule: dict) -> list[Finding]:
+    maximum = rule.get("max_words", 40)
+    severity = rule.get("severity", "warning")
+    findings = []
+    for text, positions in paragraphs(doc):
+        for offset, sentence in sentences(text):
+            words = len(WORD.findall(sentence))
+            if words <= maximum:
+                continue
+            line, col = positions[offset]
+            findings.append(
+                Finding(
+                    check="sentence_length.long",
+                    category="readability",
+                    severity=severity,
+                    line=line,
+                    col=col,
+                    excerpt=truncate(sentence, 40),
+                    message=f"Sentence runs {words} words (limit {maximum}); split it.",
+                    section=doc.section_at(line),
+                )
+            )
+    return findings

prose_preflight/checks/structure.py

@@ -0,0 +1,46 @@
+"""Required sections and their order, from the YAML list. Titles match case-insensitively."""
+
+from itertools import pairwise
+
+from prose_preflight.extract import Document
+from prose_preflight.finding import Finding
+
+
+def check(doc: Document, rule: dict) -> list[Finding]:
+    required = rule.get("required_sections", [])
+    if not required:
+        return []
+    found = {}  # required title (as configured) -> source line of its first match
+    for line, _, title in doc.headings:
+        for want in required:
+            if title.strip().lower() == want.lower() and want not in found:
+                found[want] = line
+    findings = [
+        Finding(
+            check="structure.missing_section",
+            category="structure",
+            severity=rule.get("severity", "error"),
+            line=1,
+            col=1,
+            excerpt=doc.path.name,
+            message=f"Required section {want!r} is missing.",
+        )
+        for want in required
+        if want not in found
+    ]
+    present = [want for want in required if want in found]
+    for earlier, later in pairwise(present):
+        if found[later] < found[earlier]:
+            findings.append(
+                Finding(
+                    check="structure.section_order",
+                    category="structure",
+                    severity=rule.get("order_severity", "warning"),
+                    line=found[later],
+                    col=1,
+                    excerpt=later,
+                    message=f"Section {later!r} appears before {earlier!r}; expected order is {' → '.join(required)}.",
+                    section=later,
+                )
+            )
+    return findings

prose_preflight/checks/terminology.py

@@ -0,0 +1,38 @@
+"""Preferred-form enforcement. The table is data; this module is the matcher."""
+
+from prose_preflight.checks._text import alternation, excerpt
+from prose_preflight.extract import Document
+from prose_preflight.finding import Finding
+
+
+def check(doc: Document, rule: dict) -> list[Finding]:
+    severity = rule.get("severity", "warning")
+    findings = []
+    for preferred, variants in rule.get("terms", {}).items():
+        pattern = alternation(variants, rule.get("ignore_case", True))
+        for n, line in doc.prose():
+            for m in pattern.finditer(line):
+                if m.group() == preferred:
+                    continue
+                findings.append(
+                    Finding(
+                        check="terminology.variant",
+                        category="terminology",
+                        severity=severity,
+                        line=n,
+                        col=m.start() + 1,
+                        excerpt=excerpt(doc.lines[n - 1], m.start(), m.end()),
+                        message=f"Use {preferred!r}, not {m.group()!r}.",
+                        section=doc.section_at(n),
+                        suggestion=_match_case(m.group(), preferred),
+                    )
+                )
+    return findings
+
+
+def _match_case(found: str, preferred: str) -> str:
+    """Carry the variant's case over to the suggestion. A single capital letter is
+    sentence case, not an acronym, so only multi-character ALL CAPS becomes ALL CAPS."""
+    if len(found) > 1 and found.isupper():
+        return preferred.upper()
+    return preferred[:1].upper() + preferred[1:] if found[:1].isupper() else preferred

prose_preflight/checks/units.py

@@ -0,0 +1,66 @@
+"""Number and unit *formatting* only -- spacing and dashes. Regex, not Pint."""
+
+import re
+
+from prose_preflight.checks._text import excerpt
+from prose_preflight.extract import Document
+from prose_preflight.finding import Finding
+
+NBSP = " "
+RULES = {  # rule suffix: (pattern template, replacement, message)
+    "space_before_unit": (
+        r"(?<![\w.])(\d+(?:[.,]\d+)?)[ ]({units})\b",
+        lambda m: f"{m.group(1)}{NBSP}{m.group(2)}",
+        "Use a non-breaking space between value and unit.",
+    ),
+    "percent_spacing": (
+        r"(?<![\w.])(\d+(?:[.,]\d+)?)\s+%",
+        lambda m: f"{m.group(1)}%",
+        "Write the percent sign attached to the number.",
+    ),
+    "range_dash": (
+        r"(?<![\w.-])(\d+(?:[.,]\d+)?)-(\d+(?:[.,]\d+)?)(?![\w-])",
+        lambda m: f"{m.group(1)}–{m.group(2)}",
+        "Use an en dash for numeric ranges.",
+    ),
+    "p_value_spacing": (
+        r"\bp\s*([<>=≤≥])\s*(\.\d+|0\.\d+)",
+        lambda m: (
+            f"p {m.group(1)} {m.group(2) if m.group(2)[0] == '0' else '0' + m.group(2)}"
+        ),
+        "Write p-values as 'p < 0.05', with spaces and a leading zero.",
+    ),
+}
+
+
+def check(doc: Document, rule: dict) -> list[Finding]:
+    severity = rule.get("severity", "warning")
+    units = "|".join(
+        re.escape(u) for u in sorted(rule.get("units", []), key=len, reverse=True)
+    )
+    findings = []
+    for name, (template, replace, message) in RULES.items():
+        if name in rule.get("disabled_rules", []) or (
+            "{units}" in template and not units
+        ):
+            continue
+        pattern = re.compile(template.format(units=units))
+        for n, line in doc.prose():
+            for m in pattern.finditer(line):
+                suggestion = replace(m)
+                if suggestion == m.group():
+                    continue
+                findings.append(
+                    Finding(
+                        check=f"units.{name}",
+                        category="units",
+                        severity=severity,
+                        line=n,
+                        col=m.start() + 1,
+                        excerpt=excerpt(doc.lines[n - 1], m.start(), m.end()),
+                        message=message,
+                        section=doc.section_at(n),
+                        suggestion=suggestion,
+                    )
+                )
+    return findings

prose_preflight/checks/vale.py

@@ -0,0 +1,92 @@
+"""Vale, behind the checker interface. An external binary that must never abort the run."""
+
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+
+from prose_preflight.extract import Document
+from prose_preflight.finding import Finding
+
+# The `vale` dependency ships the binary, so reaching this message means the
+# environment blocked it: no network on first run, or a binary quarantined by the OS.
+MISSING = (
+    "The Vale binary could not be found, so grammar and style were not checked. "
+    "It is normally installed with this package; a blocked first-run download is the "
+    "usual cause. Install it manually from https://vale.sh/docs/install and it will be "
+    "picked up from PATH."
+)
+
+
+def check(doc: Document, rule: dict) -> list[Finding]:
+    try:
+        alerts = _run(doc, rule)
+    except (OSError, subprocess.SubprocessError, json.JSONDecodeError) as exc:
+        return [
+            Finding(
+                check="vale.unavailable",
+                category="style",
+                severity="warning",
+                line=1,
+                col=1,
+                excerpt="",
+                message=MISSING
+                if isinstance(exc, FileNotFoundError)
+                else f"Vale failed: {exc}",
+                section=None,
+                suggestion=None,
+            )
+        ]
+    return [
+        Finding(
+            # ponytail: every Vale rule lands in `style`. Split into grammar/spelling
+            # by rule prefix once a config needs to weight them differently.
+            check=f"vale.{alert['Check']}",
+            category="style",
+            severity="error" if alert["Severity"].lower() == "error" else "warning",
+            line=alert["Line"],
+            col=alert["Span"][0],
+            excerpt=alert["Match"],
+            message=alert["Message"],
+            section=doc.section_at(alert["Line"]),
+            suggestion=next(iter(alert.get("Action", {}).get("Params") or []), None),
+        )
+        for alert in alerts
+    ]
+
+
+def _run(doc: Document, rule: dict) -> list[dict]:
+    """Invoke Vale on the document. Vale exits nonzero when it finds alerts, so only
+    the JSON on stdout decides success."""
+    with tempfile.TemporaryDirectory() as tmp:
+        ini = rule.get("config_file") or _write_ini(Path(tmp), rule)
+        path = _readable(doc, Path(tmp))
+        result = subprocess.run(
+            ["vale", "--output=JSON", f"--config={ini}", str(path)],
+            capture_output=True,
+            check=False,  # nonzero just means "alerts found"
+            text=True,
+            timeout=rule.get("timeout", 60),
+        )
+    if not result.stdout.strip():
+        raise subprocess.SubprocessError(result.stderr.strip() or "no output")
+    report = json.loads(result.stdout)
+    return [alert for alerts in report.values() for alert in alerts]
+
+
+def _readable(doc: Document, tmp: Path) -> Path:
+    """Vale parses Markdown and plain text; it has no LaTeX reader, and pointed at a
+    .tex file it would flag the preamble and every command. `doc.masked` is already
+    the prose alone, line for line, so any other format goes to Vale as masked text."""
+    if doc.path.suffix.lower() in {".md", ".markdown", ".txt"}:
+        return doc.path
+    masked = tmp / f"{doc.path.stem}.txt"
+    masked.write_text("\n".join(doc.masked))
+    return masked
+
+
+def _write_ini(tmp: Path, rule: dict) -> Path:
+    """Vale's own config is an INI string in `default.yaml`, so it stays data."""
+    ini = tmp / ".vale.ini"
+    ini.write_text(rule.get("ini", ""))
+    return ini

prose_preflight/default.yaml

@@ -0,0 +1,161 @@
+# Bundled config. A user file passed with --config is deep-merged over this one.
+# Rule keys land here alongside their checkers; a new document convention is a
+# YAML edit, never a code change.
+checks:
+  terminology:
+    enabled: true
+    severity: warning
+    ignore_case: true
+    # Preferred form: variants that should become it. Empty by default -- the right
+    # spellings are your journal's, not ours. Uncomment and edit:
+    terms: {}
+      # dataset: [data set, data-set]
+      # machine learning: [machine-learning]
+      # ground truth: [ground-truth, groundtruth]
+  readability:
+    enabled: true
+    severity: warning
+    min_words: 50 # sections shorter than this score as noise
+    min_flesch: 30.0 # below: hard to read
+    max_fog: 18.0 # above: too many long words per sentence
+  sentence_length:
+    enabled: true
+    severity: warning
+    max_words: 40
+  acronym:
+    enabled: true
+    severity: warning
+    min_uses: 1 # raise to ignore one-off mentions
+    # never expanded: too common, or not really acronyms
+    allowlist:
+      [
+        API,
+        CPU,
+        GPU,
+        RAM,
+        PDF,
+        HTML,
+        JSON,
+        YAML,
+        XML,
+        CSV,
+        URL,
+        USA,
+        EU,
+        UK,
+        ID,
+        OK,
+        TODO,
+        FAQ,
+        PhD,
+        DOI,
+        ISBN,
+        GDP,
+        AM,
+        PM,
+        UTC,
+        ISO,
+        IEEE,
+        ACM,
+      ]
+  structure:
+    enabled: true
+    severity: error # a missing section
+    order_severity: warning # a section out of order
+    # Empty by default: most documents are fragments, not whole papers. Set the list to
+    # opt in, e.g. [Abstract, Introduction, Methods, Results, Discussion, References].
+    required_sections: []
+  units:
+    enabled: true
+    severity: warning
+    disabled_rules: [] # any of: space_before_unit, percent_spacing, range_dash, p_value_spacing
+    units:
+      [
+        mm,
+        cm,
+        m,
+        km,
+        mg,
+        g,
+        kg,
+        ms,
+        s,
+        min,
+        h,
+        Hz,
+        kHz,
+        MHz,
+        GHz,
+        kB,
+        MB,
+        GB,
+        TB,
+        W,
+        kW,
+        V,
+        A,
+        K,
+        °C,
+        °F,
+        px,
+        dpi,
+        bp,
+        mol,
+        N,
+        Pa,
+        J,
+        kJ,
+      ]
+  claim:
+    enabled: true # severity is always `review`; never auto-applied
+    hedges:
+      [
+        may,
+        might,
+        could,
+        possibly,
+        perhaps,
+        arguably,
+        seems to,
+        appears to,
+        somewhat,
+        relatively,
+        fairly,
+        rather,
+        quite,
+        suggests that,
+        tends to,
+      ]
+    boosters:
+      [
+        clearly,
+        obviously,
+        undoubtedly,
+        certainly,
+        definitely,
+        significantly,
+        dramatically,
+        drastically,
+        vastly,
+        extremely,
+        profoundly,
+        unprecedented,
+        groundbreaking,
+        revolutionary,
+        novel,
+        optimal,
+        perfect,
+        proves,
+        proven,
+      ]
+  vale:
+    enabled: true
+    timeout: 60
+    # Vale's own config, kept as data. Built-in `Vale.*` rules only: no `vale sync`,
+    # no downloaded styles, so runs stay offline and deterministic. Point
+    # `config_file` at your own .vale.ini to use styles you have synced.
+    config_file: null
+    ini: |
+      MinAlertLevel = suggestion
+      [*.{md,markdown,txt}]
+      BasedOnStyles = Vale

prose_preflight/extract/__init__.py

@@ -0,0 +1,17 @@
+"""Extraction: pick a reader by suffix, return one normalized `Document`."""
+
+from pathlib import Path
+
+from prose_preflight.extract import document, latex, markdown
+from prose_preflight.extract.document import Document
+
+__all__ = ["Document", "read"]
+
+READERS = {".tex": latex.read, ".md": markdown.read, ".markdown": markdown.read}
+
+
+def read(path: str | Path) -> Document:
+    """Anything without a known suffix is read as plain text."""
+    path = Path(path)
+    reader = READERS.get(path.suffix.lower(), document.read)
+    return reader(path, path.read_text(encoding="utf-8"))