stylefix 0.7.0__py3-none-any.whl

stylefix/__init__.py

@@ -0,0 +1,3 @@
+"""stylefix: regularize Markdown to US or UK English and a style guide."""
+
+__version__ = "0.7.0"

stylefix/abbreviations.py

@@ -0,0 +1,200 @@
+"""Abbreviation forms the style guides rule on as a class.
+
+Two general rules, from the guides themselves:
+
+- Chicago 10.4 (and Oxford, Guardian): an abbreviation in full capitals
+  takes no periods, whether two letters or more (US, UK, EU, NATO, PhD,
+  MA, DC). Applied generally: every period-group of capitals is closed up.
+  The one hazard is personal initials ("J.K. Rowling"), which look the
+  same; a group followed by a capitalised word is therefore adjudicated,
+  as is the sentence-end case ("in the U.S. The ...").
+- AP "abbreviations and acronyms" (and GPO): periods in most two-letter
+  abbreviations (U.S., U.N., U.K., B.A., B.C.), with named exceptions (AP,
+  GI, ID, EU); none in longer ones (FBI, NATO, USA). AP applies "most" to
+  countries, places, degrees and eras, not to AI, IT, PR or TV, so the keep
+  rule is applied by category. APA keeps U.S. but writes degrees without
+  periods (PhD).
+
+Also: clock times (10 a.m. / 10am), vs., and the Latin tags e.g., i.e.,
+etc. And the noun rule of GPO and APA: "U.S." only as an adjective; the
+noun is "the United States". That one is a warning by default and an
+enforcement on request (--abbreviation-nouns enforce), decided per
+occurrence because it is a part-of-speech question."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from adjudicate import Edit, Item, prose_regions, sentence_context
+
+# Period-groups of capitals, allowing a lowercase letter after a capital
+# (Ph.D., B.Sc., D.Phil.), two or more parts.
+_GROUP = re.compile(r"(?<![A-Za-z.])(?:[A-Z][a-z]?\.\s?){2,}(?<=\.)(?![A-Za-z])")
+# Categories AP and GPO give periods to.
+COUNTRIES = {"US": "U.S.", "UK": "U.K.", "UN": "U.N.", "DC": "D.C."}
+DEGREES = {"BA": "B.A.", "MA": "M.A.", "PhD": "Ph.D.", "MD": "M.D.", "JD": "J.D.", "BS": "B.S.",
+           "MS": "M.S.", "BSc": "B.Sc.", "MSc": "M.Sc.", "LLB": "LL.B.", "LLM": "LL.M.",
+           "DPhil": "D.Phil."}
+ERAS = {"BC": "B.C.", "AD": "A.D."}
+KEEP_EXCEPTIONS = {"EU", "AP", "GI", "ID", "USA", "OK"}          # AP's named exceptions
+KNOWN = set(COUNTRIES) | set(DEGREES) | set(ERAS) | KEEP_EXCEPTIONS | {
+    "NY", "LA", "UAE", "USSR", "NATO", "FBI", "CIA", "NASA", "CEO", "VP", "CE", "BCE", "NYC"}
+
+_TIME = re.compile(r"(?<=\d)\s?([aApP])\.?[mM]\.?(?![A-Za-z])")
+_VS = re.compile(r"\bvs\.?(?=[\s,;:)])")
+_EG = re.compile(r"\b[eE]\.?[gG]\.(?=[\s,;:)])|\beg(?=[\s,])")
+_IE = re.compile(r"\b[iI]\.?[eE]\.(?=[\s,;:)])|\bie(?=[\s,])")
+_ETC = re.compile(r"\betc\.?(?![A-Za-z])")
+_NOUN_US = re.compile(r"\b[Tt]he (U\.S\.|US)(?![A-Za-z])")
+# "the US" followed by something that cannot be a modified noun: a verb,
+# a preposition, a conjunction, or punctuation. Used for the warning count.
+_NOUN_TAIL = re.compile(r"\s+(?:and|or|is|has|have|had|will|would|should|could|can|must|may|might|"
+                        r"to|in|on|at|by|for|with|from|as|than|that|which|who|does|did|do|was|were|"
+                        r"are|be|also|still|now|then|already|itself|alone|too)\b|[.,;:!?)]")
+
+
+@dataclass(frozen=True)
+class Abbreviations:
+    caps: str | None               # "drop" (Chicago rule) | "keep" (AP/GPO categories) | None
+    degrees_periods: bool | None   # under keep: Ph.D. (True) or PhD (False)
+    time: str | None               # "a.m." | "am"
+    vs: str | None                 # "vs." | "vs" | "v"
+    latin_periods: bool | None
+    noun_rule: bool = False        # guide says "U.S." is an adjective only
+
+
+PROFILES = {
+    "chicago": Abbreviations("drop", None, "a.m.", "vs.", True),
+    "gpo": Abbreviations("keep", True, "a.m.", "vs.", True, noun_rule=True),
+    "ap": Abbreviations("keep", True, "a.m.", "vs.", True),
+    "apa": Abbreviations("keep", False, "a.m.", "vs.", True, noun_rule=True),
+    "oxford": Abbreviations("drop", None, "a.m.", "vs", True),
+    "guardian": Abbreviations("drop", None, "am", "v", False),
+    "none": Abbreviations(None, None, None, None, None),
+}
+
+DROP_RULE = ("The style guide writes abbreviations in capitals without periods. This one is followed "
+             "by a capitalised word, so it is one of: the last word of its sentence, part of a name "
+             "or title ('U.S. Treasury'), or a person's initials ('J.K. Rowling'). The readings:\n"
+             "  A (sentence ends here): {a}\n  B (name or title continues): {b}\n"
+             "Choose '{end}' for A, '{bare}' for B. KEEP if these are a person's initials, which keep "
+             "their periods.")
+NOUN_RULE = ("{guide} uses '{abbr}' only as an adjective ('{abbr} policy', '{abbr} exports') and writes "
+             "the noun out: 'the United States'. Here it follows 'the'. Choose 'United States' if the "
+             "abbreviation stands alone as a noun (subject or object: 'the US has', 'in the US', "
+             "'the US and its allies'). Choose '{abbr}' if it modifies a following noun ('the US "
+             "economy'). KEEP only if neither.")
+
+
+def _line_col(text: str, pos: int) -> tuple[int, int]:
+    return text.count("\n", 0, pos) + 1, pos - (text.rfind("\n", 0, pos) + 1) + 1
+
+
+def _edit(text: str, start: int, end: int, replacement: str) -> Edit:
+    line, col = _line_col(text, start)
+    return Edit(line, col, col + (end - start) - 1, text[start:end], replacement, "abbreviations")
+
+
+def _item(text: str, s: int, e: int, options: tuple[str, ...], rule: str, next_id: int) -> Item:
+    line, col = _line_col(text, s)
+    return Item(next_id, text[s:e], options, rule, sentence_context(text, s, e),
+                line, col, col + (e - s) - 1, source="abbreviation")
+
+
+def bare(group: str) -> str:
+    return group.replace(".", "").replace(" ", "")
+
+
+def plan(text: str, guide: str, nouns: str = "default", next_id: int = 1
+         ) -> tuple[list[Edit], list[Item], list[tuple[int, str]]]:
+    """Deterministic edits, adjudication items, and noun-rule warnings
+    (line, phrase). `nouns`: default (warn if the guide has the rule),
+    warn, enforce, off."""
+    prof = PROFILES.get(guide, PROFILES["none"])
+    noun_mode = nouns if nouns != "default" else ("warn" if prof.noun_rule else "off")
+    edits: list[Edit] = []
+    items: list[Item] = []
+    warnings: list[tuple[int, str]] = []
+    claimed: set[int] = set()          # start offsets already handled by a noun item
+
+    for rs, re_ in prose_regions(text):
+        region = text[rs:re_]
+
+        # --- the noun rule: "the U.S." / "the US"
+        if prof.noun_rule and noun_mode != "off":
+            for m in _NOUN_US.finditer(region):
+                s, e = rs + m.start(1), rs + m.end(1)
+                tail = region[m.end():m.end() + 12]
+                if noun_mode == "enforce":
+                    canonical = COUNTRIES["US"] if prof.caps == "keep" else "US"
+                    rule = NOUN_RULE.format(guide={"gpo": "GPO", "apa": "APA"}.get(guide, "The guide"),
+                                            abbr=canonical)
+                    items.append(_item(text, s, e, ("United States", canonical, "KEEP"), rule, next_id))
+                    next_id += 1
+                    claimed.add(s)
+                elif _NOUN_TAIL.match(tail):
+                    warnings.append((_line_col(text, s)[0], text[s - 4:e + min(12, len(tail))].strip()))
+
+        # --- capitals: drop (general) or keep (by category)
+        if prof.caps == "drop":
+            for m in _GROUP.finditer(region):
+                s, e = rs + m.start(), rs + m.end()
+                if s in claimed:
+                    continue
+                group = m.group(0)
+                canonical = bare(group)
+                following = region[m.end():m.end() + 3]
+                if re.match(r"\s*$", following) or following.startswith("\n"):
+                    edits.append(_edit(text, s, e, canonical + "."))       # paragraph end
+                elif re.match(r"\s+[A-Z“\"(]", following):
+                    lo, hi = max(rs, s - 45), min(re_, e + 45)
+                    before, after = text[lo:s], text[e:hi]
+                    rule = DROP_RULE.format(a="…" + before + canonical + "." + after + "…",
+                                            b="…" + before + canonical + after + "…",
+                                            end=canonical + ".", bare=canonical)
+                    items.append(_item(text, s, e, (canonical + ".", canonical, "KEEP"), rule, next_id))
+                    next_id += 1
+                else:
+                    edits.append(_edit(text, s, e, canonical))
+        elif prof.caps == "keep":
+            table = dict(COUNTRIES) | dict(ERAS)
+            if prof.degrees_periods:
+                table |= DEGREES
+            else:
+                # APA: degrees without periods; U.S. with.
+                for k, v in DEGREES.items():
+                    table[v] = k
+            alts = "|".join(re.escape(k) for k in sorted(table, key=len, reverse=True))
+            for m in re.finditer(rf"(?<![A-Za-z.])(?:{alts})(?![A-Za-z])", region):
+                s, e = rs + m.start(), rs + m.end()
+                if s in claimed:
+                    continue
+                want = table[m.group(0)]
+                if want.endswith(".") and text[e:e + 1] == "." and text[e:e + 2] != "..":
+                    e += 1                                               # absorb the sentence period
+                edits.append(_edit(text, s, e, want))
+
+        # --- times
+        if prof.time:
+            for m in _TIME.finditer(region):
+                letter = m.group(1).lower()
+                want = (" " + letter + ".m.") if prof.time == "a.m." else (letter + "m")
+                if m.group(0) != want:
+                    edits.append(_edit(text, rs + m.start(), rs + m.end(), want))
+        # --- vs
+        if prof.vs:
+            for m in _VS.finditer(region):
+                if m.group(0) != prof.vs:
+                    edits.append(_edit(text, rs + m.start(), rs + m.end(), prof.vs))
+        # --- e.g., i.e., etc.
+        if prof.latin_periods is not None:
+            for rx, with_, without in ((_EG, "e.g.", "eg"), (_IE, "i.e.", "ie"), (_ETC, "etc.", "etc")):
+                want = with_ if prof.latin_periods else without
+                for m in rx.finditer(region):
+                    if m.group(0) != want:
+                        e = rs + m.end()
+                        if want.endswith(".") and text[e:e + 1] == "." and text[e:e + 2] != "..":
+                            e += 1
+                        edits.append(_edit(text, rs + m.start(), e, want))
+    return edits, items, warnings

stylefix/accept.txt

@@ -0,0 +1,94 @@
+# Terms that Vale's bundled dictionary lacks but that are correct in both
+# US and UK English. Technical, policy and internet vocabulary. One per
+# line; case-insensitive; Vale treats each as a regex, so keep it simple.
+# Users add their own in ~/.config/stylefix/accept.txt and ./.stylefix/accept.txt.
+cybersecurity
+cyber
+cyberattack
+cyberattacks
+cybercrime
+cyberspace
+sandboxing
+sandboxed
+sandbox
+auditable
+auditability
+longterm
+interoperability
+interoperable
+scalable
+scalability
+misinformation
+disinformation
+deepfake
+deepfakes
+chatbot
+chatbots
+blockchain
+cryptocurrency
+cryptocurrencies
+malware
+ransomware
+spyware
+phishing
+botnet
+botnets
+exfiltrate
+exfiltration
+containerization
+containerized
+virtualization
+hyperscaler
+hyperscalers
+preprint
+preprints
+misalignment
+misaligned
+superintelligence
+superintelligent
+multimodal
+transformer
+transformers
+finetune
+finetuned
+finetuning
+pretraining
+pretrained
+inference
+agentic
+autonomously
+onboarding
+offboarding
+roadmap
+roadmaps
+workstream
+workstreams
+stakeholder
+stakeholders
+geopolitical
+geopolitics
+geo-strategic
+whistleblower
+whistleblowers
+open-source
+open-sourced
+opensource
+frontier
+biosecurity
+bioweapon
+bioweapons
+dual-use
+zero-day
+API
+APIs
+GPU
+GPUs
+TPU
+TPUs
+LLM
+LLMs
+AGI
+NIST
+CISA
+OSTP
+DARPA

stylefix/adjudicate.py

@@ -0,0 +1,27 @@
+"""stylefix's adjudication task on top of the adjudicate library."""
+
+from __future__ import annotations
+
+from adjudicate.core import (Cache, Decision, Item, decide_batch, parse_response,  # noqa: F401
+                             render_batch, system_prompt)
+from adjudicate.core import adjudicate as _adjudicate
+from adjudicate.llm import Endpoint
+
+VARIETY = {"us": "American English", "uk": "British English"}
+
+TASK = "You are a meticulous copy editor regularizing a document to {variety}."
+KEEP_RULE = ("Choose KEEP when the marked text is part of a proper name, an organisation name, "
+             "the title of a published work, a direct quotation from a source written in another "
+             "variety, a URL or code identifier, or is already correct for {variety} in this "
+             "context. Otherwise choose the option that is correct for {variety}.")
+
+
+def system_for(target: str) -> str:
+    v = VARIETY[target]
+    return system_prompt(TASK.format(variety=v), KEEP_RULE.format(variety=v))
+
+
+def adjudicate(ep: Endpoint | None, target: str, items: list[Item], cache: Cache,
+               batch_size: int = 20, parallel: int = 4, verbose: bool = False) -> list[Decision]:
+    return _adjudicate(ep, target, items, cache, batch_size, parallel, verbose,
+                       system=system_for(target), log_name="stylefix")

stylefix/apply.py

@@ -0,0 +1,3 @@
+"""Edit application: re-exported from the adjudicate library."""
+
+from adjudicate.apply import Edit, apply_edits, line_offsets, match_case, to_offsets  # noqa: F401

stylefix/candidates.py

@@ -0,0 +1,105 @@
+"""Ambiguous words: the options the adjudicator may choose between."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Candidate:
+    options: tuple[str, ...]  # first option is the variety-specific form
+    rule: str
+
+
+def _pair(uk_or_us: str, same: str, rule: str, plural: str | None = None,
+          plural_same: str | None = None) -> dict[str, Candidate]:
+    out = {same: Candidate((uk_or_us, same, "KEEP"), rule)}
+    if plural:
+        out[plural_same or same + "s"] = Candidate((plural, plural_same or same + "s", "KEEP"), rule)
+    return out
+
+
+TO_UK: dict[str, Candidate] = {}
+TO_UK.update(_pair("licence", "license",
+                   "British: 'licence' is the noun (a driving licence); 'license' is the verb (to license software).",
+                   "licences"))
+TO_UK.update(_pair("practise", "practice",
+                   "British: 'practice' is the noun (medical practice, in practice); 'practise' is the verb (to practise the piano).",
+                   "practises"))
+TO_UK.update(_pair("programme", "program",
+                   "British: 'programme' for broadcasts, events, plans and schedules; 'program' only for computer software.",
+                   "programmes"))
+TO_UK.update(_pair("cheque", "check",
+                   "British: 'cheque' only for the bank instrument; 'check' for verifying, restraining, chess, patterns.",
+                   "cheques"))
+TO_UK.update(_pair("storey", "story",
+                   "British: 'storey' is a floor of a building; 'story' is a narrative.",
+                   "storeys", "stories"))
+TO_UK.update(_pair("tyre", "tire",
+                   "British: 'tyre' is the rubber ring on a wheel; 'tire' is the verb (to grow weary).",
+                   "tyres"))
+TO_UK.update(_pair("metre", "meter",
+                   "British: 'metre' is the unit of length (and poetic metre); 'meter' is a measuring device (gas meter).",
+                   "metres"))
+TO_UK.update(_pair("kerb", "curb",
+                   "British: 'kerb' is the edge of a pavement; 'curb' is the verb (to restrain).",
+                   "kerbs"))
+TO_UK.update(_pair("enquiry", "inquiry",
+                   "British: 'enquiry' is an ordinary question or request; 'inquiry' is a formal investigation.",
+                   "enquiries", "inquiries"))
+TO_UK.update(_pair("draught", "draft",
+                   "British: 'draught' for air currents, beer, ships' depth, draught animals, and the game; 'draft' for preliminary versions, banking, and conscription.",
+                   "draughts"))
+TO_UK.update(_pair("judgement", "judgment",
+                   "British: 'judgement' in general use; 'judgment' for a court's formal decision.",
+                   "judgements"))
+TO_UK.update(_pair("learnt", "learned",
+                   "British: 'learnt' as the past tense/participle of learn; 'learned' (two syllables) as the adjective meaning scholarly."))
+TO_UK.update(_pair("disc", "disk",
+                   "British: 'disc' for optical discs, brake discs, anatomy and flat round objects; 'disk' for magnetic computer disks.",
+                   "discs"))
+TO_UK.update(_pair("tonne", "ton",
+                   "British: 'tonne' is the metric unit (1000 kg); 'ton' is the imperial unit or figurative ('tons of').",
+                   "tonnes"))
+TO_UK.update(_pair("whisky", "whiskey",
+                   "British: 'whisky' for Scotch and Canadian; 'whiskey' for Irish and American.",
+                   "whiskies", "whiskeys"))
+TO_UK.update(_pair("mum", "mom",
+                   "British: 'mum' for mother; keep 'mom' only inside a quotation or a name.",
+                   "mums"))
+
+TO_US: dict[str, Candidate] = {}
+TO_US.update(_pair("disk", "disc",
+                   "American: 'disk' for computer disks and general use; 'disc' for optical discs (compact disc), brake discs and anatomy.",
+                   "disks"))
+TO_US.update(_pair("metric ton", "tonne",
+                   "American: 'metric ton' for 1000 kg; keep 'tonne' only inside a quotation or a proper name.",
+                   "metric tons"))
+TO_US.update(_pair("whiskey", "whisky",
+                   "American: 'whiskey' in general; 'whisky' stays for Scotch and Canadian.",
+                   "whiskeys", "whiskys"))
+TO_US.update(_pair("mom", "mum",
+                   "American: 'mom' for mother; keep 'mum' when it means silent ('keep mum') or the flower.",
+                   "moms"))
+
+TABLES = {"uk": TO_UK, "us": TO_US}
+
+
+VARIETY_NAME = {"us": "American", "uk": "British", "oxford": "British (Oxford)"}
+
+
+def lookup(target: str, word: str, oxford: bool = False) -> Candidate | None:
+    """Hand-written entry first; otherwise one built from VarCon's glosses."""
+    hit = TABLES[target].get(word.lower())
+    if hit:
+        return hit
+    from . import varcon
+    key = "oxford" if (target == "uk" and oxford) else target
+    m = varcon.maps(key)
+    forms = m.ambiguous.get(word.lower())
+    if not forms:
+        return None
+    rule = m.rule(word.lower(), VARIETY_NAME[key]) or (
+        f"{VARIETY_NAME[key]}: both spellings exist with different meanings; choose the one "
+        "that fits the sentence.")
+    return Candidate(tuple(sorted(forms, key=lambda f: (f == word.lower(), f))) + ("KEEP",), rule)

stylefix/changelog.py

@@ -0,0 +1,66 @@
+"""Plain-text change log: one line per edit, for --changes and for callers."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+
+from .adjudicate import Decision
+from .apply import Edit, match_case
+from .conventions import Stats
+from .vale_runner import Alert
+
+
+def _context(text: str, edit: Edit, width: int = 28) -> str:
+    lines = text.split("\n")
+    line = lines[edit.line - 1] if edit.line - 1 < len(lines) else ""
+    s, e = edit.start - 1, edit.end
+    left = line[max(0, s - width):s]
+    right = line[e:e + width]
+    return f"{'…' if s > width else ''}{left}[{line[s:e]}]{right}{'…' if len(line) > e + width else ''}"
+
+
+def render(*, text: str, target: str, applied: list[Edit], decisions: list[Decision],
+           stats: Stats, skipped: list[Edit] | None = None,
+           review: list[Alert] | None = None) -> str:
+    """Every change, grouped: word edits in document order, then punctuation
+    counts, then adjudications that were kept or left undecided."""
+    by_pos = {(d.item.line, d.item.start): d for d in decisions}
+    out = [f"stylefix changes (target {target.upper()}): {len(applied)} word edit(s)"]
+    for e in sorted(applied, key=lambda e: (e.line, e.start)):
+        d = by_pos.get((e.line, e.start))
+        how = "auto"
+        if e.source == "adjudicated" and d:
+            how = f"model {d.confidence:.2f}: {d.reason}"
+            if d.item.source == "style":
+                how = "style, " + how
+        elif e.source == "adjudicated":
+            how = "model"
+        shown = match_case(e.original, e.replacement).strip() or repr(e.replacement)
+        out.append(f"  L{e.line:<4} {e.original.strip()} -> {shown:<14} {how}")
+        out.append(f"        {_context(text, e)}")
+    kept = [d for d in decisions if d.choice == "KEEP"
+            or (d.choice and d.choice.lower() == d.item.word.lower())]
+    if kept:
+        out.append(f"kept after adjudication: {len(kept)}")
+        for d in kept:
+            out.append(f"  L{d.item.line:<4} {d.item.word:<20} {d.confidence:.2f}: {d.reason}")
+    undecided = [d for d in decisions if d.choice is None]
+    if undecided:
+        out.append(f"undecided (left as is, see report): {len(undecided)}")
+        for d in undecided:
+            out.append(f"  L{d.item.line:<4} {d.item.word:<20} {d.reason}")
+    if skipped:
+        out.append(f"skipped (text moved): {len(skipped)}")
+    punct = stats.as_dict()
+    if punct:
+        out.append("punctuation: " + ", ".join(f"{k.replace('_', ' ')} {v}" for k, v in punct.items()))
+    if review:
+        out.append(f"for review (not changed): {len(review)}")
+        by_check: dict[str, list[Alert]] = defaultdict(list)
+        for a in review:
+            by_check[a.check].append(a)
+        for check, items in sorted(by_check.items()):
+            out.append(f"  {check}")
+            for a in items:
+                out.append(f"    L{a.line:<4} {a.message}")
+    return "\n".join(out) + "\n"