quire-grammar 0.2.0__py3-none-any.whl

quire_grammar/__init__.py

@@ -0,0 +1,17 @@
+"""quire-grammar — an offline English grammar and style checker.
+
+Public surface (stable):
+
+    check(text, *, enable=None, disable=None, min_confidence=0.0) -> list[Correction]
+
+Everything else is an implementation detail. The phase-2 local model plugs in
+behind ``check`` without changing this signature.
+"""
+
+from __future__ import annotations
+
+from .check import check
+from .types import CATEGORIES, Correction
+
+__version__ = "0.0.1"
+__all__ = ["CATEGORIES", "Correction", "__version__", "check"]

quire_grammar/__main__.py

@@ -0,0 +1,57 @@
+"""quire-grammar CLI:
+
+    quire-grammar "Its a nice day and and the the sky is is blue."
+    echo "text" | quire-grammar -
+    quire-grammar --json chapter-3.txt
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__, check
+
+
+def _line_col(text: str, pos: int) -> tuple[int, int]:
+    head = text[:pos]
+    return head.count("\n") + 1, pos - (head.rfind("\n") + 1) + 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(prog="quire-grammar", description=__doc__)
+    ap.add_argument("source", help='text, a file path, or "-" for stdin')
+    ap.add_argument("--json", action="store_true", help="machine-readable output")
+    ap.add_argument("--min-confidence", type=float, default=0.0)
+    ap.add_argument("--disable", default="", help="comma-separated rule ids / categories")
+    ap.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    args = ap.parse_args(argv)
+
+    if args.source == "-":
+        text = sys.stdin.read()
+    else:
+        try:
+            with open(args.source, encoding="utf-8") as f:
+                text = f.read()
+        except OSError:
+            text = args.source
+
+    hits = check(text, disable=[s for s in args.disable.split(",") if s],
+                 min_confidence=args.min_confidence)
+
+    if args.json:
+        print(json.dumps([vars(h) | {"meta": h.meta} for h in hits], indent=2))
+        return 1 if hits else 0
+
+    for h in hits:
+        line, col = _line_col(text, h.start)
+        fix = f"  →  {h.suggestions[0]!r}" if h.suggestions else ""
+        print(f"{line}:{col}  [{h.category}/{h.rule_id}]  {h.message}{fix}")
+    if not hits:
+        print("no issues found")
+    return 1 if hits else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

quire_grammar/check.py

@@ -0,0 +1,56 @@
+"""The one call the rest of the world makes.
+
+    from quire_grammar import check
+    for c in check("Its a nice day and and the the sky is is blue."):
+        print(c.rule_id, c.original, "->", c.suggestions)
+
+``enable`` / ``disable`` accept rule ids *or* category names, so a UI can
+switch off "style" wholesale or turn on a single opt-in rule.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+
+from . import rules as _rules  # noqa: F401 — importing registers every rule
+from .registry import all_rules
+from .types import Correction
+
+# Phase 2: when the optional model is installed, its predictions are merged
+# in here (same Correction shape, category "agreement" etc.), de-duplicated
+# against the rule hits, and gated by a confidence threshold.
+_model_predict: Callable[[str], list[Correction]] | None
+try:  # pragma: no cover
+    from .model import predict as _model_predict
+except Exception:  # pragma: no cover
+    _model_predict = None
+
+
+def check(
+    text: str,
+    *,
+    enable: Iterable[str] | None = None,
+    disable: Iterable[str] | None = None,
+    min_confidence: float = 0.0,
+) -> list[Correction]:
+    on = set(enable or ())
+    off = set(disable or ())
+
+    out: list[Correction] = []
+    for r in all_rules():
+        keys = {r.id, r.category}
+        active = bool(keys & on) or (r.enabled_by_default and not (keys & off))
+        if active:
+            out.extend(r.fn(text))
+
+    if _model_predict is not None and "model" not in off:  # pragma: no cover
+        rule_spans = [(c.start, c.end) for c in out]
+        for c in _model_predict(text):
+            # a rule hit wins on any span overlap — it's the higher-precision
+            # signal; the model only adds what the rules missed
+            if not any(c.start < re_ and rs < c.end for rs, re_ in rule_spans):
+                out.append(c)
+
+    out = [c for c in out if c.confidence >= min_confidence]
+    out.sort(key=lambda c: (c.start, c.end, c.rule_id))
+    return out

quire_grammar/edits.py

@@ -0,0 +1,241 @@
+"""The GECToR-style edit-tag scheme — shared by the phase-2 model's training
+pipeline and its runtime decoder, so it lives in the shipped package.
+
+The model reads a sentence, splits it into word tokens, and predicts one
+**edit tag** per token. Applying the tags and re-joining rebuilds the
+corrected sentence.
+
+    $KEEP                 leave the token alone
+    $DELETE               drop the token
+    $REPLACE_{t}          swap the token for t
+    $APPEND_{t}           insert t (one or more words) after the token
+    $CASE_CAPITAL         Titlecase the token
+    $CASE_LOWER           lowercase the token
+    $CASE_UPPER           UPPERCASE the token
+
+Kept small on purpose — precision over coverage. Multi-edit tokens (GECToR
+uses iterative rounds) are rare in native-writer text; ``$APPEND_{t}`` carries
+the whole inserted run so one pass covers them. Anything the scheme cannot
+express round-trips wrong, and :func:`encode`'s caller drops the pair.
+
+Zero dependencies — part of the rules-layer contract.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+KEEP = "$KEEP"
+DELETE = "$DELETE"
+_REPLACE = "$REPLACE_"
+_APPEND = "$APPEND_"
+CASE_CAPITAL = "$CASE_CAPITAL"
+CASE_LOWER = "$CASE_LOWER"
+CASE_UPPER = "$CASE_UPPER"
+
+_CASE_TAGS = (CASE_CAPITAL, CASE_LOWER, CASE_UPPER)
+
+#: the tags that are always in the vocabulary, at fixed indices
+BASE_TAGS: tuple[str, ...] = (KEEP, DELETE, *_CASE_TAGS)
+
+
+def replace_tag(token: str) -> str:
+    return _REPLACE + token
+
+
+def append_tag(run: str) -> str:
+    return _APPEND + run
+
+
+def is_replace(tag: str) -> bool:
+    return tag.startswith(_REPLACE)
+
+
+def is_append(tag: str) -> bool:
+    return tag.startswith(_APPEND)
+
+
+def tag_value(tag: str) -> str:
+    """The payload of a ``$REPLACE_`` / ``$APPEND_`` tag."""
+    return tag.split("_", 1)[1]
+
+
+# --------------------------------------------------------------------------
+# tokenisation — word tokens with punctuation split off, plus a deterministic
+# detokeniser. Not exact for pathological spacing; callers verify and drop.
+# --------------------------------------------------------------------------
+
+_TOKEN_RE = re.compile(
+    r"""
+      \d[\d,]*(?:\.\d+)?          # 12  1,000  3.5
+    | [A-Za-z]+(?:['’][A-Za-z]+)* # word, with internal apostrophes
+    | \.\.\.|[.!?]+               # sentence punctuation, runs kept together
+    | --+|[-–—]                   # dashes / hyphen
+    | [^\s\w]                     # any other single punctuation mark
+    """,
+    re.VERBOSE,
+)
+
+# marks that hug the previous token (no space before)
+_CLOSE = set(".,;:!?)]}%…") | {"...", "n't", "'s", "'re", "'ve", "'ll", "'m", "'d"}
+# marks that hug the next token (no space after)
+_OPEN = set("([{$")
+_CONTRACTION = re.compile(r"^['’](s|re|ve|ll|m|d)$|^n['’]t$", re.I)
+
+
+def tokenize(text: str) -> list[str]:
+    return _TOKEN_RE.findall(text)
+
+
+def tokenize_spans(text: str) -> list[tuple[str, int, int]]:
+    """``(token, start, end)`` — same tokens as :func:`tokenize`, with their
+    character offsets in ``text`` (the runtime decoder needs them to place
+    ``Correction`` spans)."""
+    return [(m.group(0), m.start(), m.end()) for m in _TOKEN_RE.finditer(text)]
+
+
+def detokenize(tokens: list[str]) -> str:
+    """Join word tokens back into a sentence. Deterministic; assumes the
+    tokens describe ordinary prose."""
+    out: list[str] = []
+    quote_open = False
+    for tok in tokens:
+        if not out:
+            out.append(tok)
+            continue
+        prev = out[-1]
+        glue = (
+            tok in _CLOSE
+            or _CONTRACTION.match(tok)
+            or prev in _OPEN
+            or prev[-1:] in _OPEN
+            or (tok == '"' and quote_open)
+            or (prev == '"' and not quote_open)
+            or (tok in "-–—")
+            or (prev in "-–—")
+            or (tok in "'’" and quote_open)
+        )
+        out.append(tok if glue else " " + tok)
+        if tok == '"':
+            quote_open = not quote_open
+    return "".join(out)
+
+
+# --------------------------------------------------------------------------
+# alignment — token-level Levenshtein, then read off the edit tags
+# --------------------------------------------------------------------------
+
+
+def _case_transform(src: str, dst: str) -> str | None:
+    if src == dst or src.lower() != dst.lower():
+        return None
+    if dst == src.capitalize():
+        return CASE_CAPITAL
+    if dst == src.lower():
+        return CASE_LOWER
+    if dst == src.upper():
+        return CASE_UPPER
+    return None
+
+
+def _flush_append(tags: list[str], src_pos: int, pending: list[str]) -> None:
+    if pending:
+        tags[src_pos - 1] = append_tag(" ".join(pending))
+        pending.clear()
+
+
+def _tags_from_alignment(src: list[str], tgt: list[str]) -> list[str] | None:
+    n, m = len(src), len(tgt)
+    d = [[0] * (m + 1) for _ in range(n + 1)]
+    for i in range(1, n + 1):
+        d[i][0] = i
+    for j in range(1, m + 1):
+        d[0][j] = j
+    for i in range(1, n + 1):
+        for j in range(1, m + 1):
+            cost = 0 if src[i - 1] == tgt[j - 1] else 1
+            d[i][j] = min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost)
+
+    tags: list[str] = [KEEP] * n
+    pending_append: list[str] = []
+    i, j = n, m
+    lead_insert: list[str] = []
+    while i > 0 or j > 0:
+        if i > 0 and j > 0 and src[i - 1] == tgt[j - 1] and d[i][j] == d[i - 1][j - 1]:
+            _flush_append(tags, i, pending_append)
+            i, j = i - 1, j - 1
+        elif i > 0 and j > 0 and d[i][j] == d[i - 1][j - 1] + 1:
+            ct = _case_transform(src[i - 1], tgt[j - 1])
+            if pending_append:
+                tags[i - 1] = replace_tag(" ".join([tgt[j - 1], *pending_append]))
+                pending_append.clear()
+            else:
+                tags[i - 1] = ct or replace_tag(tgt[j - 1])
+            i, j = i - 1, j - 1
+        elif j > 0 and d[i][j] == d[i][j - 1] + 1:
+            pending_append.insert(0, tgt[j - 1])
+            j -= 1
+            if i == 0:
+                lead_insert = list(pending_append)
+        else:
+            tags[i - 1] = DELETE
+            _flush_append(tags, i, pending_append)
+            i -= 1
+    if lead_insert:
+        return None
+    return tags
+
+
+# --------------------------------------------------------------------------
+# public: encode / apply / verify
+# --------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class Tagged:
+    tokens: tuple[str, ...]
+    tags: tuple[str, ...]
+
+
+def encode(corrupt: str, clean: str) -> Tagged | None:
+    """``(corrupt, clean)`` → token tags, or ``None`` if the scheme can't
+    represent the edit or the result won't round-trip."""
+    src, tgt = tokenize(corrupt), tokenize(clean)
+    if not src:
+        return None
+    tags = _tags_from_alignment(src, tgt)
+    if tags is None:
+        return None
+    tagged = Tagged(tuple(src), tuple(tags))
+    if apply(tagged.tokens, tagged.tags) != detokenize(tgt):
+        return None
+    return tagged
+
+
+def apply(tokens: tuple[str, ...] | list[str],
+          tags: tuple[str, ...] | list[str]) -> str:
+    out: list[str] = []
+    for tok, tag in zip(tokens, tags, strict=True):
+        if tag == KEEP:
+            out.append(tok)
+        elif tag == DELETE:
+            continue
+        elif tag == CASE_CAPITAL:
+            out.append(tok.capitalize())
+        elif tag == CASE_LOWER:
+            out.append(tok.lower())
+        elif tag == CASE_UPPER:
+            out.append(tok.upper())
+        elif is_replace(tag):
+            out.extend(tokenize(tag_value(tag)))
+        elif is_append(tag):
+            out.append(tok)
+            out.extend(tokenize(tag_value(tag)))
+        else:  # unknown tag → treat as KEEP (a miss, never a crash)
+            out.append(tok)
+    return detokenize(out)
+
+
+def verify(corrupt: str, clean: str, tagged: Tagged) -> bool:
+    return apply(tagged.tokens, tagged.tags) == detokenize(tokenize(clean))

quire_grammar/model.py

@@ -0,0 +1,245 @@
+"""Phase 2 — the optional local model behind :func:`quire_grammar.check`.
+
+``check()`` imports :func:`predict` from here inside a ``try``/``except``. When
+the ``ml`` extra (``onnxruntime`` + ``tokenizers``) isn't installed, or the
+model file isn't bundled in ``models/``, :func:`predict` returns ``[]`` and
+the checker runs rules-only — exactly how Quire treats a missing WordNet or
+spell dictionary. It never raises for a missing model.
+
+The model is a GECToR-style tagger: word tokens in, one
+:mod:`quire_grammar.edits` tag per token out. This module runs the ONNX
+graph, reads the argmax tag for each word, and turns the non-``$KEEP`` ones
+into :class:`~quire_grammar.types.Correction` spans over the original text.
+"""
+
+from __future__ import annotations
+
+import importlib.resources
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from .edits import (
+    CASE_CAPITAL,
+    CASE_LOWER,
+    CASE_UPPER,
+    KEEP,
+    is_append,
+    is_replace,
+    tag_value,
+    tokenize_spans,
+)
+from .edits import (
+    apply as _apply_tags,
+)
+from .types import AGREEMENT, CONFUSABLE, MECHANICS, PUNCTUATION, Correction
+
+_FILES = ("tagger.onnx", "tokenizer.json", "tag_vocab.json")
+
+# Drop any predicted edit below this softmax mass. Tuned on held-out clean
+# Gutenberg prose: 0.9 keeps the false-flag rate on correct text near 1.3% on
+# x86 Linux while F0.5 on the synthetic test set holds at ~0.88.
+#
+# CAVEAT: int8 ONNX inference is NOT bit-portable. On a genuinely ambiguous
+# input one word scored 0.51 on x86 Linux and 0.94 on Apple Silicon — no
+# fixed threshold is stable across CPU architectures near a decision
+# boundary. The model is advisory, not autocorrect, and the deterministic
+# rules layer is the backbone; still, treat model output as "correct on this
+# machine" not "the same everywhere". v3: try QDQ static quantisation, or
+# drop the ambiguous confusable pairs (breath/breathe, affect/effect, …)
+# that the narrow rules already cover.
+# Precision over recall — raise it, never lower it, without re-checking.
+_MIN_PROB = 0.9
+_CONFIDENCE_SCALE = 0.9   # a model hit stays below a mechanical rule's 0.9
+_MAX_TOKENS = 128
+
+# homophone pairs the model may reach for — flagged as CONFUSABLE, not
+# AGREEMENT, when it swaps one for the other
+_CONFUSABLE_PAIRS: frozenset[frozenset[str]] = frozenset(
+    frozenset(p) for p in (
+        ("its", "it's"), ("there", "their"), ("there", "they're"),
+        ("their", "they're"), ("your", "you're"), ("then", "than"),
+        ("affect", "effect"), ("lose", "loose"), ("to", "too"),
+        ("passed", "past"), ("lead", "led"), ("whose", "who's"),
+        ("weather", "whether"), ("breath", "breathe"),
+    )
+)
+
+
+@dataclass(frozen=True)
+class _Model:
+    session: Any
+    tokenizer: Any
+    id2tag: dict[int, str]
+
+
+_UNSET: Any = object()
+_cache: _Model | None = _UNSET
+
+
+def _resolve_dir() -> Any:
+    """The directory holding the model files, or ``None``. Checked in order:
+    the companion ``quire-grammar-model`` package (the ``[model]`` extra),
+    then a ``models/`` folder inside this package (a local build). Both go
+    through ``importlib.resources`` so they resolve inside a frozen app."""
+    try:
+        import quire_grammar_model
+
+        d = quire_grammar_model.path()
+        if all((d / f).is_file() for f in _FILES):
+            return d
+    except ImportError:
+        pass
+    try:
+        d = importlib.resources.files("quire_grammar") / "models"
+        if all((d / f).is_file() for f in _FILES):
+            return d
+    except (ModuleNotFoundError, FileNotFoundError):
+        pass
+    return None
+
+
+def _load() -> _Model | None:
+    try:
+        import onnxruntime
+        from tokenizers import Tokenizer
+    except ImportError:
+        return None
+    d = _resolve_dir()
+    if d is None:
+        return None
+    try:
+        # str() of a Traversable is a real path for an unpacked package —
+        # and a 66 MB model is never shipped zipped
+        session = onnxruntime.InferenceSession(
+            str(d / "tagger.onnx"), providers=["CPUExecutionProvider"])
+        tokenizer = Tokenizer.from_file(str(d / "tokenizer.json"))
+        tags = json.loads((d / "tag_vocab.json").read_text(encoding="utf-8"))["tags"]
+        return _Model(session, tokenizer, dict(enumerate(tags)))
+    except Exception:  # a corrupt bundle must not take the checker down
+        return None
+
+
+def _model() -> _Model | None:
+    global _cache
+    if _cache is _UNSET:
+        _cache = _load()
+    return _cache
+
+
+def reset_cache() -> None:
+    """Forget the loaded model — for tests, or after swapping the bundle."""
+    global _cache
+    _cache = _UNSET
+
+
+def is_available() -> bool:
+    return _model() is not None
+
+
+def _softmax(row: Any) -> Any:
+    import numpy as np
+
+    shifted = row - row.max()
+    ex = np.exp(shifted)
+    return ex / ex.sum()
+
+
+def _category(original: str, replacement: str) -> str:
+    if frozenset((original.lower(), replacement.lower())) in _CONFUSABLE_PAIRS:
+        return CONFUSABLE
+    if replacement and not replacement[0].isalnum():
+        return PUNCTUATION
+    return AGREEMENT
+
+
+def _to_correction(text: str, span: tuple[str, int, int], tag: str,
+                   conf: float) -> Correction | None:
+    word, start, end = span
+    if is_replace(tag):
+        repl = tag_value(tag)
+        return Correction(
+            start, end, word, _category(word, repl), "model:replace",
+            f"“{word}” here reads as “{repl}”.", (repl,), conf)
+    if is_append(tag):
+        become = _apply_tags((word,), (tag,))
+        inserted = tag_value(tag)
+        cat = PUNCTUATION if inserted[:1] and not inserted[0].isalnum() else AGREEMENT
+        return Correction(
+            start, end, word, cat, "model:append",
+            f"“{word}” may want “{become}” here.", (become,), conf)
+    if tag in (CASE_CAPITAL, CASE_LOWER, CASE_UPPER):
+        fixed = {CASE_CAPITAL: word.capitalize(), CASE_LOWER: word.lower(),
+                 CASE_UPPER: word.upper()}[tag]
+        if fixed == word:
+            return None
+        return Correction(
+            start, end, word, MECHANICS, "model:case",
+            f"“{word}” may need to be “{fixed}” here.", (fixed,), conf)
+    if tag == "$DELETE":
+        stop = end
+        while stop < len(text) and text[stop] == " ":
+            stop += 1
+        return Correction(
+            start, stop, text[start:stop], _category(word, ""), "model:delete",
+            f"“{word}” may be extra here.", ("",), conf)
+    return None
+
+
+def decode(text: str, per_word: list[tuple[str, float] | None]) -> list[Correction]:
+    """Turn per-word ``(tag, probability)`` predictions — aligned to
+    ``tokenize_spans(text)`` — into ``Correction`` spans. Pure: the numpy /
+    ONNX half of :func:`predict` hands off here, and the tests drive it
+    directly."""
+    spans = tokenize_spans(text)
+    out: list[Correction] = []
+    for span, pred in zip(spans, per_word, strict=False):
+        if pred is None:
+            continue
+        tag, prob = pred
+        if tag == KEEP or prob < _MIN_PROB:
+            continue
+        corr = _to_correction(text, span, tag, round(prob * _CONFIDENCE_SCALE, 3))
+        if corr is not None:
+            out.append(corr)
+    return out
+
+
+def predict(text: str) -> list[Correction]:
+    """Model corrections for ``text``. ``[]`` when the model isn't available."""
+    model = _model()
+    if model is None or not text.strip():
+        return []
+
+    import numpy as np
+
+    spans = tokenize_spans(text)
+    if not spans:
+        return []
+    words = [w for w, _, _ in spans]
+
+    enc = model.tokenizer.encode(words, is_pretokenized=True)
+    ids = list(enc.ids)[:_MAX_TOKENS]
+    word_ids = list(enc.word_ids)[:_MAX_TOKENS]
+
+    feed = {
+        "input_ids": np.asarray([ids], dtype=np.int64),
+        "attention_mask": np.ones((1, len(ids)), dtype=np.int64),
+    }
+    logits = model.session.run(None, feed)[0][0]  # (seq, n_tags)
+
+    first_pos: dict[int, int] = {}
+    for sub_i, wid in enumerate(word_ids):
+        if wid is not None and wid not in first_pos:
+            first_pos[wid] = sub_i
+
+    per_word: list[tuple[str, float] | None] = []
+    for wi in range(len(spans)):
+        pos = first_pos.get(wi)
+        if pos is None:  # word past the token limit
+            per_word.append(None)
+            continue
+        probs = _softmax(logits[pos])
+        k = int(probs.argmax())
+        per_word.append((model.id2tag.get(k, KEEP), float(probs[k])))
+    return decode(text, per_word)

quire_grammar/registry.py

@@ -0,0 +1,45 @@
+"""The rule registry. Each rule is a pure function ``str -> Iterable[Correction]``
+registered with an id and a category. No state, no I/O, no dependencies.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+
+from .types import CATEGORIES, Correction
+
+Checker = Callable[[str], Iterable[Correction]]
+
+
+@dataclass(frozen=True)
+class Rule:
+    id: str
+    category: str
+    fn: Checker
+    enabled_by_default: bool = True
+
+
+_RULES: dict[str, Rule] = {}
+
+
+def rule(id: str, category: str, *, default: bool = True) -> Callable[[Checker], Checker]:
+    """Decorator: register ``fn`` as a rule."""
+    if category not in CATEGORIES:
+        raise ValueError(f"{id}: unknown category {category!r}")
+
+    def deco(fn: Checker) -> Checker:
+        if id in _RULES:
+            raise ValueError(f"duplicate rule id: {id}")
+        _RULES[id] = Rule(id, category, fn, default)
+        return fn
+
+    return deco
+
+
+def all_rules() -> list[Rule]:
+    return list(_RULES.values())
+
+
+def rule_ids() -> list[str]:
+    return list(_RULES)

quire_grammar/rules/__init__.py

@@ -0,0 +1,11 @@
+"""Importing this package registers every rule as a side effect."""
+
+from __future__ import annotations
+
+from . import (  # noqa: F401
+    agreement,
+    confusables,
+    mechanics,
+    punctuation,
+    style,
+)