fetchit-engine 0.1.1__py3-none-any.whl

fetchit_engine/__init__.py

@@ -0,0 +1,31 @@
+"""fetchit-engine: deterministic text cleanup and AI-writing heuristics.
+
+Public API mirrors @fetchit/engine (JavaScript). See core.py for the
+CleanResult contract; parity is enforced by packages/engine-core/vectors.json.
+"""
+from .core import (
+    ENGINE_VERSION,
+    RULESET_VERSION,
+    MIN_CHARS,
+    MIN_WORDS,
+    clean,
+    rebuild_text,
+    remove_em_dashes,
+    apply_edits,
+    find_ai_spans,
+    analyze_ai_signals,
+)
+
+__all__ = [
+    "ENGINE_VERSION",
+    "RULESET_VERSION",
+    "MIN_CHARS",
+    "MIN_WORDS",
+    "clean",
+    "rebuild_text",
+    "remove_em_dashes",
+    "apply_edits",
+    "find_ai_spans",
+    "analyze_ai_signals",
+]
+__version__ = ENGINE_VERSION

fetchit_engine/core.py

@@ -0,0 +1,506 @@
+"""Fetch It AI text engine (canonical core).
+
+Emits the language-neutral CleanResult contract shared with @fetchit/engine
+(JavaScript). Ported in lockstep; the parity vectors in
+packages/engine-core/vectors.json are the contract both sides must satisfy.
+
+Offsets in every result are CODE POINTS into the input text (offsetUnit
+"codePoint"), so Python and JavaScript agree regardless of astral characters.
+JS hosts that paint into the DOM convert to UTF-16 with the converter shipped
+in the JS package.
+
+This module has no third-party dependencies and no Qt/DOM coupling.
+"""
+import json
+import os
+import re
+
+ENGINE_VERSION = "0.1.1"
+
+# --- ruleset (single source of truth, shared with the JS package) -----------
+_RULESET_PATH = os.path.join(
+    os.path.dirname(__file__), "ruleset.json"
+)
+
+
+def _load_ruleset(path=_RULESET_PATH):
+    with open(path, "r", encoding="utf-8") as fh:
+        return json.load(fh)
+
+
+RULESET = _load_ruleset()
+RULESET_VERSION = RULESET["rulesetVersion"]
+_PHRASES = RULESET["phrases"]                 # [{id, text}]
+_ENUMERATORS = RULESET["enumerators"]         # [{id, text}]
+_INVISIBLE_RANGES = RULESET["invisibleRanges"]
+_ODD_SPACE_RANGES = RULESET["oddSpaceRanges"]
+_ODD_SPACE_RULE_ID = RULESET["oddSpaceRuleId"]
+_ODD_SPACE_LABEL = RULESET["oddSpaceLabel"]
+_T = RULESET["thresholds"]
+MIN_CHARS = _T["MIN_CHARS"]
+MIN_WORDS = _T["MIN_WORDS"]
+_LEVEL_MODERATE = _T["levelModerate"]
+_LEVEL_HIGH = _T["levelHigh"]
+
+# --- regexes that are logic, not data (kept in code; may contain dashes) -----
+# Em dash (U+2014) / horizontal bar (U+2015) in any spacing; en dash (U+2013)
+# only when spaced on both sides. Only spaces/tabs are consumed so line breaks
+# survive. Mirrors _EM_DASH in text_tools.py.
+_EM_DASH = re.compile(r"[ \t]*[—―][ \t]*|[ \t]+–[ \t]+")
+_MULTI_SPACE = re.compile(r"[ \t]{2,}")
+_SPACE_BEFORE_PUNCT = re.compile(r"[ \t]+([,.;:!?])")
+
+# re.ASCII pins \b, \d, \w, \s to ASCII so they match JavaScript's default
+# (ASCII) semantics exactly. Without it Python treats accented letters as word
+# characters and JS does not, which breaks \b matches near Unicode letters.
+_RULE_OF_THREE = re.compile(
+    r"\b[A-Za-z]+,\s+[A-Za-z]+,\s+and\s+[A-Za-z]+\b", re.IGNORECASE | re.ASCII
+)
+_LIST_MARKER = re.compile(r"^\s*(?:[-*•·]|\d+[.)])\s+", re.MULTILINE | re.ASCII)
+_CONTRACTION = re.compile(r"\b[A-Za-z]+'(?:t|s|re|ve|ll|d|m)\b", re.IGNORECASE | re.ASCII)
+_WORD = re.compile(r"[A-Za-z']+")
+# Sentence split: same as the JS splitSentences(). ASCII whitespace only.
+_SENTENCE_SPLIT = re.compile(r"(?<=[.!?])[ \t\n\r\f\v]+", re.ASCII)
+# One shared whitespace set for trimming and word splitting, identical in both
+# engines. It is the union of what Python str.strip() and JS String.trim()
+# consider whitespace, so gating on length never diverges by language.
+_WS = frozenset(
+    [0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x85, 0xA0,
+     0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000, 0xFEFF]
+    + list(range(0x2000, 0x200B))
+)
+
+
+def _strip(text):
+    a, b = 0, len(text)
+    while a < b and ord(text[a]) in _WS:
+        a += 1
+    while b > a and ord(text[b - 1]) in _WS:
+        b -= 1
+    return text[a:b]
+
+
+def _split_ws(s):
+    """Split on ASCII whitespace runs, dropping empties. Matches JS splitWords."""
+    return [w for w in re.split(r"[ \t\n\r\f\v]+", s) if w]
+
+# Rule metadata for edits produced by the dash/space normalization passes.
+_DASH_RULE = ("dash.spaced", "dash", "Replaced a spaced dash with a space")
+_COLLAPSE_RULE = ("space.collapse", "space", "Collapsed repeated spaces")
+_SPACE_BEFORE_RULE = ("punct.space-before", "space", "Removed a space before punctuation")
+
+# Priority when several passes touch the same characters and their cells merge.
+# Higher wins the label. Dash beats space cleanup beats invisible.
+_RULE_PRIORITY = {
+    "dash.spaced": 40,
+    "space.lookalike": 30,
+    "space.collapse": 20,
+    "punct.space-before": 15,
+}
+
+
+def _ascii_lower(text):
+    """Lowercase only A-Z. Length- and position-preserving, so offsets computed
+    against the result map 1:1 back onto the input. Every rule phrase is ASCII,
+    so this never misses a match, and it is identical in JS."""
+    out = []
+    for ch in text:
+        o = ord(ch)
+        if 65 <= o <= 90:
+            out.append(chr(o + 32))
+        else:
+            out.append(ch)
+    return "".join(out)
+
+
+def _invisible_rule(codepoint):
+    for r in _INVISIBLE_RANGES:
+        if r["min"] <= codepoint <= r["max"]:
+            return r
+    return None
+
+
+def _is_odd_space(codepoint):
+    for r in _ODD_SPACE_RANGES:
+        if r["min"] <= codepoint <= r["max"]:
+            return True
+    return False
+
+
+class _Cell:
+    """One unit of the working buffer, carrying provenance back to the input.
+
+    text     current character(s), "" for a deletion
+    src0/src1 half-open code-point range in the INPUT this cell represents
+    rule     (ruleId, category, message) if this cell is the product of an edit,
+             else None
+    """
+    __slots__ = ("text", "src0", "src1", "rule")
+
+    def __init__(self, text, src0, src1, rule=None):
+        self.text = text
+        self.src0 = src0
+        self.src1 = src1
+        self.rule = rule
+
+
+def _regex_pass(cells, regex, replacement, rule):
+    """Run a regex over the string the cells currently spell, replacing each
+    match's cell range with a single edit cell. Match indices are code points
+    (Python native), which line up with cell boundaries because these regexes
+    only ever match BMP space/tab/dash/punct characters, never astral ones."""
+    current = "".join(c.text for c in cells)
+    if not regex.search(current):
+        return cells, False
+    # offset (in code points) where each cell's text starts in `current`
+    starts = []
+    pos = 0
+    for c in cells:
+        starts.append(pos)
+        pos += len(c.text)
+    starts.append(pos)
+
+    def cell_at(offset):
+        # first cell index whose start == offset (boundaries always align)
+        lo, hi = 0, len(cells)
+        while lo < hi:
+            mid = (lo + hi) // 2
+            if starts[mid] < offset:
+                lo = mid + 1
+            else:
+                hi = mid
+        return lo
+
+    out = []
+    last = 0
+    changed = False
+    for m in regex.finditer(current):
+        a, b = m.start(), m.end()
+        ci, cj = cell_at(a), cell_at(b)
+        out.extend(cells[last:ci])
+        merged = cells[ci:cj]
+        src0 = merged[0].src0
+        src1 = merged[-1].src1
+        # if any merged cell already carried a higher-priority rule, keep it
+        best = rule
+        best_pri = _RULE_PRIORITY.get(rule[0], 0)
+        for c in merged:
+            if c.rule and _RULE_PRIORITY.get(c.rule[0], 0) > best_pri:
+                best = c.rule
+                best_pri = _RULE_PRIORITY.get(c.rule[0], 0)
+        rep = m.expand(replacement) if "\\" in replacement else replacement
+        out.append(_Cell(rep, src0, src1, best))
+        last = cj
+        changed = True
+    out.extend(cells[last:])
+    return out, changed
+
+
+def _build_cells(text, disabled=frozenset()):
+    """Pass A: invisible removal and look-alike-space normalization, cell-wise
+    by code point. A disabled rule leaves its characters untouched. Returns the
+    cell list (input order preserved)."""
+    cells = []
+    for i, ch in enumerate(text):
+        cp = ord(ch)
+        inv = _invisible_rule(cp)
+        if inv and inv["id"] not in disabled:
+            cells.append(_Cell("", i, i + 1, (inv["id"], "invisible", "Removed " + inv["label"])))
+        elif _is_odd_space(cp) and _ODD_SPACE_RULE_ID not in disabled:
+            cells.append(_Cell(" ", i, i + 1, (_ODD_SPACE_RULE_ID, "space", "Normalized a " + _ODD_SPACE_LABEL)))
+        else:
+            cells.append(_Cell(ch, i, i + 1, None))
+    return cells
+
+
+def _cells_to_edits(cells, text):
+    """Turn edit-cells into CleanResult edit records (input code-point coords)."""
+    edits = []
+    n = 0
+    for c in cells:
+        if c.rule is None:
+            continue
+        original = text[c.src0:c.src1]
+        if original == c.text:
+            continue  # no-op (e.g. a look-alike space that was already " ")
+        n += 1
+        rule_id, category, message = c.rule
+        edits.append({
+            "id": "e%d" % n,
+            "ruleId": rule_id,
+            "category": category,
+            "severity": "auto",
+            "start": c.src0,
+            "end": c.src1,
+            "original": original,
+            "replacement": c.text,
+            "message": message,
+        })
+    return edits
+
+
+def _clean_cells(text, disabled=frozenset()):
+    """Full clean pipeline over cells. Mirrors clean_text() in text_tools.py:
+    rebuild (invisible + odd space), then, only if a dash was present, the em
+    dash pass plus multi-space collapse and space-before-punct tidy. A disabled
+    rule id skips its pass entirely, so cleaned text and the edit list agree."""
+    cells = _build_cells(text, disabled)
+    rebuilt = "".join(c.text for c in cells)
+    if _EM_DASH.search(rebuilt):
+        if _DASH_RULE[0] not in disabled:
+            cells, _ = _regex_pass(cells, _EM_DASH, " ", _DASH_RULE)
+        if _COLLAPSE_RULE[0] not in disabled:
+            cells, _ = _regex_pass(cells, _MULTI_SPACE, " ", _COLLAPSE_RULE)
+        if _SPACE_BEFORE_RULE[0] not in disabled:
+            cells, _ = _regex_pass(cells, _SPACE_BEFORE_PUNCT, "\\1", _SPACE_BEFORE_RULE)
+    cleaned = "".join(c.text for c in cells)
+    return cleaned, cells
+
+
+# --- public building blocks --------------------------------------------------
+
+def rebuild_text(text):
+    """Rebuild from visible characters only (invisible dropped, look-alike
+    spaces normalized). Returns (new_text, changed_count). Back-compatible with
+    text_tools.rebuild_text."""
+    cells = _build_cells(text)
+    changed = sum(1 for c in cells if c.rule is not None and text[c.src0:c.src1] != c.text)
+    return "".join(c.text for c in cells), changed
+
+
+def remove_em_dashes(text):
+    """Replace spaced dashes with a single space. Returns (new_text, count).
+    Back-compatible with text_tools.remove_em_dashes."""
+    count = len(_EM_DASH.findall(text))
+    if not count:
+        return text, 0
+    new = _EM_DASH.sub(" ", text)
+    new = _MULTI_SPACE.sub(" ", new)
+    new = _SPACE_BEFORE_PUNCT.sub(r"\1", new)
+    return new, count
+
+
+def apply_edits(text, accepted_ids, edits):
+    """Apply the chosen edits to text (code-point splices, right to left).
+    Accepting every edit reproduces CleanResult.cleaned.text exactly."""
+    accepted = set(accepted_ids)
+    chosen = [e for e in edits if e["id"] in accepted]
+    chosen.sort(key=lambda e: e["start"], reverse=True)
+    chars = list(text)
+    for e in chosen:
+        chars[e["start"]:e["end"]] = list(e["replacement"])
+    return "".join(chars)
+
+
+def find_ai_spans(text):
+    """(start, end) code-point spans of AI-associated wording, sorted and
+    non-overlapping. Matches on an ASCII-lowercased copy so offsets map 1:1 to
+    the input (no lowercase round-trip drift)."""
+    return [(s["start"], s["end"]) for s in _find_ai_flags(text)]
+
+
+def _find_ai_flags(text):
+    lowered = _ascii_lower(text)
+    raw = []
+    for p in _PHRASES:
+        phrase = p["text"]
+        start = lowered.find(phrase)
+        while start != -1:
+            raw.append((start, start + len(phrase), p["id"], phrase))
+            start = lowered.find(phrase, start + len(phrase))
+    for w in _ENUMERATORS:
+        for m in re.finditer(r"\b" + re.escape(w["text"]) + r"\b", lowered, re.ASCII):
+            raw.append((m.start(), m.end(), w["id"], w["text"]))
+    raw.sort(key=lambda s: (s[0], -(s[1] - s[0])))
+    chosen = []
+    reached = 0
+    for start, end, rid, txt in raw:
+        if start >= reached:
+            chosen.append({
+                "ruleId": rid, "start": start, "end": end,
+                "text": text[start:end], "matched": txt,
+            })
+            reached = end
+    return chosen
+
+
+def analyze_ai_signals(text):
+    """Heuristic AI-writing scan. Returns a report with structured signals
+    (each carrying a stable id and its point contribution)."""
+    stripped = _strip(text)
+    if not stripped:
+        return {"status": "empty"}
+    words = _WORD.findall(stripped)
+    if len(stripped) < MIN_CHARS or len(words) < MIN_WORDS:
+        return {"status": "too_short"}
+
+    score = 0
+    signals = []
+    lowered = _ascii_lower(stripped)
+
+    # 1. Em/en dash density.
+    dash_count = sum(stripped.count(d) for d in ("—", "–", "―"))
+    dash_rate = dash_count / len(stripped) * 1000
+    if dash_rate > 3:
+        score += 25
+        signals.append({"id": "signal.dash-density", "points": 25,
+                        "message": "Heavy em-dash use (%d dashes)" % dash_count})
+    elif dash_rate > 1.2:
+        score += 12
+        signals.append({"id": "signal.dash-density", "points": 12,
+                        "message": "Frequent em-dash use (%d dashes)" % dash_count})
+
+    # 2. Stock AI phrases.
+    found = [p["text"] for p in _PHRASES if p["text"] in lowered]
+    if found:
+        occurrences = sum(lowered.count(p) for p in found)
+        pts = min(30, 10 * len(found))
+        score += pts
+        shown = ", ".join('"%s"' % p for p in found[:4])
+        extra = " and %d more" % (len(found) - 4) if len(found) > 4 else ""
+        signals.append({"id": "signal.stock-phrases", "points": pts,
+                        "message": "AI-associated wording: %s%s (%d occurrence(s))"
+                        % (shown, extra, occurrences)})
+
+    # 3. Contraction rate.
+    contractions = len(_CONTRACTION.findall(stripped))
+    if len(words) >= 120 and contractions / len(words) * 100 < 0.5:
+        score += 10
+        signals.append({"id": "signal.contractions", "points": 10,
+                        "message": "Almost no contractions (stiff, formal tone)"})
+
+    # 4. Sentence-length uniformity.
+    sentences = [s for s in _SENTENCE_SPLIT.split(stripped) if len(_split_ws(s)) >= 3]
+    if len(sentences) >= 6:
+        lengths = [len(_split_ws(s)) for s in sentences]
+        mean = sum(lengths) / len(lengths)
+        variance = sum((v - mean) ** 2 for v in lengths) / len(lengths)
+        if mean and (variance ** 0.5) / mean < 0.35:
+            score += 15
+            signals.append({"id": "signal.sentence-uniformity", "points": 15,
+                            "message": "Unusually uniform sentence lengths"})
+
+    # 5. Rule of three.
+    triads = len(_RULE_OF_THREE.findall(stripped))
+    if triads >= 3:
+        score += 12
+        signals.append({"id": "signal.rule-of-three", "points": 12,
+                        "message": "Frequent rule-of-three lists (%d triads)" % triads})
+    elif triads == 2:
+        score += 6
+        signals.append({"id": "signal.rule-of-three", "points": 6,
+                        "message": "Some rule-of-three phrasing"})
+
+    # 6. List structure.
+    enum_hits = sum(len(re.findall(r"\b" + re.escape(w["text"]) + r"\b", lowered, re.ASCII))
+                    for w in _ENUMERATORS)
+    marker_hits = len(_LIST_MARKER.findall(text))
+    list_signal = enum_hits + marker_hits
+    if list_signal >= 4:
+        score += 12
+        signals.append({"id": "signal.list-structure", "points": 12,
+                        "message": "Heavily list-structured (enumerators / bullets)"})
+    elif list_signal >= 2:
+        score += 6
+        signals.append({"id": "signal.list-structure", "points": 6,
+                        "message": "Somewhat list-structured"})
+
+    score = min(score, 100)
+    level = "high" if score >= _LEVEL_HIGH else "moderate" if score >= _LEVEL_MODERATE else "low"
+    return {"status": "ok", "score": score, "level": level, "signals": signals}
+
+
+# --- the one-call entry point -----------------------------------------------
+
+def clean(text, options=None):
+    """Clean text and return the full CleanResult.
+
+    options (all optional):
+      mode: "review" (default) | "auto" - advisory only; the result is identical.
+            Callers apply auto edits unattended and show suggest flags to a human.
+      rules.disable: list of rule ids to drop from edits and flags.
+      rules.customPhrases: extra phrases flagged as AI-associated wording.
+    """
+    options = options or {}
+    rules = options.get("rules") or {}
+    disabled = set(rules.get("disable") or [])
+    custom = list(rules.get("customPhrases") or [])
+
+    cleaned, cells = _clean_cells(text, disabled)
+    edits = _cells_to_edits(cells, text)
+
+    flag_hits = _find_ai_flags(text)
+    if custom:
+        lowered = _ascii_lower(text)
+        for phrase in custom:
+            needle = _ascii_lower(phrase)
+            if not needle:
+                continue
+            start = lowered.find(needle)
+            while start != -1:
+                flag_hits.append({"ruleId": "ai-custom." + re.sub(r"[^a-z0-9]+", "-", needle).strip("-"),
+                                  "start": start, "end": start + len(needle),
+                                  "text": text[start:start + len(needle)], "matched": needle})
+                start = lowered.find(needle, start + len(needle))
+        flag_hits.sort(key=lambda s: (s["start"], -(s["end"] - s["start"])))
+        deduped = []
+        reached = 0
+        for h in flag_hits:
+            if h["start"] >= reached:
+                deduped.append(h)
+                reached = h["end"]
+        flag_hits = deduped
+
+    flags = []
+    fn = 0
+    for h in flag_hits:
+        if h["ruleId"] in disabled:
+            continue
+        fn += 1
+        flags.append({
+            "id": "f%d" % fn,
+            "ruleId": h["ruleId"],
+            "category": "ai-wording",
+            "severity": "suggest",
+            "start": h["start"],
+            "end": h["end"],
+            "text": h["text"],
+            "replacement": None,
+            "message": "AI-associated wording",
+        })
+
+    report = analyze_ai_signals(text)
+
+    # Count the characters an edit actually CONSUMED, not the rule that fired.
+    #
+    # Attributing by ruleId undercounts: a broader rule can swallow a span that
+    # contained invisible characters. " <ZWSP> " is collapsed by space.collapse,
+    # which removed the zero-width space while reporting hidden: 0. Since no
+    # replacement ever contains an invisible or look-alike character, counting
+    # them in `original` is exact regardless of which rule did the removing.
+    def _count_consumed(pred):
+        return sum(
+            1 for e in edits for ch in e["original"] if pred(ord(ch))
+        )
+
+    invisible_n = _count_consumed(lambda cp: _invisible_rule(cp) is not None)
+    oddspace_n = _count_consumed(_is_odd_space)
+    dashes_n = sum(1 for e in edits if e["ruleId"] == "dash.spaced")
+
+    return {
+        "engineVersion": ENGINE_VERSION,
+        "rulesetVersion": RULESET_VERSION,
+        "offsetUnit": "codePoint",
+        "input": {"length": len(text)},
+        "cleaned": {"text": cleaned, "length": len(cleaned)},
+        "edits": edits,
+        "flags": flags,
+        "aiReport": report,
+        "summary": {
+            "invisible": invisible_n,
+            "oddSpaces": oddspace_n,
+            "dashes": dashes_n,
+            "hidden": invisible_n + oddspace_n,
+            "flagged": len(flags),
+        },
+    }

fetchit_engine/ruleset.json

@@ -0,0 +1,389 @@
+{
+  "rulesetVersion": "2026-08-18",
+  "phrases": [
+    {
+      "id": "ai-wording.delve",
+      "text": "delve"
+    },
+    {
+      "id": "ai-wording.delving",
+      "text": "delving"
+    },
+    {
+      "id": "ai-wording.tapestry",
+      "text": "tapestry"
+    },
+    {
+      "id": "ai-wording.seamlessly",
+      "text": "seamlessly"
+    },
+    {
+      "id": "ai-wording.meticulously",
+      "text": "meticulously"
+    },
+    {
+      "id": "ai-wording.myriad",
+      "text": "myriad"
+    },
+    {
+      "id": "ai-wording.plethora",
+      "text": "plethora"
+    },
+    {
+      "id": "ai-wording.underscore",
+      "text": "underscore"
+    },
+    {
+      "id": "ai-wording.multifaceted",
+      "text": "multifaceted"
+    },
+    {
+      "id": "ai-wording.unleash",
+      "text": "unleash"
+    },
+    {
+      "id": "ai-wording.unveil",
+      "text": "unveil"
+    },
+    {
+      "id": "ai-wording.nuanced",
+      "text": "nuanced"
+    },
+    {
+      "id": "ai-wording.pivotal",
+      "text": "pivotal"
+    },
+    {
+      "id": "ai-wording.commendable",
+      "text": "commendable"
+    },
+    {
+      "id": "ai-wording.noteworthy",
+      "text": "noteworthy"
+    },
+    {
+      "id": "ai-wording.intricate",
+      "text": "intricate"
+    },
+    {
+      "id": "ai-wording.realm",
+      "text": "realm"
+    },
+    {
+      "id": "ai-wording.landscape",
+      "text": "landscape"
+    },
+    {
+      "id": "ai-wording.testament",
+      "text": "testament"
+    },
+    {
+      "id": "ai-wording.it-is-important-to-note",
+      "text": "it is important to note"
+    },
+    {
+      "id": "ai-wording.it-s-important-to-note",
+      "text": "it's important to note"
+    },
+    {
+      "id": "ai-wording.it-s-worth-noting",
+      "text": "it's worth noting"
+    },
+    {
+      "id": "ai-wording.it-is-worth-noting",
+      "text": "it is worth noting"
+    },
+    {
+      "id": "ai-wording.that-being-said",
+      "text": "that being said"
+    },
+    {
+      "id": "ai-wording.when-it-comes-to",
+      "text": "when it comes to"
+    },
+    {
+      "id": "ai-wording.first-and-foremost",
+      "text": "first and foremost"
+    },
+    {
+      "id": "ai-wording.at-the-end-of-the-day",
+      "text": "at the end of the day"
+    },
+    {
+      "id": "ai-wording.in-today-s-fast-paced",
+      "text": "in today's fast-paced"
+    },
+    {
+      "id": "ai-wording.in-an-era-of",
+      "text": "in an era of"
+    },
+    {
+      "id": "ai-wording.in-the-realm-of",
+      "text": "in the realm of"
+    },
+    {
+      "id": "ai-wording.on-the-other-hand",
+      "text": "on the other hand"
+    },
+    {
+      "id": "ai-wording.more-than-just",
+      "text": "more than just"
+    },
+    {
+      "id": "ai-wording.plays-a-crucial-role",
+      "text": "plays a crucial role"
+    },
+    {
+      "id": "ai-wording.plays-a-vital-role",
+      "text": "plays a vital role"
+    },
+    {
+      "id": "ai-wording.plays-a-significant-role",
+      "text": "plays a significant role"
+    },
+    {
+      "id": "ai-wording.a-wide-range-of",
+      "text": "a wide range of"
+    },
+    {
+      "id": "ai-wording.a-wide-array-of",
+      "text": "a wide array of"
+    },
+    {
+      "id": "ai-wording.various-factors",
+      "text": "various factors"
+    },
+    {
+      "id": "ai-wording.it-is-essential-to",
+      "text": "it is essential to"
+    },
+    {
+      "id": "ai-wording.it-is-crucial-to",
+      "text": "it is crucial to"
+    },
+    {
+      "id": "ai-wording.there-are-several",
+      "text": "there are several"
+    },
+    {
+      "id": "ai-wording.ever-evolving",
+      "text": "ever-evolving"
+    },
+    {
+      "id": "ai-wording.ever-changing",
+      "text": "ever-changing"
+    },
+    {
+      "id": "ai-wording.game-changer",
+      "text": "game-changer"
+    },
+    {
+      "id": "ai-wording.game-changer-2",
+      "text": "game changer"
+    },
+    {
+      "id": "ai-wording.testament-to",
+      "text": "testament to"
+    },
+    {
+      "id": "ai-wording.paving-the-way",
+      "text": "paving the way"
+    },
+    {
+      "id": "ai-wording.look-no-further",
+      "text": "look no further"
+    },
+    {
+      "id": "ai-wording.beacon-of",
+      "text": "beacon of"
+    },
+    {
+      "id": "ai-wording.elevate-your",
+      "text": "elevate your"
+    },
+    {
+      "id": "ai-wording.unlock-the-full-potential",
+      "text": "unlock the full potential"
+    },
+    {
+      "id": "ai-wording.unlock-the-power",
+      "text": "unlock the power"
+    },
+    {
+      "id": "ai-wording.embark-on-a-journey",
+      "text": "embark on a journey"
+    },
+    {
+      "id": "ai-wording.navigating-the-complexities",
+      "text": "navigating the complexities"
+    },
+    {
+      "id": "ai-wording.navigating-the",
+      "text": "navigating the"
+    },
+    {
+      "id": "ai-wording.holistic-approach",
+      "text": "holistic approach"
+    },
+    {
+      "id": "ai-wording.at-its-core",
+      "text": "at its core"
+    },
+    {
+      "id": "ai-wording.let-s-dive-in",
+      "text": "let's dive in"
+    },
+    {
+      "id": "ai-wording.dive-deeper",
+      "text": "dive deeper"
+    },
+    {
+      "id": "ai-wording.rich-tapestry",
+      "text": "rich tapestry"
+    },
+    {
+      "id": "ai-wording.treasure-trove",
+      "text": "treasure trove"
+    },
+    {
+      "id": "ai-wording.in-conclusion",
+      "text": "in conclusion,"
+    },
+    {
+      "id": "ai-wording.in-summary",
+      "text": "in summary,"
+    },
+    {
+      "id": "ai-wording.to-sum-up",
+      "text": "to sum up"
+    },
+    {
+      "id": "ai-wording.ultimately",
+      "text": "ultimately,"
+    }
+  ],
+  "enumerators": [
+    {
+      "id": "ai-enumerator.firstly",
+      "text": "firstly"
+    },
+    {
+      "id": "ai-enumerator.secondly",
+      "text": "secondly"
+    },
+    {
+      "id": "ai-enumerator.thirdly",
+      "text": "thirdly"
+    },
+    {
+      "id": "ai-enumerator.fourthly",
+      "text": "fourthly"
+    },
+    {
+      "id": "ai-enumerator.lastly",
+      "text": "lastly"
+    },
+    {
+      "id": "ai-enumerator.moreover",
+      "text": "moreover"
+    },
+    {
+      "id": "ai-enumerator.furthermore",
+      "text": "furthermore"
+    },
+    {
+      "id": "ai-enumerator.additionally",
+      "text": "additionally"
+    },
+    {
+      "id": "ai-enumerator.consequently",
+      "text": "consequently"
+    }
+  ],
+  "invisibleRanges": [
+    {
+      "min": 173,
+      "max": 173,
+      "id": "invisible.soft-hyphen",
+      "label": "soft hyphen"
+    },
+    {
+      "min": 6158,
+      "max": 6158,
+      "id": "invisible.format",
+      "label": "Mongolian vowel separator"
+    },
+    {
+      "min": 8203,
+      "max": 8205,
+      "id": "invisible.zero-width",
+      "label": "zero-width character"
+    },
+    {
+      "min": 8206,
+      "max": 8207,
+      "id": "invisible.bidi",
+      "label": "bidirectional mark"
+    },
+    {
+      "min": 8234,
+      "max": 8238,
+      "id": "invisible.bidi",
+      "label": "bidirectional control"
+    },
+    {
+      "min": 8288,
+      "max": 8292,
+      "id": "invisible.zero-width",
+      "label": "word joiner / invisible operator"
+    },
+    {
+      "min": 8293,
+      "max": 8303,
+      "id": "invisible.format",
+      "label": "format control"
+    },
+    {
+      "min": 65279,
+      "max": 65279,
+      "id": "invisible.zero-width",
+      "label": "byte-order mark"
+    },
+    {
+      "min": 917504,
+      "max": 917631,
+      "id": "invisible.tag-block",
+      "label": "Unicode tag character"
+    }
+  ],
+  "oddSpaceRanges": [
+    {
+      "min": 160,
+      "max": 160
+    },
+    {
+      "min": 8192,
+      "max": 8202
+    },
+    {
+      "min": 8239,
+      "max": 8239
+    },
+    {
+      "min": 8287,
+      "max": 8287
+    },
+    {
+      "min": 12288,
+      "max": 12288
+    }
+  ],
+  "oddSpaceRuleId": "space.lookalike",
+  "oddSpaceLabel": "look-alike space",
+  "thresholds": {
+    "MIN_CHARS": 200,
+    "MIN_WORDS": 40,
+    "levelModerate": 20,
+    "levelHigh": 45
+  }
+}

fetchit_engine/spell_core.py

@@ -0,0 +1,117 @@
+"""Structured spellcheck, extracted from the desktop app's spellcheck.py.
+
+Qt-free: this is the reusable detection and ranking logic with the Qt widget
+classes removed, returning misspellings and ranked suggestions as plain data.
+The desktop SpellHighlighter / SpellCheckTextEdit can be refactored to consume
+this later; the tool web engine does not use it (browsers spellcheck natively).
+
+Backed by pyspellchecker, which is an optional extra:
+    pip install "fetchit-engine[spell]"
+If it is not installed, get_spellchecker() returns None and the functions here
+degrade to empty results, exactly like the desktop app does.
+
+Offsets in check_text results are CODE POINTS into the text, matching the rest
+of the engine (offsetUnit "codePoint"). Results use the same shape as engine
+flags: severity "suggest", a ruleId, category, start/end, and text.
+"""
+import re
+
+WORD_RE = re.compile(r"[A-Za-z']{2,}")
+
+_spell = None
+_spell_load_failed = False
+
+
+def get_spellchecker():
+    """Lazily load the dictionary (~0.3s once). Returns None if pyspellchecker
+    is unavailable, so callers keep working without spellcheck."""
+    global _spell, _spell_load_failed
+    if _spell is None and not _spell_load_failed:
+        try:
+            from spellchecker import SpellChecker
+
+            _spell = SpellChecker()
+        except Exception:
+            _spell_load_failed = True
+    return _spell
+
+
+def is_checkable(word):
+    """Skip things that are not prose: words with digits, ALL-CAPS acronyms,
+    and CamelCase / mid-word capitals. Same rule as the desktop app."""
+    if any(ch.isdigit() for ch in word):
+        return False
+    if word.isupper():
+        return False
+    if any(ch.isupper() for ch in word[1:]):
+        return False
+    return True
+
+
+def suggest(word, max_suggestions=5):
+    """Ranked replacement suggestions for one word: the best correction first,
+    then remaining candidates alphabetically, capped. Capitalization of the
+    original is restored. Returns [] if the word is fine, too long/short, not
+    checkable, or the dictionary is unavailable. Mirrors the desktop ranking."""
+    spell = get_spellchecker()
+    if spell is None:
+        return []
+    w = word.strip("'")
+    # Long garbage strings make candidate search slow; skip them (as the app does).
+    if not w or not (2 <= len(w) <= 15) or not is_checkable(w):
+        return []
+    lw = w.lower()
+    if lw not in spell.unknown([lw]):
+        return []
+    best = spell.correction(lw)
+    candidates = set(spell.candidates(lw) or set())
+    candidates.discard(lw)
+    ordered = []
+    if best and best != lw:
+        ordered.append(best)
+    ordered.extend(sorted(c for c in candidates if c != best))
+    ordered = ordered[:max_suggestions]
+    if w[0].isupper():
+        ordered = [c.capitalize() for c in ordered]
+    return ordered
+
+
+def check_text(text, with_suggestions=True, max_suggestions=5):
+    """Find misspelled words in text. Returns a list of records:
+
+        {ruleId, category, severity, start, end, text, suggestions, message}
+
+    start/end are code-point offsets. Every unknown checkable word (2+ letters)
+    is reported; suggestions are provided for words 2-15 characters long when
+    with_suggestions is true (longer words are still reported, with []).
+    Returns [] if the dictionary is unavailable."""
+    found = []
+    for m in WORD_RE.finditer(text):
+        raw = m.group()
+        word = raw.strip("'")
+        if word and is_checkable(word):
+            start = m.start() + raw.index(word)
+            found.append((start, start + len(word), word))
+
+    spell = get_spellchecker()
+    if spell is None or not found:
+        return []
+
+    unknown = spell.unknown({w.lower() for _, _, w in found})
+    if not unknown:
+        return []
+
+    results = []
+    for start, end, word in found:
+        if word.lower() in unknown:
+            results.append({
+                "ruleId": "spell.misspelling",
+                "category": "spell",
+                "severity": "suggest",
+                "start": start,
+                "end": end,
+                "text": word,
+                "suggestions": suggest(word, max_suggestions) if with_suggestions else [],
+                "message": "Possible misspelling",
+            })
+    return results

fetchit_engine-0.1.1.dist-info/METADATA

@@ -0,0 +1,43 @@
+Metadata-Version: 2.4
+Name: fetchit-engine
+Version: 0.1.1
+Summary: Deterministic text cleanup and AI-writing heuristics. Runs entirely in your process; text never leaves it.
+Author: Outta Work Solutions
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://fetchitai.com/developers
+Project-URL: Repository, https://github.com/OuttaWorkSolutions/fetchit-engine
+Project-URL: Changelog, https://github.com/OuttaWorkSolutions/fetchit-engine/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/OuttaWorkSolutions/fetchit-engine/issues
+Keywords: ai text,cleanup,invisible characters,em dash,text hygiene
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICE
+Provides-Extra: spell
+Requires-Dist: pyspellchecker>=0.7; extra == "spell"
+Dynamic: license-file
+
+# fetchit-engine
+
+Deterministic text cleanup and AI-writing heuristics that run entirely in your
+process. Your users text never leaves it. Pure standard library, no dependencies.
+
+```python
+from fetchit_engine import clean, apply_edits
+
+r = clean(ai_draft, {"mode": "auto"})   # apply only auto-safe edits unattended
+publish(r["cleaned"]["text"])
+if r["aiReport"].get("level") == "high":
+    review_queue.put(ai_draft, r)        # hand a human the full result
+```
+
+Offsets are code points. See the [repository README](https://github.com/OuttaWorkSolutions/fetchit-engine#the-design-in-four-claims)
+for the full CleanResult contract, shared 1:1 with @fetchitai/engine (JavaScript).
+
+
+## License
+
+Apache-2.0. Free for everyone, including commercial use.

fetchit_engine-0.1.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+fetchit_engine/__init__.py,sha256=Z37F_L0nx1r7xw5Qr1jbZw9M4DwdkVcm481YR5wEd50,676
+fetchit_engine/core.py,sha256=YyA1bn6Wb0hycE0SZzCJnX2H_eNJkk5A63xF_dF-FGI,19195
+fetchit_engine/ruleset.json,sha256=GNimU9WFU36pOTVhIlAjjtm9TrmVFq69LCAIcLziLBM,8232
+fetchit_engine/spell_core.py,sha256=hCWdmetoqnrregMs0nEMOqUkwA6qvk89IpGmo8aDCj4,4261
+fetchit_engine-0.1.1.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+fetchit_engine-0.1.1.dist-info/licenses/NOTICE,sha256=akoIiN0j_ZZBtzA_mVqs0MNy8oXezJHK8nluu998v4w,146
+fetchit_engine-0.1.1.dist-info/METADATA,sha256=9A04o9h4lL4b6QuxpkKsEMTmVMfI04wI4jC4tfmq9Ik,1762
+fetchit_engine-0.1.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+fetchit_engine-0.1.1.dist-info/top_level.txt,sha256=IK43FAp244fOwCdK8vAtdozPviRmGLOzTFG2yq69wCA,15
+fetchit_engine-0.1.1.dist-info/RECORD,,

fetchit_engine-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fetchit_engine-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

fetchit_engine-0.1.1.dist-info/licenses/NOTICE

@@ -0,0 +1,5 @@
+Fetch It AI engine
+Copyright 2026 Outta Work Solutions
+
+This product includes software developed at Outta Work Solutions
+(https://fetchitai.com).

fetchit_engine-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+fetchit_engine