fetchit-engine 0.1.2__tar.gz → 0.3.0__tar.gz

{fetchit_engine-0.1.2/fetchit_engine.egg-info → fetchit_engine-0.3.0}/PKG-INFO

@@ -1,43 +1,43 @@
-Metadata-Version: 2.4
-Name: fetchit-engine
-Version: 0.1.2
-Summary: Deterministic text cleanup and AI-writing heuristics. Runs entirely in your process; text never leaves it.
-Author: Outta Work Solutions
-License-Expression: Apache-2.0
-Project-URL: Homepage, https://fetchitai.com/developers
-Project-URL: Repository, https://github.com/OuttaWorkSolutions/fetchit-engine
-Project-URL: Changelog, https://github.com/OuttaWorkSolutions/fetchit-engine/blob/main/CHANGELOG.md
-Project-URL: Issues, https://github.com/OuttaWorkSolutions/fetchit-engine/issues
-Keywords: ai text,cleanup,invisible characters,em dash,text hygiene
-Classifier: Programming Language :: Python :: 3
-Classifier: Operating System :: OS Independent
-Classifier: Topic :: Text Processing :: Linguistic
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-License-File: NOTICE
-Provides-Extra: spell
-Requires-Dist: pyspellchecker>=0.7; extra == "spell"
-Dynamic: license-file
-
-# fetchit-engine
-
-Deterministic text cleanup and AI-writing heuristics that run entirely in your
-process. Your users text never leaves it. Pure standard library, no dependencies.
-
-```python
-from fetchit_engine import clean, apply_edits
-
-r = clean(ai_draft, {"mode": "auto"})   # apply only auto-safe edits unattended
-publish(r["cleaned"]["text"])
-if r["aiReport"].get("level") == "high":
-    review_queue.put(ai_draft, r)        # hand a human the full result
-```
-
-Offsets are code points. See the [repository README](https://github.com/OuttaWorkSolutions/fetchit-engine#the-design-in-four-claims)
-for the full CleanResult contract, shared 1:1 with @fetchitai/engine (JavaScript).
-
-
-## License
-
-Apache-2.0. Free for everyone, including commercial use.
+Metadata-Version: 2.4
+Name: fetchit-engine
+Version: 0.3.0
+Summary: Deterministic text cleanup and AI-writing heuristics. Runs entirely in your process; text never leaves it.
+Author: Outta Work Solutions
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://fetchitai.com/developers
+Project-URL: Repository, https://github.com/OuttaWorkSolutions/fetchit-engine
+Project-URL: Changelog, https://github.com/OuttaWorkSolutions/fetchit-engine/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/OuttaWorkSolutions/fetchit-engine/issues
+Keywords: ai text,cleanup,invisible characters,em dash,text hygiene
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICE
+Provides-Extra: spell
+Requires-Dist: pyspellchecker>=0.7; extra == "spell"
+Dynamic: license-file
+
+# fetchit-engine
+
+Deterministic text cleanup and AI-writing heuristics that run entirely in your
+process. Your users text never leaves it. Pure standard library, no dependencies.
+
+```python
+from fetchit_engine import clean, apply_edits
+
+r = clean(ai_draft, {"mode": "auto"})   # apply only auto-safe edits unattended
+publish(r["cleaned"]["text"])
+if r["aiReport"].get("level") == "high":
+    review_queue.put(ai_draft, r)        # hand a human the full result
+```
+
+Offsets are code points. See the [repository README](https://github.com/OuttaWorkSolutions/fetchit-engine#the-design-in-four-claims)
+for the full CleanResult contract, shared 1:1 with @fetchitai/engine (JavaScript).
+
+
+## License
+
+Apache-2.0. Free for everyone, including commercial use.

{fetchit_engine-0.1.2 → fetchit_engine-0.3.0}/fetchit_engine/core.py

@@ -15,7 +15,7 @@ import json
 import os
 import re
 
-ENGINE_VERSION = "0.1.2"
+ENGINE_VERSION = "0.3.0"
 
 # --- ruleset (single source of truth, shared with the JS package) -----------
 _RULESET_PATH = os.path.join(
@@ -36,6 +36,14 @@ _INVISIBLE_RANGES = RULESET["invisibleRanges"]
 _ODD_SPACE_RANGES = RULESET["oddSpaceRanges"]
 _ODD_SPACE_RULE_ID = RULESET["oddSpaceRuleId"]
 _ODD_SPACE_LABEL = RULESET["oddSpaceLabel"]
+_CONFUSABLE_RULE_ID = RULESET["confusableRuleId"]
+_CONFUSABLE_LABEL = RULESET["confusableLabel"]
+_TYPOGRAPHY_RULE_ID = RULESET["typographyRuleId"]
+_TYPOGRAPHY_LABEL = RULESET["typographyLabel"]
+# code point -> replacement, built once from the shared tables so both
+# languages derive the same lookup from the same data.
+_CONFUSABLES = {c["cp"]: c["to"] for c in RULESET["confusables"]}
+_TYPOGRAPHY = {c["cp"]: c["to"] for c in RULESET["typography"]}
 _T = RULESET["thresholds"]
 MIN_CHARS = _T["MIN_CHARS"]
 MIN_WORDS = _T["MIN_WORDS"]
@@ -43,10 +51,12 @@ _LEVEL_MODERATE = _T["levelModerate"]
 _LEVEL_HIGH = _T["levelHigh"]
 
 # --- regexes that are logic, not data (kept in code; may contain dashes) -----
-# Em dash (U+2014) / horizontal bar (U+2015) in any spacing; en dash (U+2013)
-# only when spaced on both sides. Only spaces/tabs are consumed so line breaks
-# survive. Mirrors _EM_DASH in text_tools.py.
-_EM_DASH = re.compile(r"[ \t]*[—―][ \t]*|[ \t]+–[ \t]+")
+# A RUN of em dashes (U+2014) / horizontal bars (U+2015), optionally
+# space-separated, is one match, so "a——b" produces one comma, not two edits;
+# an en dash (U+2013) matches only when spaced on both sides, so numeric
+# ranges like 3–5 survive. Only spaces/tabs are consumed so line breaks
+# survive. Mirrors EM_DASH_RE in the JS engine.
+_EM_DASH = re.compile(r"[ \t]*[—―](?:[ \t]*[—―])*[ \t]*|[ \t]+–[ \t]+")
 _MULTI_SPACE = re.compile(r"[ \t]{2,}")
 _SPACE_BEFORE_PUNCT = re.compile(r"[ \t]+([,.;:!?])")
 
@@ -57,8 +67,16 @@ _RULE_OF_THREE = re.compile(
     r"\b[A-Za-z]+,\s+[A-Za-z]+,\s+and\s+[A-Za-z]+\b", re.IGNORECASE | re.ASCII
 )
 _LIST_MARKER = re.compile(r"^\s*(?:[-*•·]|\d+[.)])\s+", re.MULTILINE | re.ASCII)
-_CONTRACTION = re.compile(r"\b[A-Za-z]+'(?:t|s|re|ve|ll|d|m)\b", re.IGNORECASE | re.ASCII)
-_WORD = re.compile(r"[A-Za-z']+")
+# U+2019 is included deliberately. Word processors and AI assistants emit the
+# curly apostrophe, and matching only the straight one made this signal report
+# "almost no contractions" on prose that was full of them, inflating the score
+# by 10 points, while _WORD split "don’t" into two words.
+_CONTRACTION = re.compile(r"\b[A-Za-z]+['’](?:t|s|re|ve|ll|d|m)\b", re.IGNORECASE | re.ASCII)
+_WORD = re.compile(r"[A-Za-z'’]+")
+# Markdown that survived a paste out of a chat window into running prose.
+_MARKDOWN = re.compile(
+    r"\*\*[^*\n]+\*\*|__[^_\n]+__|^#{1,6}[ \t]|\[[^\]\n]+\]\([^)\n]+\)", re.MULTILINE
+)
 # Sentence split: same as the JS splitSentences(). ASCII whitespace only.
 _SENTENCE_SPLIT = re.compile(r"(?<=[.!?])[ \t\n\r\f\v]+", re.ASCII)
 # One shared whitespace set for trimming and word splitting, identical in both
@@ -85,14 +103,62 @@ def _split_ws(s):
     return [w for w in re.split(r"[ \t\n\r\f\v]+", s) if w]
 
 # Rule metadata for edits produced by the dash/space normalization passes.
-_DASH_RULE = ("dash.spaced", "dash", "Replaced a spaced dash with a space")
+_DASH_RULE_ID = "dash.spaced"
+_DASH_COMMA_RULE = (_DASH_RULE_ID, "dash", "Replaced a dash with a comma")
+_DASH_SPACE_RULE = (_DASH_RULE_ID, "dash", "Replaced a spaced dash with a space")
+
+
+def _dash_rule_for(rep):
+    return _DASH_COMMA_RULE if rep[:1] == "," else _DASH_SPACE_RULE
+
+
 _COLLAPSE_RULE = ("space.collapse", "space", "Collapsed repeated spaces")
+_CONFUSABLE_RULE = (_CONFUSABLE_RULE_ID, "homoglyph", "Replaced a " + _CONFUSABLE_LABEL)
+_TYPOGRAPHY_RULE = (_TYPOGRAPHY_RULE_ID, "typography", "Normalized a " + _TYPOGRAPHY_LABEL)
 _SPACE_BEFORE_RULE = ("punct.space-before", "space", "Removed a space before punctuation")
 
+# What a matched dash run becomes. A clause dash reads as a pause, so the
+# default replacement is a comma. The comma is withheld (single space instead,
+# the old behavior, tidied by the collapse and space-before-punct passes) when
+# a comma cannot sit there: at a text or line boundary, next to punctuation or
+# a bracket it would double up against, next to a dash the match could not
+# consume, or between digits, where the dash is a range rather than a pause.
+# When the next character is whitespace the comma takes no trailing space, so
+# it hugs the word before a line break. Mirrors dashReplacement in the JS
+# engine.
+_DASH_CP = frozenset((0x2014, 0x2015, 0x2013))
+_NO_COMMA_BEFORE = frozenset(ord(c) for c in ",.;:!?([{")
+_NO_COMMA_AFTER = frozenset(ord(c) for c in ",.;:!?)]}")
+
+
+def _cp_before(s, i):
+    return ord(s[i - 1]) if i > 0 else -1
+
+
+def _cp_after(s, i):
+    return ord(s[i]) if i < len(s) else -1
+
+
+def _dash_replacement(before_cp, after_cp):
+    blocked_before = (before_cp < 0 or before_cp in _WS
+                      or before_cp in _NO_COMMA_BEFORE or before_cp in _DASH_CP)
+    blocked_after = after_cp < 0 or after_cp in _NO_COMMA_AFTER or after_cp in _DASH_CP
+    if blocked_before or blocked_after:
+        return " "
+    if 0x30 <= before_cp <= 0x39 and 0x30 <= after_cp <= 0x39:
+        return " "
+    return "," if after_cp in _WS else ", "
+
+
+def _dash_match_replacement(m):
+    return _dash_replacement(_cp_before(m.string, m.start()), _cp_after(m.string, m.end()))
+
 # Priority when several passes touch the same characters and their cells merge.
 # Higher wins the label. Dash beats space cleanup beats invisible.
 _RULE_PRIORITY = {
+    "homoglyph.mixed-script": 45,
     "dash.spaced": 40,
+    "typography.smart": 35,
     "space.lookalike": 30,
     "space.collapse": 20,
     "punct.space-before": 15,
@@ -181,14 +247,22 @@ def _regex_pass(cells, regex, replacement, rule):
         merged = cells[ci:cj]
         src0 = merged[0].src0
         src1 = merged[-1].src1
+        if callable(replacement):
+            rep = replacement(m)
+        elif "\\" in replacement:
+            rep = m.expand(replacement)
+        else:
+            rep = replacement
+        # `rule` may depend on the replacement chosen (the dash pass labels
+        # comma and space outcomes differently while keeping one ruleId).
+        base = rule(rep) if callable(rule) else rule
         # if any merged cell already carried a higher-priority rule, keep it
-        best = rule
-        best_pri = _RULE_PRIORITY.get(rule[0], 0)
+        best = base
+        best_pri = _RULE_PRIORITY.get(base[0], 0)
         for c in merged:
             if c.rule and _RULE_PRIORITY.get(c.rule[0], 0) > best_pri:
                 best = c.rule
                 best_pri = _RULE_PRIORITY.get(c.rule[0], 0)
-        rep = m.expand(replacement) if "\\" in replacement else replacement
         out.append(_Cell(rep, src0, src1, best))
         last = cj
         changed = True
@@ -208,6 +282,8 @@ def _build_cells(text, disabled=frozenset()):
             cells.append(_Cell("", i, i + 1, (inv["id"], "invisible", "Removed " + inv["label"])))
         elif _is_odd_space(cp) and _ODD_SPACE_RULE_ID not in disabled:
             cells.append(_Cell(" ", i, i + 1, (_ODD_SPACE_RULE_ID, "space", "Normalized a " + _ODD_SPACE_LABEL)))
+        elif cp in _TYPOGRAPHY and _TYPOGRAPHY_RULE_ID not in disabled:
+            cells.append(_Cell(_TYPOGRAPHY[cp], i, i + 1, _TYPOGRAPHY_RULE))
         else:
             cells.append(_Cell(ch, i, i + 1, None))
     return cells
@@ -239,25 +315,70 @@ def _cells_to_edits(cells, text):
     return edits
 
 
+def _is_latin_letter(t):
+    return len(t) == 1 and (65 <= ord(t) <= 90 or 97 <= ord(t) <= 122)
+
+
+def _is_confusable_text(t):
+    return len(t) == 1 and ord(t) in _CONFUSABLES
+
+
+def _homoglyph_pass(cells, disabled=frozenset()):
+    """A confusable is only a problem when it hides inside a word that is
+    otherwise Latin. Replacing them wholesale would destroy genuine Cyrillic or
+    Greek text, so a run is rewritten only when it MIXES scripts. Cells whose
+    text is empty (an invisible character already removed) are transparent, so
+    "a<ZWSP>pple" with a Cyrillic a is still seen as one word.
+    Mirrors homoglyphPass in the JS engine."""
+    if _CONFUSABLE_RULE_ID in disabled:
+        return cells
+    wordish = lambda c: _is_latin_letter(c.text) or _is_confusable_text(c.text)
+    i = 0
+    while i < len(cells):
+        if not wordish(cells[i]):
+            i += 1
+            continue
+        j = i
+        last_word = i
+        has_latin = False
+        has_confusable = False
+        while j < len(cells) and (wordish(cells[j]) or cells[j].text == ""):
+            if _is_latin_letter(cells[j].text):
+                has_latin = True
+                last_word = j
+            elif _is_confusable_text(cells[j].text):
+                has_confusable = True
+                last_word = j
+            j += 1
+        if has_latin and has_confusable:
+            for k in range(i, last_word + 1):
+                if not _is_confusable_text(cells[k].text):
+                    continue
+                cells[k] = _Cell(_CONFUSABLES[ord(cells[k].text)],
+                                 cells[k].src0, cells[k].src1, _CONFUSABLE_RULE)
+        i = max(j, i + 1)
+    return cells
+
+
 def _clean_cells(text, disabled=frozenset()):
     """Full clean pipeline over cells. Mirrors clean_text() in text_tools.py:
     rebuild (invisible + odd space), then, only if a dash was present, the em
     dash pass plus multi-space collapse and space-before-punct tidy. A disabled
     rule id skips its pass entirely, so cleaned text and the edit list agree."""
-    cells = _build_cells(text, disabled)
+    cells = _homoglyph_pass(_build_cells(text, disabled), disabled)
     current = "".join(c.text for c in cells)
     # Run the dash/space passes to a FIXED POINT, not once. A single pass is
-    # not idempotent: removing an em dash can manufacture the spacing that arms
-    # the spaced-en-dash rule ("X—– Y" -> "X – Y", and only a second clean
-    # reached "X Y"), which broke the documented clean(clean(x)) == clean(x)
-    # contract. Each enabled dash pass strictly reduces the dash count, so this
-    # terminates; the equality check breaks when the dash rule is disabled.
-    # Mirrors cleanCells() in the JS engine exactly.
+    # not idempotent: replacing an em dash can manufacture the spacing that
+    # arms the spaced-en-dash rule ("X—– Y" -> "X – Y", and only the next
+    # iteration reaches "X, Y"), which would break the documented
+    # clean(clean(x)) == clean(x) contract. Each enabled dash pass strictly
+    # reduces the dash count, so this terminates; the equality check breaks
+    # when the dash rule is disabled. Mirrors cleanCells() in the JS engine.
     for _guard in range(8):
         if not _EM_DASH.search(current):
             break
-        if _DASH_RULE[0] not in disabled:
-            cells, _ = _regex_pass(cells, _EM_DASH, " ", _DASH_RULE)
+        if _DASH_RULE_ID not in disabled:
+            cells, _ = _regex_pass(cells, _EM_DASH, _dash_match_replacement, _dash_rule_for)
         if _COLLAPSE_RULE[0] not in disabled:
             cells, _ = _regex_pass(cells, _MULTI_SPACE, " ", _COLLAPSE_RULE)
         if _SPACE_BEFORE_RULE[0] not in disabled:
@@ -281,12 +402,12 @@ def rebuild_text(text):
 
 
 def remove_em_dashes(text):
-    """Replace spaced dashes with a single space. Returns (new_text, count).
-    Back-compatible with text_tools.remove_em_dashes."""
+    """Replace clause dashes with a comma where one fits, else a space.
+    Returns (new_text, count); count is the number of matched dash groups."""
     count = len(_EM_DASH.findall(text))
     if not count:
         return text, 0
-    new = _EM_DASH.sub(" ", text)
+    new = _EM_DASH.sub(_dash_match_replacement, text)
     new = _MULTI_SPACE.sub(" ", new)
     new = _SPACE_BEFORE_PUNCT.sub(r"\1", new)
     return new, count
@@ -404,6 +525,16 @@ def analyze_ai_signals(text):
                         "message": "Some rule-of-three phrasing"})
 
     # 6. List structure.
+    markdown_hits = len(_MARKDOWN.findall(text))
+    if markdown_hits >= 2:
+        score += 10
+        signals.append({"id": "signal.markdown-artifacts", "points": 10,
+                        "message": "Markdown left in the text (%d marks), a sign of a paste from a chat window" % markdown_hits})
+    elif markdown_hits == 1:
+        score += 5
+        signals.append({"id": "signal.markdown-artifacts", "points": 5,
+                        "message": "A markdown mark left in the text"})
+
     enum_hits = sum(len(re.findall(r"\b" + re.escape(w["text"]) + r"\b", lowered, re.ASCII))
                     for w in _ENUMERATORS)
     marker_hits = len(_LIST_MARKER.findall(text))
@@ -500,6 +631,10 @@ def clean(text, options=None):
     # Dashes count by consumed CHARACTER too: with the fixed-point dash pass, a
     # chain like "—– " merges into one edit that removed two dashes.
     dashes_n = _count_consumed(lambda cp: cp in (0x2014, 0x2015, 0x2013))
+    # Same measured-by-consumption rule: a confusable left inside a genuinely
+    # Cyrillic word is never consumed, so it is never counted.
+    homoglyphs_n = _count_consumed(lambda cp: cp in _CONFUSABLES)
+    typography_n = _count_consumed(lambda cp: cp in _TYPOGRAPHY)
 
     return {
         "engineVersion": ENGINE_VERSION,
@@ -514,6 +649,8 @@ def clean(text, options=None):
             "invisible": invisible_n,
             "oddSpaces": oddspace_n,
             "dashes": dashes_n,
+            "homoglyphs": homoglyphs_n,
+            "typography": typography_n,
             "hidden": invisible_n + oddspace_n,
             "flagged": len(flags),
         },