safetype 1.5.0__tar.gz

safetype-1.5.0/.gitignore

@@ -0,0 +1,16 @@
+node_modules/
+dist/
+coverage/
+.turbo/
+*.log
+.DS_Store
+test-results/
+playwright-report/
+packages/*/e2e/dist/
+packages/vue/e2e/dist/
+.angular/
+
+.venv/
+python/dist/
+__pycache__/
+.pytest_cache/

safetype-1.5.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 SafeType.js contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

safetype-1.5.0/PKG-INFO

@@ -0,0 +1,122 @@
+Metadata-Version: 2.5
+Name: safetype
+Version: 1.5.0
+Summary: Rule-based content moderation (profanity, hate, harassment) for Python servers. Same engine as SafeType.js.
+Project-URL: Homepage, https://github.com/karammm/safetype
+Project-URL: Documentation, https://github.com/karammm/safetype/tree/main/python#readme
+Project-URL: Issues, https://github.com/karammm/safetype/issues
+Project-URL: Changelog, https://github.com/karammm/safetype/blob/main/CHANGELOG.md
+Author: SafeType.js contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: content-safety,filter,harassment,hate-speech,moderation,profanity
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# safetype (Python)
+
+Rule-based content moderation for Python servers: profanity, hate speech and harassment.
+It's the **same engine and English dictionary** as the [SafeType.js](https://github.com/karammm/safetype)
+JavaScript packages, so your backend can re-check exactly what the browser checked.
+It has no dependencies and runs fully offline.
+
+```bash
+pip install safetype
+```
+
+```python
+from safetype import create_moderation_engine
+
+engine = create_moderation_engine(mode="mask")
+engine.moderate("What the fuck are you doing?").text
+# 'What the f*ck are you doing?'
+```
+
+## Server-side enforcement
+
+Client-side moderation can be bypassed (devtools, curl, modified clients). Re-check before you store:
+
+```python
+engine = create_moderation_engine(
+    mode="mask",
+    policy={"hate": {"action": "block"}, "harassment": {"action": "block"}},
+)
+
+check = engine.validate(comment)
+if not check.valid:
+    raise ValidationError([v.category for v in check.violations])
+store(check.result.text)                      # profanity masked
+if check.result.requires_review:
+    queue_for_human_review(comment)           # e.g. quoted or reported speech
+```
+
+FastAPI:
+
+```python
+@app.post("/comments")
+def create_comment(body: CommentIn):
+    check = engine.validate(body.text)
+    if not check.valid:
+        raise HTTPException(422, detail=[{"category": v.category, "rule": v.rule_id} for v in check.violations])
+    return save(check.result.text)
+```
+
+Django form:
+
+```python
+def clean_body(self):
+    check = engine.validate(self.cleaned_data["body"])
+    if not check.valid:
+        raise forms.ValidationError("Please keep it respectful.")
+    return check.result.text
+```
+
+## What it handles
+
+- Five actions (`allow`, `mask`, `suggest`, `warn`, `block`) and per-category policies
+- Profanity is masked deterministically. Hate and harassment rules are lower-confidence signals that context can weaken:
+  quoted or reported speech becomes `requires_review` instead of being acted on
+- Unicode (accents, full-width, Cyrillic/Greek lookalikes, zero-width), leet (`sh1t`), repeated letters, spaced letters
+- No Scunthorpe problem, and masking is idempotent
+- Custom words, custom rules, allow lists, mask styles (`vowels`, `partial`, `full`, or a function)
+
+## API
+
+```python
+engine = create_moderation_engine(
+    mode="mask", language="en", categories=None, policy=None, packs=None,
+    custom_words=None, allow_list=None, rules=None,
+    mask_char="*", mask_style="vowels", detect_spaced_letters=True,
+)
+engine.moderate(text)   # ModerationResult(text, suggested_text, matches, action, allowed, requires_review, mapping)
+engine.mask(text)       # str
+engine.validate(text)   # ValidationResult(valid, result, violations)
+engine.add_rule({"id": "app.x", "pattern": "(foo|bar*)", "category": "custom"})
+engine.remove_rule("app.x")
+engine.set_policy({"harassment": {"action": "block", "min_confidence": 0.7}})
+engine.set_mode("warn")
+
+safetype.moderate(text); safetype.mask(text)   # shared default engine
+```
+
+Offsets (`match.start` / `match.end`) are Python string indices (code points).
+
+## Parity with JavaScript
+
+The test suite replays 684 cases recorded from the JavaScript engine and requires identical results.
+The dictionary is exported from the JavaScript build, so both always share the same rules.
+
+## Limitations
+
+Rule-based detection misses novel spellings and sarcasm, and it can flag reclaimed or quoted language.
+Treat hate and harassment results as signals for review, not verdicts. Contextual (AI) classifiers
+are available in the JavaScript engine and planned for Python.
+
+MIT licensed.

safetype-1.5.0/README.md

@@ -0,0 +1,100 @@
+# safetype (Python)
+
+Rule-based content moderation for Python servers: profanity, hate speech and harassment.
+It's the **same engine and English dictionary** as the [SafeType.js](https://github.com/karammm/safetype)
+JavaScript packages, so your backend can re-check exactly what the browser checked.
+It has no dependencies and runs fully offline.
+
+```bash
+pip install safetype
+```
+
+```python
+from safetype import create_moderation_engine
+
+engine = create_moderation_engine(mode="mask")
+engine.moderate("What the fuck are you doing?").text
+# 'What the f*ck are you doing?'
+```
+
+## Server-side enforcement
+
+Client-side moderation can be bypassed (devtools, curl, modified clients). Re-check before you store:
+
+```python
+engine = create_moderation_engine(
+    mode="mask",
+    policy={"hate": {"action": "block"}, "harassment": {"action": "block"}},
+)
+
+check = engine.validate(comment)
+if not check.valid:
+    raise ValidationError([v.category for v in check.violations])
+store(check.result.text)                      # profanity masked
+if check.result.requires_review:
+    queue_for_human_review(comment)           # e.g. quoted or reported speech
+```
+
+FastAPI:
+
+```python
+@app.post("/comments")
+def create_comment(body: CommentIn):
+    check = engine.validate(body.text)
+    if not check.valid:
+        raise HTTPException(422, detail=[{"category": v.category, "rule": v.rule_id} for v in check.violations])
+    return save(check.result.text)
+```
+
+Django form:
+
+```python
+def clean_body(self):
+    check = engine.validate(self.cleaned_data["body"])
+    if not check.valid:
+        raise forms.ValidationError("Please keep it respectful.")
+    return check.result.text
+```
+
+## What it handles
+
+- Five actions (`allow`, `mask`, `suggest`, `warn`, `block`) and per-category policies
+- Profanity is masked deterministically. Hate and harassment rules are lower-confidence signals that context can weaken:
+  quoted or reported speech becomes `requires_review` instead of being acted on
+- Unicode (accents, full-width, Cyrillic/Greek lookalikes, zero-width), leet (`sh1t`), repeated letters, spaced letters
+- No Scunthorpe problem, and masking is idempotent
+- Custom words, custom rules, allow lists, mask styles (`vowels`, `partial`, `full`, or a function)
+
+## API
+
+```python
+engine = create_moderation_engine(
+    mode="mask", language="en", categories=None, policy=None, packs=None,
+    custom_words=None, allow_list=None, rules=None,
+    mask_char="*", mask_style="vowels", detect_spaced_letters=True,
+)
+engine.moderate(text)   # ModerationResult(text, suggested_text, matches, action, allowed, requires_review, mapping)
+engine.mask(text)       # str
+engine.validate(text)   # ValidationResult(valid, result, violations)
+engine.add_rule({"id": "app.x", "pattern": "(foo|bar*)", "category": "custom"})
+engine.remove_rule("app.x")
+engine.set_policy({"harassment": {"action": "block", "min_confidence": 0.7}})
+engine.set_mode("warn")
+
+safetype.moderate(text); safetype.mask(text)   # shared default engine
+```
+
+Offsets (`match.start` / `match.end`) are Python string indices (code points).
+
+## Parity with JavaScript
+
+The test suite replays 684 cases recorded from the JavaScript engine and requires identical results.
+The dictionary is exported from the JavaScript build, so both always share the same rules.
+
+## Limitations
+
+Rule-based detection misses novel spellings and sarcasm, and it can flag reclaimed or quoted language.
+Treat hate and harassment results as signals for review, not verdicts. Contextual (AI) classifiers
+are available in the JavaScript engine and planned for Python.
+
+MIT licensed.

safetype-1.5.0/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["hatchling>=1.24"]
+build-backend = "hatchling.build"
+
+[project]
+name = "safetype"
+version = "1.5.0"
+description = "Rule-based content moderation (profanity, hate, harassment) for Python servers. Same engine as SafeType.js."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.9"
+authors = [{ name = "SafeType.js contributors" }]
+keywords = ["moderation", "profanity", "filter", "content-safety", "hate-speech", "harassment"]
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Intended Audience :: Developers",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Topic :: Text Processing :: Filters",
+  "Typing :: Typed",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/karammm/safetype"
+Documentation = "https://github.com/karammm/safetype/tree/main/python#readme"
+Issues = "https://github.com/karammm/safetype/issues"
+Changelog = "https://github.com/karammm/safetype/blob/main/CHANGELOG.md"
+
+[dependency-groups]
+dev = ["pytest>=8"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/safetype"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/safetype", "tests", "README.md", "LICENSE"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

safetype-1.5.0/src/safetype/__init__.py

@@ -0,0 +1,70 @@
+"""SafeType: rule-based content moderation for Python servers.
+
+The same engine and English dictionary as the SafeType.js JavaScript packages, so the server can
+re-check exactly what the browser checked.
+
+    >>> from safetype import create_moderation_engine
+    >>> engine = create_moderation_engine(mode="mask")
+    >>> engine.moderate("What the fuck are you doing?").text
+    'What the f*ck are you doing?'
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from ._pattern import PatternError
+from ._transform import mask_text, match_case
+from .engine import ModerationEngine, create_moderation_engine, load_pack
+from .types import (
+    ModerationAction,
+    ModerationCategory,
+    ModerationMatch,
+    ModerationResult,
+    OffsetMapping,
+    RuleDict,
+    TextEdit,
+    ValidationResult,
+)
+
+__version__ = "1.5.0"
+
+_default: Optional[ModerationEngine] = None
+
+
+def _engine() -> ModerationEngine:
+    global _default
+    if _default is None:
+        _default = create_moderation_engine()
+    return _default
+
+
+def moderate(text: str) -> ModerationResult:
+    """Moderates with a shared default engine (``mode="mask"``, English)."""
+    return _engine().moderate(text)
+
+
+def mask(text: str) -> str:
+    """Masks with a shared default engine."""
+    return _engine().mask(text)
+
+
+__all__ = [
+    "ModerationAction",
+    "ModerationCategory",
+    "ModerationEngine",
+    "ModerationMatch",
+    "ModerationResult",
+    "OffsetMapping",
+    "PatternError",
+    "RuleDict",
+    "TextEdit",
+    "ValidationResult",
+    "__version__",
+    "create_moderation_engine",
+    "load_pack",
+    "mask",
+    "mask_text",
+    "match_case",
+    "moderate",
+]

safetype-1.5.0/src/safetype/_context.py

@@ -0,0 +1,46 @@
+"""Synchronous context heuristics (mirrors classifier.ts). They only lower confidence."""
+
+from __future__ import annotations
+
+from ._matcher import RawMatch
+from ._tokenize import Token
+
+CONTEXT_FACTOR = {"direct": 1.0, "quoted": 0.5, "mention": 0.4}
+
+_MENTION_WORDS = frozenset(
+    "word words term slur phrase say said saying says spell spelled called calling means meaning define definition using use".split()
+)
+
+_PAIRS = (('"', '"'), ("“", "”"), ("‘", "’"), ("«", "»"), ("「", "」"))
+
+
+def find_quoted_ranges(text: str) -> list[tuple[int, int]]:
+    ranges: list[tuple[int, int]] = []
+    for open_, close in _PAIRS:
+        start_from = 0
+        while start_from < len(text):
+            s = text.find(open_, start_from)
+            if s == -1:
+                break
+            e = text.find(close, s + 1)
+            if e == -1:
+                break
+            newline = text.find("\n", s)
+            if newline != -1 and newline < e:
+                start_from = newline + 1
+                continue
+            ranges.append((s + 1, e))
+            start_from = e + 1
+    return ranges
+
+
+def detect_context(match: RawMatch, tokens: list[Token], quoted: list[tuple[int, int]]) -> str:
+    if match.token_index is not None:
+        for k in (1, 2):
+            i = match.token_index - k
+            if i >= 0 and tokens[i].value in _MENTION_WORDS:
+                return "mention"
+    for s, e in quoted:
+        if match.start >= s and match.end <= e:
+            return "quoted"
+    return "direct"

safetype-1.5.0/src/safetype/_matcher.py

@@ -0,0 +1,176 @@
+"""Rule compilation, indexing and matching (mirrors matcher.ts)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+from ._normalize import NormalizedText
+from ._pattern import Alternative, Element, PatternError, Run, parse_pattern, runs_match, to_runs
+from ._tokenize import Token
+from .types import ALL_CATEGORIES, SEVERITY_RANK, RuleDict
+
+DEFAULT_PRIORITY = {"custom": 40, "hate": 30, "harassment": 20, "profanity": 10}
+
+
+@dataclass
+class CompiledRule:
+    id: str
+    category: str
+    severity: str
+    confidence: float
+    priority: int
+    replacement: Optional[str]
+    label: Optional[str]
+    boundary: str
+    elements: tuple[Element, ...]
+
+
+@dataclass
+class RawMatch:
+    start: int
+    end: int
+    rule: CompiledRule
+    confidence: float
+    token_index: Optional[int] = None
+    context: str = "direct"
+    source: Optional[str] = None
+
+
+def compile_rule(rule: RuleDict, boundary: str = "word") -> CompiledRule:
+    rule_id = rule.get("id")
+    if not isinstance(rule_id, str) or not rule_id:
+        raise TypeError("Rule requires a non-empty id")
+    if not isinstance(rule.get("pattern"), str):
+        raise TypeError(f'Rule "{rule_id}" requires a pattern')
+    category = rule.get("category")
+    if category not in ALL_CATEGORIES:
+        raise TypeError(f'Rule "{rule_id}" has unknown category "{category}"')
+    severity = rule.get("severity", "medium")
+    if severity not in SEVERITY_RANK:
+        raise TypeError(f'Rule "{rule_id}" has unknown severity "{severity}"')
+    confidence = rule.get("confidence", 1)
+    if not (0 <= confidence <= 1):
+        raise ValueError(f'Rule "{rule_id}" confidence must be within [0, 1]')
+    try:
+        elements = parse_pattern(rule["pattern"])
+    except PatternError as error:
+        raise PatternError(f'Rule "{rule_id}": {error}') from None
+    if boundary == "substring" and len(elements) != 1:
+        raise PatternError(f'Rule "{rule_id}": substring packs support single-token patterns only')
+    return CompiledRule(
+        id=rule_id,
+        category=category,
+        severity=severity,
+        confidence=confidence,
+        priority=rule.get("priority", DEFAULT_PRIORITY[category]),
+        replacement=rule.get("replacement"),
+        label=rule.get("label"),
+        boundary=boundary,
+        elements=elements,
+    )
+
+
+@dataclass
+class _Entry:
+    rule: CompiledRule
+    alt: Alternative
+
+
+@dataclass
+class RuleIndex:
+    exact: dict[str, list[_Entry]] = field(default_factory=dict)
+    prefix: dict[str, list[_Entry]] = field(default_factory=dict)
+    substring: list[CompiledRule] = field(default_factory=list)
+
+    @classmethod
+    def build(cls, rules) -> "RuleIndex":
+        index = cls()
+        for rule in rules:
+            if rule.boundary == "substring":
+                index.substring.append(rule)
+                continue
+            for alt in rule.elements[0].alternatives:
+                target = index.prefix if alt.prefix else index.exact
+                key = alt.key[0] if alt.prefix else alt.key
+                target.setdefault(key, []).append(_Entry(rule, alt))
+        return index
+
+    def candidates(self, runs: tuple[Run, ...], key: str) -> list[_Entry]:
+        out = [e for e in self.exact.get(key, []) if runs_match(runs, e.alt)]
+        out += [e for e in self.prefix.get(key[:1], []) if runs_match(runs, e.alt)]
+        return out
+
+
+@dataclass
+class _TokenInfo:
+    runs: tuple[Run, ...]
+    key: str
+    blocked: bool
+
+
+def _token_info(token: Token, allow_list: set[str]) -> _TokenInfo:
+    runs = to_runs(token.value)
+    return _TokenInfo(runs, "".join(c for c, _ in runs), token.masked or token.value in allow_list)
+
+
+def _element_matches(element: Element, info: _TokenInfo) -> bool:
+    return not info.blocked and any(runs_match(info.runs, alt) for alt in element.alternatives)
+
+
+def _match_rest(elements, e: int, infos, t: int) -> int:
+    """Index of the last token consumed, preferring the longest match, or -1."""
+    if e == len(elements):
+        return t - 1
+    element = elements[e]
+    best = -1
+    if t < len(infos) and _element_matches(element, infos[t]):
+        best = _match_rest(elements, e + 1, infos, t + 1)
+    if element.optional:
+        best = max(best, _match_rest(elements, e + 1, infos, t))
+    return best
+
+
+def find_matches(index: RuleIndex, normalized: NormalizedText, tokens: list[Token], spaced: list[Token], allow_list: set[str]) -> list[RawMatch]:
+    infos = [_token_info(t, allow_list) for t in tokens]
+    out: list[RawMatch] = []
+    seen: set[tuple[str, int, int]] = set()
+
+    def push(match: RawMatch) -> None:
+        key = (match.rule.id, match.start, match.end)
+        if key not in seen:
+            seen.add(key)
+            out.append(match)
+
+    for i, token in enumerate(tokens):
+        info = infos[i]
+        if info.blocked:
+            continue
+        seen_rules: set[str] = set()
+        for entry in index.candidates(info.runs, info.key):
+            rule = entry.rule
+            if rule.id in seen_rules:
+                continue
+            seen_rules.add(rule.id)
+            last = _match_rest(rule.elements, 1, infos, i + 1)
+            if last < i:
+                continue
+            push(RawMatch(token.start, tokens[last].end, rule, rule.confidence, token_index=i))
+
+    for token in spaced:
+        info = _token_info(token, allow_list)
+        if info.blocked:
+            continue
+        for entry in index.candidates(info.runs, info.key):
+            if len(entry.rule.elements) == 1:
+                push(RawMatch(token.start, token.end, entry.rule, entry.rule.confidence, token_index=token.first_index))
+
+    text = normalized.text
+    for rule in index.substring:
+        for alt in rule.elements[0].alternatives:
+            at = text.find(alt.value)
+            while at != -1:
+                n_end = at + len(alt.value)
+                push(RawMatch(normalized.starts[at], normalized.ends[n_end - 1], rule, rule.confidence))
+                at = text.find(alt.value, at + 1)
+    return out

safetype-1.5.0/src/safetype/_normalize.py

@@ -0,0 +1,65 @@
+"""Unicode-aware normalization with an offset map back to the original text (mirrors normalizer.ts)."""
+
+from __future__ import annotations
+
+import unicodedata
+from dataclasses import dataclass
+from functools import lru_cache
+
+_ZERO_WIDTH = frozenset("​‌‍⁠­᠎")
+
+# Cyrillic and Greek letters that look like Latin letters (lowercase forms).
+_CONFUSABLES = {
+    "а": "a", "в": "b", "е": "e", "ё": "e", "к": "k", "м": "m", "н": "h", "о": "o", "р": "p", "с": "c", "т": "t",
+    "у": "y", "х": "x", "ѕ": "s", "і": "i", "ї": "i", "ј": "j", "һ": "h", "ԁ": "d", "ԛ": "q", "ԝ": "w", "ү": "y",
+    "α": "a", "β": "b", "ε": "e", "η": "n", "ι": "i", "κ": "k", "ν": "v", "ο": "o", "ρ": "p", "τ": "t", "υ": "u",
+    "χ": "x", "ω": "w",
+}
+
+# Symbol/digit substitutions, applied only inside tokens that already contain a letter.
+LEET = {"@": "a", "4": "a", "$": "s", "5": "s", "0": "o", "1": "i", "!": "i", "|": "i", "3": "e", "€": "e", "7": "t"}
+
+
+@lru_cache(maxsize=10_000)
+def normalize_char(ch: str) -> str:
+    if ch in _ZERO_WIDTH:
+        return ""
+    decomposed = unicodedata.normalize("NFKD", ch.lower())
+    base = "".join(c for c in decomposed if not unicodedata.category(c).startswith("M")).lower()
+    return "".join(_CONFUSABLES.get(c, c) for c in base)
+
+
+@dataclass(frozen=True)
+class NormalizedText:
+    text: str
+    starts: list[int]
+    """Original start offset for each character of ``text``."""
+    ends: list[int]
+    """Original end offset (exclusive) for each character of ``text``."""
+
+
+def normalize(value: str) -> NormalizedText:
+    out: list[str] = []
+    starts: list[int] = []
+    ends: list[int] = []
+    for i, ch in enumerate(value):
+        n = normalize_char(ch)
+        for c in n:
+            out.append(c)
+            starts.append(i)
+            ends.append(i + 1)
+    return NormalizedText("".join(out), starts, ends)
+
+
+def is_letter(ch: str) -> bool:
+    return unicodedata.category(ch).startswith("L")
+
+
+def apply_leet(value: str) -> str:
+    if not any(is_letter(c) for c in value):
+        return value
+    return "".join(LEET.get(c, c) for c in value)
+
+
+def normalize_term(value: str) -> str:
+    return apply_leet(normalize(value).text)