maskflow-core 0.1.0__tar.gz

maskflow_core-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+.env
+node_modules/
+dist/
+build/
+*.egg-info/
+.DS_Store
+.pytest_cache/
+.idea/

maskflow_core-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

maskflow_core-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: maskflow-core
+Version: 0.1.0
+Summary: PII detection and reversible masking engine for MaskFlow
+Requires-Python: >=3.9
+Requires-Dist: click>=8.0
+Requires-Dist: spacy<4,>=3.7
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'

maskflow_core-0.1.0/README.md

@@ -0,0 +1,53 @@
+# maskflow-core
+
+PII detection and reversible masking engine — the foundation for every other MaskFlow phase (CLI, GitHub Action, SDK, ...).
+
+Detects 12 PII types via a three-layer pipeline: regex + structural validation (Luhn for cards, mod-97 for IBANs) -> keyword-context confidence boosting -> spacy NER for names and dates. Overlapping matches are resolved by confidence, and detection returns non-overlapping `Finding`s sorted by position.
+
+## Install
+
+Requires [uv](https://docs.astral.sh/uv/).
+
+```bash
+cd core
+uv sync --extra dev
+uv run python -m spacy download en_core_web_sm
+```
+
+The spaCy model is a separate download rather than a pip dependency -- PyPI doesn't allow packages
+to declare a direct URL as a dependency, and this is the standard pattern for any spaCy-based
+package.
+
+## Usage
+
+```python
+from maskflow_core import detect, mask, unmask
+
+detect("Email me at alice@example.com or call 415-555-0132.")
+# [Finding(type=PIIType.EMAIL, value='alice@example.com', ...), Finding(type=PIIType.PHONE, ...)]
+
+result = mask("Email me at alice@example.com or call 415-555-0132.")
+result.masked_text  # "Email me at <EMAIL_1> or call <PHONE_1>."
+result.mapping       # {'<EMAIL_1>': 'alice@example.com', '<PHONE_1>': '415-555-0132'}
+
+unmask(result.masked_text, result.mapping)  # original text, restored
+```
+
+`mask`/`unmask` are pure functions — the engine never writes to disk or a database. Persisting the mapping is the caller's responsibility (the CLI and SDK phases handle that).
+
+## PII types (v1)
+
+Email, phone, SSN, credit card, IP address (v4/v6), AWS access key, API key / generic secret, JWT, IBAN, street address, person name, date of birth.
+
+## Tests
+
+```bash
+uv run pytest
+```
+
+`tests/fixtures/pii_samples.py` has 100+ labeled examples; `test_detection.py` enforces a 95% accuracy floor against them and checks that PII-free text produces zero findings.
+
+## Known limitations
+
+- spacy's NER occasionally flags a capitalized sentence-starter as a `PERSON_NAME` false positive (e.g. "Email me at..." -> "Email" tagged as a name). Not a training data issue — future phases add a `.maskflowrc.yml` exclude-config for exactly this kind of false positive.
+- Phone/address regexes are US-shaped; international formats are a future improvement.

maskflow_core-0.1.0/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "maskflow-core"
+version = "0.1.0"
+description = "PII detection and reversible masking engine for MaskFlow"
+requires-python = ">=3.9"
+dependencies = [
+    "spacy>=3.7,<4",
+    "click>=8.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+]
+
+[tool.uv]
+package = true
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/maskflow_core"]

maskflow_core-0.1.0/src/maskflow_core/__init__.py

@@ -0,0 +1,5 @@
+from .detection import detect
+from .entities import Finding, PIIType
+from .masking import MaskResult, mask, unmask
+
+__all__ = ["detect", "mask", "unmask", "Finding", "PIIType", "MaskResult"]

maskflow_core-0.1.0/src/maskflow_core/context.py

@@ -0,0 +1,36 @@
+"""Keyword-proximity confidence boosting.
+
+Structural matches (email, credit card w/ Luhn, AWS keys, ...) are already
+high-confidence on their own. Ambiguous matches (a bare 9-digit number, a
+street-shaped line of text) need a nearby keyword to be trusted.
+"""
+from .entities import PIIType
+
+WINDOW = 40
+
+CONTEXT_KEYWORDS: dict[PIIType, tuple[str, ...]] = {
+    PIIType.SSN: ("ssn", "social security"),
+    PIIType.DATE_OF_BIRTH: ("dob", "date of birth", "born on", "birthdate"),
+    PIIType.ADDRESS: ("address", "lives at", "located at", "ship to", "mailing"),
+    PIIType.API_KEY: ("api key", "apikey", "secret", "token", "credential"),
+    PIIType.PHONE: ("phone", "call", "tel", "mobile", "cell", "contact"),
+    PIIType.CREDIT_CARD: ("card number", "credit card", "visa", "mastercard", "cc#"),
+    PIIType.PERSON_NAME: ("name is", "name:", "my name", "signed", "regards"),
+}
+
+BOOST = 0.4
+MAX_CONFIDENCE = 0.99
+
+
+def apply_context_boost(text: str, start: int, end: int, pii_type: PIIType, base_confidence: float) -> float:
+    keywords = CONTEXT_KEYWORDS.get(pii_type)
+    if not keywords:
+        return base_confidence
+
+    window_start = max(0, start - WINDOW)
+    window_end = min(len(text), end + WINDOW)
+    nearby = text[window_start:start].lower() + " " + text[end:window_end].lower()
+
+    if any(keyword in nearby for keyword in keywords):
+        return min(MAX_CONFIDENCE, base_confidence + BOOST)
+    return base_confidence

maskflow_core-0.1.0/src/maskflow_core/detection.py

@@ -0,0 +1,55 @@
+"""Orchestrates the regex, context, and NER passes into one deduplicated finding list."""
+from .context import apply_context_boost
+from .entities import Finding, PIIType
+from .ner import detect_ner
+from .patterns import PATTERNS
+
+DEFAULT_MIN_CONFIDENCE = 0.5
+
+
+def _regex_pass(text: str) -> list[Finding]:
+    candidates: list[Finding] = []
+
+    for pii_type, rules in PATTERNS.items():
+        for regex, base_confidence, validator in rules:
+            for match in regex.finditer(text):
+                if match.re.groups:
+                    start, end = match.span(1)
+                    value = match.group(1)
+                else:
+                    start, end = match.span(0)
+                    value = match.group(0)
+
+                confidence = base_confidence
+                if validator is not None:
+                    adjusted = validator(value)
+                    if adjusted is None:
+                        continue
+                    confidence = adjusted
+
+                confidence = apply_context_boost(text, start, end, pii_type, confidence)
+                candidates.append(Finding(pii_type, value, start, end, round(confidence, 2)))
+
+    return candidates
+
+
+def _spans_overlap(a: Finding, b: Finding) -> bool:
+    return a.start < b.end and b.start < a.end
+
+
+def _merge_overlaps(candidates: list[Finding]) -> list[Finding]:
+    accepted: list[Finding] = []
+    for finding in sorted(candidates, key=lambda f: f.confidence, reverse=True):
+        if not any(_spans_overlap(finding, kept) for kept in accepted):
+            accepted.append(finding)
+    return sorted(accepted, key=lambda f: f.start)
+
+
+def detect(text: str, min_confidence: float = DEFAULT_MIN_CONFIDENCE) -> list[Finding]:
+    """Detect PII in `text`, returning non-overlapping Findings sorted by position."""
+    candidates = _regex_pass(text) + detect_ner(text)
+    merged = _merge_overlaps(candidates)
+    return [f for f in merged if f.confidence >= min_confidence]
+
+
+__all__ = ["detect", "Finding", "PIIType"]

maskflow_core-0.1.0/src/maskflow_core/entities.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+class PIIType(str, Enum):
+    EMAIL = "EMAIL"
+    PHONE = "PHONE"
+    SSN = "SSN"
+    CREDIT_CARD = "CREDIT_CARD"
+    IP_ADDRESS = "IP_ADDRESS"
+    AWS_KEY = "AWS_KEY"
+    API_KEY = "API_KEY"
+    JWT = "JWT"
+    IBAN = "IBAN"
+    ADDRESS = "ADDRESS"
+    PERSON_NAME = "PERSON_NAME"
+    DATE_OF_BIRTH = "DATE_OF_BIRTH"
+
+
+@dataclass(frozen=True)
+class Finding:
+    type: PIIType
+    value: str
+    start: int
+    end: int
+    confidence: float
+
+    @property
+    def span(self) -> tuple[int, int]:
+        return (self.start, self.end)

maskflow_core-0.1.0/src/maskflow_core/masking.py

@@ -0,0 +1,39 @@
+"""Pure, stateless mask/unmask. No files, no DB -- the caller owns persistence of the mapping."""
+from typing import NamedTuple
+
+from .detection import DEFAULT_MIN_CONFIDENCE, detect
+
+
+class MaskResult(NamedTuple):
+    masked_text: str
+    mapping: dict[str, str]
+
+
+def mask(text: str, min_confidence: float = DEFAULT_MIN_CONFIDENCE) -> MaskResult:
+    """Replace detected PII with `<TYPE_n>` tokens, returning the masked text and a
+    {token: original_value} mapping the caller can use to unmask a later response."""
+    findings = detect(text, min_confidence=min_confidence)
+
+    mapping: dict[str, str] = {}
+    counters: dict[str, int] = {}
+    pieces: list[str] = []
+    cursor = 0
+
+    for finding in findings:  # detect() returns non-overlapping findings sorted by start
+        counters[finding.type.value] = counters.get(finding.type.value, 0) + 1
+        token = f"<{finding.type.value}_{counters[finding.type.value]}>"
+        mapping[token] = finding.value
+        pieces.append(text[cursor:finding.start])
+        pieces.append(token)
+        cursor = finding.end
+
+    pieces.append(text[cursor:])
+    return MaskResult("".join(pieces), mapping)
+
+
+def unmask(masked_text: str, mapping: dict[str, str]) -> str:
+    """Restore original values from a mapping produced by `mask()`."""
+    result = masked_text
+    for token, original in mapping.items():
+        result = result.replace(token, original)
+    return result

maskflow_core-0.1.0/src/maskflow_core/ner.py

@@ -0,0 +1,49 @@
+"""spacy-based detection for entity types regex can't reliably catch: names and dates."""
+from functools import lru_cache
+
+import spacy
+
+from .context import apply_context_boost
+from .entities import Finding, PIIType
+
+MODEL_NAME = "en_core_web_sm"
+
+PERSON_BASE_CONFIDENCE = 0.75
+DATE_BASE_CONFIDENCE = 0.3
+DATE_OF_BIRTH_THRESHOLD = 0.6
+
+
+@lru_cache(maxsize=1)
+def _get_nlp():
+    try:
+        return spacy.load(MODEL_NAME)
+    except OSError as exc:
+        raise OSError(
+            f"spaCy model '{MODEL_NAME}' isn't installed. Run: "
+            f"python -m spacy download {MODEL_NAME}"
+        ) from exc
+
+
+def detect_ner(text: str) -> list[Finding]:
+    nlp = _get_nlp()
+    doc = nlp(text)
+    findings: list[Finding] = []
+
+    for ent in doc.ents:
+        if ent.label_ == "PERSON":
+            confidence = apply_context_boost(
+                text, ent.start_char, ent.end_char, PIIType.PERSON_NAME, PERSON_BASE_CONFIDENCE
+            )
+            findings.append(
+                Finding(PIIType.PERSON_NAME, ent.text, ent.start_char, ent.end_char, confidence)
+            )
+        elif ent.label_ == "DATE":
+            confidence = apply_context_boost(
+                text, ent.start_char, ent.end_char, PIIType.DATE_OF_BIRTH, DATE_BASE_CONFIDENCE
+            )
+            if confidence >= DATE_OF_BIRTH_THRESHOLD:
+                findings.append(
+                    Finding(PIIType.DATE_OF_BIRTH, ent.text, ent.start_char, ent.end_char, confidence)
+                )
+
+    return findings

maskflow_core-0.1.0/src/maskflow_core/patterns.py

@@ -0,0 +1,123 @@
+"""Regex patterns and structural validators for each PII type.
+
+Each entry in PATTERNS maps a PIIType to (compiled_regex, base_confidence, validator).
+`validator(value) -> float | None` returns an adjusted confidence, or None to reject
+the match entirely (e.g. a 16-digit number that fails the Luhn check).
+"""
+import re
+
+from .entities import PIIType
+
+EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
+
+PHONE_RE = re.compile(
+    r"(?<!\d)(?:\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}(?!\d)"
+)
+
+SSN_DASHED_RE = re.compile(r"(?<!\d)\d{3}-\d{2}-\d{4}(?!\d)")
+SSN_PLAIN_RE = re.compile(r"(?<!\d)\d{9}(?!\d)")
+
+CREDIT_CARD_RE = re.compile(
+    r"(?<!\d)(?:\d[ -]?){13,19}(?!\d)"
+)
+
+IPV4_RE = re.compile(
+    r"\b(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\b"
+)
+IPV6_RE = re.compile(r"\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\b")
+
+AWS_KEY_RE = re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")
+
+API_KEY_RE = re.compile(
+    r"\b(?:sk-ant-[A-Za-z0-9_-]{20,}"
+    r"|sk-[A-Za-z0-9_-]{20,}"
+    r"|gh[pousr]_[A-Za-z0-9]{20,}"
+    r"|xox[baprs]-[A-Za-z0-9-]{10,}"
+    r"|AIza[A-Za-z0-9_-]{35})\b"
+)
+
+GENERIC_SECRET_ASSIGNMENT_RE = re.compile(
+    r"(?i)\b\w*(?:key|secret|token|password|credential)\w*"
+    r"\s*[:=]\s*['\"]?([A-Za-z0-9_\-/+]{16,})['\"]?"
+)
+
+JWT_RE = re.compile(
+    r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b"
+)
+
+IBAN_RE = re.compile(r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b")
+
+ADDRESS_RE = re.compile(
+    r"\b\d{1,6}\s+(?:[A-Z][a-zA-Z]*\s){1,4}"
+    r"(?:Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd|Lane|Ln|"
+    r"Drive|Dr|Court|Ct|Way|Place|Pl|Terrace|Ter)\.?\b"
+    r"(?:,?\s+(?:Apt|Suite|Ste|Unit)\.?\s*#?\w+)?"
+)
+
+
+def luhn_is_valid(digits: str) -> bool:
+    digits = [int(d) for d in digits]
+    checksum = 0
+    parity = len(digits) % 2
+    for i, d in enumerate(digits):
+        if i % 2 == parity:
+            d *= 2
+            if d > 9:
+                d -= 9
+        checksum += d
+    return checksum % 10 == 0
+
+
+def _validate_credit_card(value: str) -> float | None:
+    digits = re.sub(r"[ -]", "", value)
+    if not (13 <= len(digits) <= 19) or not digits.isdigit():
+        return None
+    return 0.97 if luhn_is_valid(digits) else None
+
+
+def iban_is_valid(value: str) -> bool:
+    value = value.upper()
+    rearranged = value[4:] + value[:4]
+    converted = "".join(str(int(c, 36)) for c in rearranged)
+    try:
+        return int(converted) % 97 == 1
+    except ValueError:
+        return False
+
+
+def _validate_iban(value: str) -> float | None:
+    return 0.9 if iban_is_valid(value) else None
+
+
+def _validate_ssn_dashed(value: str) -> float | None:
+    area = value[:3]
+    if area in ("000", "666") or area.startswith("9"):
+        return None
+    return 0.95
+
+
+def _validate_ssn_plain(value: str) -> float | None:
+    # Bare 9-digit numbers are ambiguous (order IDs, phone numbers without
+    # formatting, etc.) -- start low, let context.py decide if it's really an SSN.
+    return 0.35
+
+
+# type -> (regex, base_confidence, validator)
+PATTERNS: dict[PIIType, list[tuple[re.Pattern, float, "callable | None"]]] = {
+    PIIType.EMAIL: [(EMAIL_RE, 0.95, None)],
+    PIIType.PHONE: [(PHONE_RE, 0.85, None)],
+    PIIType.SSN: [
+        (SSN_DASHED_RE, 0.95, _validate_ssn_dashed),
+        (SSN_PLAIN_RE, 0.35, _validate_ssn_plain),
+    ],
+    PIIType.CREDIT_CARD: [(CREDIT_CARD_RE, 0.9, _validate_credit_card)],
+    PIIType.IP_ADDRESS: [(IPV4_RE, 0.75, None), (IPV6_RE, 0.85, None)],
+    PIIType.AWS_KEY: [(AWS_KEY_RE, 0.97, None)],
+    PIIType.API_KEY: [
+        (API_KEY_RE, 0.95, None),
+        (GENERIC_SECRET_ASSIGNMENT_RE, 0.6, None),
+    ],
+    PIIType.JWT: [(JWT_RE, 0.9, None)],
+    PIIType.IBAN: [(IBAN_RE, 0.6, _validate_iban)],
+    PIIType.ADDRESS: [(ADDRESS_RE, 0.7, None)],
+}