edshield 0.2.0__py3-none-any.whl

edshield/__init__.py

@@ -0,0 +1,150 @@
+"""edshield — local-first student-privacy layer for AI in education.
+
+    from edshield import extract_pii, deidentify
+
+    r = extract_pii("Hi, I'm Maya Chen, my email is maya.c@gmail.com")
+    print([(e.label, e.text) for e in r.entities])
+
+    print(deidentify("...", policy="coppa").deidentified_text)
+
+Nothing here uses the network. With `edshield[hf]` installed and a model on
+disk, `extract_pii` adds neural detection of student names; without it the
+rule layer runs alone and says so with a warning.
+"""
+
+from __future__ import annotations
+
+import os
+import warnings
+from typing import Iterable, List, Optional
+
+from . import ner
+from .ner import ModelUnavailableError
+from .types import Entity, AnalysisResult, DeidResult, PIILO_LABELS, ALL_LABELS, MODEL_AUTHORITY_LABELS
+from .rules import detect_rules, resolve_overlaps, propagate_names
+from .deid import apply_deidentification, check_no_leak, available_policies, load_policy
+
+__version__ = "0.2.0"
+__all__ = [
+    "analyze_text",
+    "extract_pii",
+    "deidentify",
+    "check_no_leak",
+    "available_policies",
+    "load_policy",
+    "Entity",
+    "AnalysisResult",
+    "DeidResult",
+    "PIILO_LABELS",
+    "ALL_LABELS",
+    "MODEL_AUTHORITY_LABELS",
+    "ModelUnavailableError",
+]
+
+
+def _model_entities(
+    text: str, model_name: Optional[str], device: str, threshold: float, o_threshold: Optional[float]
+) -> Optional[List[Entity]]:
+    """Model detections, or None when no model ran. An empty list means it
+    ran and found nothing.
+
+    A model the caller asked for by name (or through EDSHIELD_MODEL) must
+    work: failure raises ModelUnavailableError. With no model named, the
+    default is tried and, if it is not available, the rules run alone and a
+    RuntimeWarning says so.
+    """
+    if model_name == "rules":
+        return None
+    requested = model_name or os.environ.get("EDSHIELD_MODEL")
+    try:
+        return ner.detect_model(
+            text, model_name=model_name, device=device, threshold=threshold, o_threshold=o_threshold
+        )
+    except Exception as exc:  # noqa: BLE001
+        if requested:
+            if isinstance(exc, ModelUnavailableError):
+                raise
+            raise ModelUnavailableError(f"model '{requested}' failed: {exc}") from exc
+        warnings.warn(
+            f"edshield: no model available ({exc}); running rules only, which misses most names. "
+            "Pass model_name='rules' to choose this on purpose.",
+            RuntimeWarning,
+            stacklevel=3,
+        )
+        return None
+
+
+def analyze_text(
+    text: str,
+    model_name: Optional[str] = None,
+    device: str = "cpu",
+    labels: Optional[Iterable[str]] = None,
+    threshold: float = 0.5,
+    use_rules: bool = True,
+    propagate: bool = True,
+    model_authority: Optional[Iterable[str]] = MODEL_AUTHORITY_LABELS,
+    o_threshold: Optional[float] = None,
+) -> AnalysisResult:
+    """Detect PII entities. Combines rule detectors with the model (if any).
+
+    When a model ran, the labels in `model_authority` come from the model
+    only and the rules are not run for them. Without a model the rules cover
+    every label. Pass `model_authority=()` to union both layers on all labels.
+
+    `o_threshold` switches the model from argmax to the recall-first rule
+    "entity whenever P(O) < o_threshold"; see `edshield.ner.decode`.
+    """
+    model_ents = _model_entities(text, model_name, device, threshold, o_threshold)
+    model_ran = model_ents is not None
+    model_ents = model_ents or []
+    if labels:
+        wanted = set(labels)
+        model_ents = [e for e in model_ents if e.label in wanted]
+
+    ents: List[Entity] = []
+    if use_rules:
+        rule_labels = list(labels) if labels else list(ALL_LABELS)
+        if model_ran:
+            deferred = set(model_authority or ())
+            rule_labels = [l for l in rule_labels if l not in deferred]
+        if rule_labels:
+            ents.extend(detect_rules(text, rule_labels))
+    ents.extend(model_ents)
+    if propagate:
+        ents = propagate_names(text, ents)
+    used = model_name if model_ran else "rules"
+    return AnalysisResult(text=text, entities=resolve_overlaps(ents), model_name=used)
+
+
+def extract_pii(text: str, model_name: Optional[str] = None, **kw) -> AnalysisResult:
+    """Alias for analyze_text restricted to PII labels."""
+    return analyze_text(text, model_name=model_name, labels=kw.pop("labels", ALL_LABELS), **kw)
+
+
+def deidentify(
+    text: str,
+    method: Optional[str] = None,
+    policy: str = "ferpa",
+    model_name: Optional[str] = None,
+    device: str = "cpu",
+    date_shift_days: Optional[int] = None,
+    seed: Optional[int] = None,
+    verify: bool = True,
+    o_threshold: Optional[float] = None,
+) -> DeidResult:
+    """Detect then transform. `method=None` uses the policy's default method.
+    `verify=True` raises if any acted-on value is still present verbatim.
+    With `o_threshold`, the policy's confidence floor applies to 1 - P(O)."""
+    res = analyze_text(text, model_name=model_name, device=device, o_threshold=o_threshold)
+    out = apply_deidentification(
+        text, res.entities, method=method, policy=policy, date_shift_days=date_shift_days, seed=seed
+    )
+    out.audit.update(
+        detector=res.model_name or ner.resolve_model_id(None),
+        model_authority=sorted(MODEL_AUTHORITY_LABELS) if res.model_name != "rules" else [],
+        o_threshold=o_threshold,
+        verified=verify,
+    )
+    if verify and out.leaks:
+        raise RuntimeError(f"De-identification leak detected: {out.leaks}")
+    return out

edshield/cli.py

@@ -0,0 +1,63 @@
+"""Command-line interface.
+
+    edshield extract essay.txt
+    edshield redact essay.txt --policy coppa --method mask
+    edshield serve --port 8080
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def _read(path: str) -> str:
+    return sys.stdin.read() if path == "-" else Path(path).read_text(encoding="utf-8")
+
+
+def main(argv=None) -> int:
+    p = argparse.ArgumentParser(prog="edshield", description="Local-first student-privacy layer")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    ex = sub.add_parser("extract", help="Detect PII and print entities as JSON")
+    ex.add_argument("path", help="file path or - for stdin")
+    ex.add_argument("--model", default=None, help="model short name, HF id, local dir, or 'rules'")
+    ex.add_argument("--device", default="cpu")
+
+    rd = sub.add_parser("redact", help="De-identify a document under a policy")
+    rd.add_argument("path")
+    rd.add_argument("--policy", default="ferpa")
+    rd.add_argument("--method", default=None, choices=["mask", "replace", "hash", "shift_dates"],
+                    help="default: the policy's own method")
+    rd.add_argument("--model", default=None)
+    rd.add_argument("--device", default="cpu")
+    rd.add_argument("--json", action="store_true", help="emit JSON instead of text")
+
+    sv = sub.add_parser("serve", help="Run the REST service")
+    sv.add_argument("--host", default="127.0.0.1")
+    sv.add_argument("--port", type=int, default=8080)
+
+    pl = sub.add_parser("policies", help="List built-in policies")
+
+    a = p.parse_args(argv)
+
+    if a.cmd == "extract":
+        from edshield import extract_pii
+        r = extract_pii(_read(a.path), model_name=a.model, device=a.device)
+        print(json.dumps(r.to_dict(), indent=2))
+    elif a.cmd == "redact":
+        from edshield import deidentify
+        r = deidentify(_read(a.path), method=a.method, policy=a.policy, model_name=a.model, device=a.device)
+        print(json.dumps(r.to_dict(), indent=2) if a.json else r.deidentified_text)
+    elif a.cmd == "serve":
+        import uvicorn
+        uvicorn.run("edshield.service:app", host=a.host, port=a.port)
+    elif a.cmd == "policies":
+        from edshield import available_policies
+        print("\n".join(available_policies()))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

edshield/deid.py

@@ -0,0 +1,241 @@
+"""De-identification: turn detected entities into masked, surrogate, hashed
+or date-shifted text under a named policy.
+
+Policies live in edshield/policies/*.yaml and say which labels to act on,
+which method to use per label, and the minimum confidence to honour.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import random
+import re
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import yaml
+
+from .types import Entity, DeidResult
+
+POLICY_DIR = Path(__file__).resolve().parent / "policies"
+
+
+def _policy_path(name_or_path: str) -> Path:
+    p = Path(name_or_path)
+    if not p.exists():
+        p = POLICY_DIR / f"{name_or_path}.yaml"
+    if not p.exists():
+        raise FileNotFoundError(f"Unknown policy '{name_or_path}'. Built-ins: {available_policies()}")
+    return p
+
+
+def load_policy(name_or_path: str) -> dict:
+    with open(_policy_path(name_or_path), encoding="utf-8") as fh:
+        return yaml.safe_load(fh)
+
+
+def policy_fingerprint(name_or_path: str) -> str:
+    """SHA-256 of the policy file, so an audit record pins the exact rules applied."""
+    text = _policy_path(name_or_path).read_text(encoding="utf-8").replace("\r\n", "\n")
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def available_policies() -> List[str]:
+    return sorted(p.stem for p in POLICY_DIR.glob("*.yaml"))
+
+
+# --- Surrogates -------------------------------------------------------------
+
+class SurrogateFactory:
+    """Consistent fake replacements: the same original value always maps to
+    the same surrogate within one document (or one session if reused)."""
+
+    def __init__(self, seed: Optional[int] = None, locale: str = "en_US"):
+        try:
+            from faker import Faker  # noqa: WPS433
+        except ImportError as exc:  # pragma: no cover
+            raise RuntimeError("`replace` needs `pip install faker`") from exc
+        self.fake = Faker(locale)
+        if seed is not None:
+            self.fake.seed_instance(seed)
+        self.rng = random.Random(seed)
+        self._memo: Dict[str, str] = {}
+
+    def for_entity(self, ent: Entity) -> str:
+        key = f"{ent.label}::{ent.text.strip().lower()}"
+        if key in self._memo:
+            return self._memo[key]
+        f = self.fake
+        n_words = len(ent.text.split())
+        if ent.label in ("NAME_STUDENT", "NAME_RELATED"):
+            val = f.first_name() if n_words == 1 else f"{f.first_name()} {f.last_name()}"
+        elif ent.label == "EMAIL":
+            val = f.free_email()
+        elif ent.label == "USERNAME":
+            val = f.user_name()
+        elif ent.label == "ID_NUM":
+            digits = sum(ch.isdigit() for ch in ent.text)
+            val = "".join(self.rng.choice("0123456789") for _ in range(max(6, digits)))
+        elif ent.label == "PHONE_NUM":
+            val = f.numerify("555-###-####")
+        elif ent.label == "URL_PERSONAL":
+            val = f"https://example.com/{f.user_name()}"
+        elif ent.label == "STREET_ADDRESS":
+            val = f.street_address()
+        elif ent.label == "SSN":
+            val = f.numerify("900-##-####")
+        elif ent.label == "DATE":
+            val = f.date(pattern="%m/%d/%Y")
+        else:
+            val = f"[{ent.label}]"
+        self._memo[key] = val
+        return val
+
+
+def _hash(value: str, salt: str, length: int = 10) -> str:
+    return hashlib.sha256((salt + value).encode("utf-8")).hexdigest()[:length]
+
+
+_DATE_FORMATS = ["%m/%d/%Y", "%m-%d-%Y", "%Y-%m-%d", "%B %d, %Y", "%b %d, %Y", "%B %d %Y"]
+
+
+def _shift_date(value: str, days: int) -> str:
+    for fmt in _DATE_FORMATS:
+        try:
+            d = datetime.strptime(value.replace("Sept", "Sep"), fmt)
+            return (d + timedelta(days=days)).strftime(fmt)
+        except ValueError:
+            continue
+    return "[DATE]"
+
+
+# --- Main entry -------------------------------------------------------------
+
+def apply_deidentification(
+    text: str,
+    entities: List[Entity],
+    method: Optional[str] = None,
+    policy: str = "ferpa",
+    date_shift_days: Optional[int] = None,
+    seed: Optional[int] = None,
+    salt: Optional[str] = None,
+) -> DeidResult:
+    pol = load_policy(policy)
+    rules: dict = pol.get("labels", {})
+    # The caller's method applies only if given and the policy allows it.
+    if method is None or not pol.get("allow_method_override", True):
+        method = pol.get("default_method", "mask")
+    min_conf = float(pol.get("min_confidence", 0.0))
+    salt = salt or os.environ.get("EDSHIELD_HASH_SALT", "edshield")
+    if date_shift_days == 0:
+        raise ValueError("date_shift_days=0 would leave every date unchanged; pass a non-zero shift or None")
+    if date_shift_days is None:
+        date_shift_days = int(pol.get("date_shift_days", 0))
+    if not date_shift_days:
+        rng = random.Random(seed)
+        date_shift_days = rng.choice([-1, 1]) * rng.randint(1, 365)
+
+    factory: Optional[SurrogateFactory] = None
+    replacements: Dict[str, str] = {}
+
+    def render(ent: Entity) -> str:
+        nonlocal factory
+        cfg = rules.get(ent.label, {})
+        # Precedence: an explicit per-label method in the policy always wins;
+        # otherwise the method resolved above.
+        if isinstance(cfg, dict) and cfg.get("method"):
+            m = cfg["method"]
+        else:
+            m = method
+        if m == "mask":
+            out = cfg.get("mask", f"[{ent.label}]") if isinstance(cfg, dict) else f"[{ent.label}]"
+        elif m == "replace":
+            if factory is None:
+                factory = SurrogateFactory(seed=seed)
+            out = factory.for_entity(ent)
+        elif m == "hash":
+            out = _hash(ent.text, salt)
+        elif m == "shift_dates":
+            out = _shift_date(ent.text, date_shift_days) if ent.label == "DATE" else f"[{ent.label}]"
+        elif m == "keep":
+            out = ent.text
+        else:
+            raise ValueError(f"Unknown method '{m}'")
+        replacements[ent.text] = out
+        return out
+
+    # Filter by policy: label must be enabled and confidence above threshold.
+    acted: List[Entity] = []
+    for e in entities:
+        cfg = rules.get(e.label)
+        enabled = cfg is not None and (cfg is True or (isinstance(cfg, dict) and cfg.get("enabled", True)))
+        label_min = float(cfg.get("min_confidence", min_conf)) if isinstance(cfg, dict) else min_conf
+        if enabled and e.confidence >= label_min:
+            acted.append(e)
+
+    acted.sort(key=lambda e: e.start)
+    out_parts: List[str] = []
+    untouched: List[str] = []
+    unchanged: List[str] = []
+    cursor = 0
+    for e in acted:
+        untouched.append(text[cursor:e.start])
+        out_parts.append(untouched[-1])
+        rendered = render(e)
+        if rendered == e.text and e.text not in unchanged:
+            unchanged.append(e.text)
+        out_parts.append(rendered)
+        cursor = e.end
+    untouched.append(text[cursor:])
+    out_parts.append(untouched[-1])
+    # Verify against the text that was left in place, not the output, so a
+    # surrogate or shifted date that happens to equal another original value
+    # is not mistaken for a leak.
+    leaks = unchanged + [v for v in check_no_leak("\n".join(untouched), acted) if v not in unchanged]
+    output = "".join(out_parts)
+    counts: Dict[str, int] = {}
+    for e in acted:
+        counts[e.label] = counts.get(e.label, 0) + 1
+    audit = {
+        "edshield_version": _version(),
+        "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        "policy": pol.get("name", policy),
+        "policy_sha256": policy_fingerprint(policy),
+        "method": method,
+        "entities_acted_on": len(acted),
+        "entities_below_policy_threshold": len(entities) - len(acted),
+        "by_label": dict(sorted(counts.items())),
+        "input_chars": len(text),
+        "output_sha256": hashlib.sha256(output.encode("utf-8")).hexdigest(),
+        "leaks_found": len(leaks),
+    }
+    return DeidResult(
+        original_text=text,
+        deidentified_text=output,
+        entities=acted,
+        method=method,
+        policy=policy,
+        replacements=replacements,
+        leaks=leaks,
+        audit=audit,
+    )
+
+
+def _version() -> str:
+    from . import __version__  # noqa: WPS433 - the package imports this module
+    return __version__
+
+
+def check_no_leak(deidentified_text: str, entities: List[Entity]) -> List[str]:
+    """Return original entity strings that still appear verbatim, as whole
+    words, in the text ("Ann" inside "Annual" is not a leak)."""
+    leaks = []
+    for e in entities:
+        value = e.text.strip()
+        if len(value) < 3 or value in leaks:
+            continue
+        if re.search(r"(?<!\w)" + re.escape(value) + r"(?!\w)", deidentified_text):
+            leaks.append(value)
+    return leaks

edshield/models.jsonl

@@ -0,0 +1,6 @@
+# One JSON object per line. `status`: planned | training | available.
+# "available" means published on the Hugging Face Hub as `hf_id`. `local_path`, relative to a clone of the repository, is used first when it exists.
+{"name": "piilo_deberta_small", "hf_id": "edshield/piilo-deberta-v3-small", "base": "microsoft/deberta-v3-small", "task": "token-classification", "labels": ["NAME_STUDENT","EMAIL","USERNAME","ID_NUM","PHONE_NUM","URL_PERSONAL","STREET_ADDRESS"], "params": "142M", "train_data": "PIILO/CRAPII (CC BY 4.0)", "status": "available", "local_path": "models/piilo-deberta-v3-small-v2", "eval": "eval/results/deberta_small_v2_validation.json"}
+{"name": "piilo_deberta_base", "hf_id": "edshield/piilo-deberta-v3-base", "base": "microsoft/deberta-v3-base", "task": "token-classification", "labels": ["NAME_STUDENT","EMAIL","USERNAME","ID_NUM","PHONE_NUM","URL_PERSONAL","STREET_ADDRESS"], "params": "184M", "train_data": "PIILO/CRAPII (CC BY 4.0)", "status": "planned", "local_path": "models/piilo-deberta-v3-base"}
+{"name": "piilo_modernbert_base", "hf_id": "edshield/piilo-modernbert-base", "base": "answerdotai/ModernBERT-base", "task": "token-classification", "labels": ["NAME_STUDENT","EMAIL","USERNAME","ID_NUM","PHONE_NUM","URL_PERSONAL","STREET_ADDRESS"], "params": "149M", "train_data": "PIILO/CRAPII (CC BY 4.0)", "status": "planned", "local_path": "models/piilo-modernbert-base"}
+{"name": "piilo_deberta_small_onnx", "hf_id": "edshield/piilo-deberta-v3-small-onnx", "base": "edshield/piilo-deberta-v3-small", "task": "token-classification", "runtime": ["onnxruntime", "transformers.js"], "quant": ["fp32","int8"], "status": "available", "local_path": "demo/models/piilo-deberta-v3-small-onnx"}