privyscope 0.1.0__py3-none-any.whl

privyscope/__init__.py

@@ -0,0 +1,30 @@
+"""privyscope — multilingual PII detection & masking engine (Apache-2.0).
+
+This is the language-neutral **core**: the two-stage hybrid engine (regex filter
++ ONNX BIOES/Viterbi NER), the ``privyscope`` console command, and the plugin
+machinery that discovers installed language packs (``privyscope-ko``,
+``privyscope-en``, …). Install at least one language plugin to do real work::
+
+    pip install privyscope-ko            # pulls in this core automatically
+
+Public surface::
+
+    from privyscope import Privyscope
+    engine = Privyscope.from_pretrained(lang="ko")   # one language
+    auto   = Privyscope.auto()                        # route per text
+"""
+from ._api import Privyscope
+from ._core.schema import RedactionResult, DetectedSpan, SCHEMA_VERSION
+from ._core.plugins import LanguagePlugin, installed_languages
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Privyscope",
+    "RedactionResult",
+    "DetectedSpan",
+    "SCHEMA_VERSION",
+    "LanguagePlugin",
+    "installed_languages",
+    "__version__",
+]

privyscope/__main__.py

@@ -0,0 +1,130 @@
+"""Unified CLI entrypoint: ``privyscope redact | eval | train`` (SRS FR-2.1)."""
+from __future__ import annotations
+
+import sys
+from typing import List, Optional, Sequence
+
+from ._api import AutoPrivyscope, Privyscope
+from ._cli.args import build_parser
+from ._cli.render import print_result, supports_color
+from ._core.plugins import installed_languages, resolve_plugin
+
+_PROVIDERS = {
+    "cpu": ["CPUExecutionProvider"],
+    "cuda": ["CUDAExecutionProvider", "CPUExecutionProvider"],
+    "coreml": ["CoreMLExecutionProvider", "CPUExecutionProvider"],
+}
+
+
+def _parse_operating_point(value: str):
+    """A preset name, or comma-separated raw transition biases (FR-2.5)."""
+    if value and "," in value:
+        return [float(x) for x in value.split(",")]
+    return value
+
+
+def _read_input(args) -> Optional[str]:
+    """Resolve redact input from arg / file / stdin; None triggers interactive mode."""
+    if args.text is not None:
+        return args.text
+    if args.file:
+        with open(args.file, "r", encoding="utf-8") as fh:
+            return fh.read().rstrip("\n")
+    if not sys.stdin.isatty():
+        data = sys.stdin.read()
+        return data.rstrip("\n") if data else None
+    return None
+
+
+def _build_engine(args):
+    """Build the inference engine for redact/eval.
+
+    ``--lang X`` → that language. No ``--lang`` with one plugin installed → that
+    plugin. No ``--lang`` with several installed → an auto-routing dispatcher.
+    Returns an ``Privyscope`` or an ``AutoPrivyscope`` (same redact surface).
+    """
+    operating_point = _parse_operating_point(getattr(args, "operating_point", "balanced"))
+    cache_dir = getattr(args, "cache_dir", None)
+    regex_only = getattr(args, "regex_only", False)
+    providers = _PROVIDERS.get(getattr(args, "device", "cpu"), None)
+    lang = getattr(args, "lang", None)
+
+    if lang is None and len(installed_languages()) > 1:
+        return AutoPrivyscope(
+            operating_point=operating_point,
+            cache_dir=cache_dir,
+            providers=providers,
+            regex_only=regex_only,
+        )
+    return Privyscope.from_pretrained(
+        operating_point=operating_point,
+        lang=lang,
+        cache_dir=cache_dir,
+        revision=getattr(args, "revision", None),
+        regex_only=regex_only,
+        providers=providers,
+    )
+
+
+def _cmd_redact(args) -> int:
+    engine = _build_engine(args)
+    entity_types = args.entity_types.split(",") if args.entity_types else None
+    text = _read_input(args)
+    as_json = args.json or not supports_color()
+
+    if text is None:  # interactive mode (FR-2.1)
+        print("privyscope interactive redact — enter text (Ctrl-D to exit)", file=sys.stderr)
+        for line in sys.stdin:
+            line = line.rstrip("\n")
+            if not line:
+                continue
+            print_result(engine.redact(line, entity_types, args.output_mode), as_json)
+        return 0
+
+    print_result(engine.redact(text, entity_types, args.output_mode), as_json)
+    return 0
+
+
+def _cmd_eval(args) -> int:
+    from ._eval.runner import run_eval
+
+    engine = _build_engine(args)
+    report = run_eval(engine, args.dataset, eval_mode=args.eval_mode, limit=args.limit)
+    import json
+
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    return 0
+
+
+def _cmd_train(args) -> int:
+    try:
+        from ._train.runner import run_train
+    except ImportError as exc:
+        print(
+            f"training requires the optional extra: pip install 'privyscope[train]' ({exc})",
+            file=sys.stderr,
+        )
+        return 2
+    # Default the base checkpoint from the selected language plugin (FR-2.6).
+    if not getattr(args, "base_model", None):
+        plugin = resolve_plugin(getattr(args, "lang", None))
+        if plugin.default_base_model:
+            args.base_model = plugin.default_base_model
+    summary = run_train(args)
+    import json
+
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+    return 0
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    args = build_parser().parse_args(argv)
+    return {
+        "redact": _cmd_redact,
+        "eval": _cmd_eval,
+        "train": _cmd_train,
+    }[args.command](args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

privyscope/_api.py

@@ -0,0 +1,258 @@
+"""Public Python API (SRS FR-2.7).
+
+    from privyscope import Privyscope            # or: from privyscope_ko import Privyscope
+
+    # one language (explicit, or the sole one installed)
+    engine = Privyscope.from_pretrained(lang="ko")
+    result = engine.redact("홍길동의 전화번호는 010-1234-5678")
+    result.masked_text      # "<PER>의 전화번호는 <PHONE>"
+
+    # multiple languages installed → route each text automatically
+    auto = Privyscope.auto()
+    auto.redact("홍길동 010-1234-5678")          # → Korean engine
+    auto.redact("John Smith 555-123-4567")       # → English engine
+
+The engine is a two-stage hybrid pipeline (SRS §3.4): a regex filter (always
+available) plus an optional ONNX NER stage. ``regex_only=True`` runs the engine
+with no model weights — useful offline and for structurally-obvious PII.
+
+Language-specific data (regex rules, entity config, default weights repo) comes
+from an installed language plugin discovered via the ``privyscope.languages``
+entry-point group; the core hardcodes nothing language-specific.
+"""
+from __future__ import annotations
+
+import os
+import threading
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Sequence, Union
+
+from ._core.bioes import Span
+from ._core.detect import detect_language
+from ._core.download import resolve_weights
+from ._core.pipeline import assemble, merge_union
+from ._core.plugins import LanguagePlugin, installed_languages, resolve_plugin
+from ._core.regex_filter import RegexFilter
+from ._core.schema import RedactionResult
+from ._core.transitions import CATEGORIES, resolve_operating_point
+
+OperatingPoint = Union[str, Sequence[float], dict, None]
+
+
+def _regex_rules_path(plugin: LanguagePlugin) -> Path:
+    """Locate the active language's regex ruleset (env override wins)."""
+    env = os.environ.get("PRIVYSCOPE_REGEX_RULES")
+    if env:
+        return Path(env)
+    return plugin.regex_rules_path()
+
+
+class Privyscope:
+    """Two-stage PII redaction engine for a single language."""
+
+    def __init__(
+        self,
+        regex_filter: RegexFilter,
+        runtime=None,
+        operating_point: OperatingPoint = "balanced",
+    ) -> None:
+        self._regex = regex_filter
+        self._runtime = runtime
+        self._operating_point = operating_point
+
+    # -- construction -------------------------------------------------------
+    @classmethod
+    def from_pretrained(
+        cls,
+        operating_point: OperatingPoint = "balanced",
+        *,
+        lang: Optional[str] = None,
+        repo_id: Optional[str] = None,
+        revision: Optional[str] = None,
+        cache_dir: Optional[str] = None,
+        regex_rules: Optional[str] = None,
+        providers: Optional[Sequence[str]] = None,
+        regex_only: bool = False,
+    ) -> "Privyscope":
+        """Load the engine for one language, fetching ONNX weights on first use.
+
+        ``lang`` selects the language plugin; if omitted, the sole installed
+        plugin is used (raises if several are installed — pass ``lang`` or use
+        :meth:`auto`). Pass ``regex_only=True`` to skip the NER stage (no weights
+        required). ``cache_dir`` pointing at a local bundle enables offline use.
+        """
+        plugin = resolve_plugin(lang)
+        rules_path = regex_rules or _regex_rules_path(plugin)
+        regex = RegexFilter.from_yaml(rules_path)
+
+        runtime = None
+        if not regex_only:
+            from ._core.runtime import NerRuntime
+
+            repo = repo_id or plugin.default_repo
+            bundle = resolve_weights(repo_id=repo, revision=revision, cache_dir=cache_dir)
+            runtime = NerRuntime(bundle, providers=providers)
+        return cls(regex, runtime, operating_point)
+
+    @classmethod
+    def regex_only(
+        cls, *, lang: Optional[str] = None, regex_rules: Optional[str] = None
+    ) -> "Privyscope":
+        """Convenience constructor for a Stage-1-only engine (no weights)."""
+        return cls.from_pretrained(regex_only=True, lang=lang, regex_rules=regex_rules)
+
+    @classmethod
+    def auto(
+        cls,
+        operating_point: OperatingPoint = "balanced",
+        *,
+        cache_dir: Optional[str] = None,
+        providers: Optional[Sequence[str]] = None,
+        regex_only: bool = False,
+    ) -> "AutoPrivyscope":
+        """Return a dispatcher that routes each text to its language engine.
+
+        Languages are detected per text (Unicode-script heuristic) and restricted
+        to the installed plugins. Per-language engines are built lazily and cached.
+        """
+        return AutoPrivyscope(
+            operating_point=operating_point,
+            cache_dir=cache_dir,
+            providers=providers,
+            regex_only=regex_only,
+        )
+
+    # -- inference ----------------------------------------------------------
+    def _effective_biases(self, operating_point: OperatingPoint) -> List[float]:
+        """Learned baseline biases shifted by the operating-point offset (FR-2.5)."""
+        offset = resolve_operating_point(
+            operating_point if operating_point is not None else self._operating_point
+        )
+        base = self._runtime.default_biases if self._runtime else [0.0] * len(CATEGORIES)
+        return [b + o for b, o in zip(base, offset)]
+
+    def redact(
+        self,
+        text: str,
+        entity_types: Optional[Iterable[str]] = None,
+        output_mode: str = "typed",
+        operating_point: OperatingPoint = None,
+    ) -> RedactionResult:
+        """Detect and mask PII in a single string (FR-2.2)."""
+        regex_spans: List[Span] = self._regex.find(text)
+        ner_spans: List[Span] = []
+        mismatch = False
+        if self._runtime is not None:
+            ner_spans, mismatch = self._runtime.predict(text, self._effective_biases(operating_point))
+        merged = merge_union(regex_spans, ner_spans)
+        return assemble(text, merged, output_mode, entity_types, mismatch)
+
+    def batch_redact(
+        self,
+        texts: Sequence[str],
+        entity_types: Optional[Iterable[str]] = None,
+        output_mode: str = "typed",
+        operating_point: OperatingPoint = None,
+    ) -> List[RedactionResult]:
+        return [self.redact(t, entity_types, output_mode, operating_point) for t in texts]
+
+    @property
+    def has_ner(self) -> bool:
+        return self._runtime is not None
+
+
+class AutoPrivyscope:
+    """Multi-language dispatcher (see :meth:`Privyscope.auto`).
+
+    Mirrors the :class:`Privyscope` inference surface (``redact`` /
+    ``batch_redact`` / ``has_ner``) and routes each call to a lazily-built,
+    cached per-language engine chosen by :func:`detect_language`.
+
+    **Thread-safety:** per-language engine construction is guarded by a lock, so
+    concurrent callers (e.g. a multi-threaded web server) never build or download
+    the same engine twice. Once built, ONNX Runtime sessions are safe for
+    concurrent inference. Call :meth:`preload` at startup to build engines
+    eagerly and avoid a first-request latency spike.
+    """
+
+    def __init__(
+        self,
+        *,
+        operating_point: OperatingPoint,
+        cache_dir: Optional[str],
+        providers: Optional[Sequence[str]],
+        regex_only: bool,
+    ) -> None:
+        self._available = sorted(installed_languages())
+        if not self._available:
+            raise ValueError(
+                "no privyscope language plugin installed; install one, "
+                "e.g. pip install privyscope-ko"
+            )
+        self._kwargs = dict(
+            operating_point=operating_point,
+            cache_dir=cache_dir,
+            providers=providers,
+            regex_only=regex_only,
+        )
+        self._regex_only = regex_only
+        self._engines: Dict[str, Privyscope] = {}
+        self._lock = threading.Lock()
+
+    @property
+    def available_languages(self) -> List[str]:
+        return list(self._available)
+
+    def _engine_for(self, lang: str) -> Privyscope:
+        # Fast path: already built (dict reads are atomic under the GIL).
+        engine = self._engines.get(lang)
+        if engine is not None:
+            return engine
+        # Slow path: build under lock with a double-check so concurrent callers
+        # for the same language share one engine (and one weights download).
+        with self._lock:
+            engine = self._engines.get(lang)
+            if engine is None:
+                engine = Privyscope.from_pretrained(lang=lang, **self._kwargs)
+                self._engines[lang] = engine
+            return engine
+
+    def preload(self, langs: Optional[Iterable[str]] = None) -> "AutoPrivyscope":
+        """Eagerly build (and cache) engines so the first request is warm.
+
+        Builds every installed language by default, or just ``langs``. Returns
+        ``self`` for chaining, e.g. ``engine = Privyscope.auto().preload()``.
+        Ideal at web-server startup: fail fast on missing weights and avoid a
+        cold-start spike on the first redact call.
+        """
+        for lang in (langs if langs is not None else self._available):
+            self._engine_for(lang)
+        return self
+
+    def route(self, text: str) -> str:
+        """Return the language code chosen for ``text``."""
+        return detect_language(text, self._available) or self._available[0]
+
+    def redact(
+        self,
+        text: str,
+        entity_types: Optional[Iterable[str]] = None,
+        output_mode: str = "typed",
+        operating_point: OperatingPoint = None,
+    ) -> RedactionResult:
+        return self._engine_for(self.route(text)).redact(
+            text, entity_types, output_mode, operating_point
+        )
+
+    def batch_redact(
+        self,
+        texts: Sequence[str],
+        entity_types: Optional[Iterable[str]] = None,
+        output_mode: str = "typed",
+        operating_point: OperatingPoint = None,
+    ) -> List[RedactionResult]:
+        return [self.redact(t, entity_types, output_mode, operating_point) for t in texts]
+
+    @property
+    def has_ner(self) -> bool:
+        return not self._regex_only

privyscope/_cli/__init__.py

@@ -0,0 +1,4 @@
+"""CLI argument parsing and terminal rendering."""
+from .args import build_parser
+
+__all__ = ["build_parser"]

privyscope/_cli/args.py

@@ -0,0 +1,75 @@
+"""CLI argument parsing (SRS FR-2.1)."""
+from __future__ import annotations
+
+import argparse
+
+from .. import __version__
+
+
+def _version_string() -> str:
+    """Core version plus the language plugins discovered at runtime."""
+    from .._core.plugins import installed_languages
+
+    langs = installed_languages()
+    if langs:
+        detail = ", ".join(f"{c} ({p.display_name})" for c, p in sorted(langs.items()))
+    else:
+        detail = "none — install one, e.g. pip install privyscope-ko"
+    return f"privyscope {__version__} (languages: {detail})"
+
+
+def _add_lang(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--lang",
+        help="language code (e.g. ko, en); omit to auto-detect per text when "
+        "several are installed, or use the sole installed language",
+    )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="privyscope",
+        description="privyscope — multilingual PII detection & masking (redact / eval / train)",
+    )
+    p.add_argument("--version", action="version", version=_version_string())
+    sub = p.add_subparsers(dest="command", required=True)
+
+    # -- redact -------------------------------------------------------------
+    r = sub.add_parser("redact", help="detect and mask PII in text")
+    r.add_argument("text", nargs="?", help="text to redact; omit to read stdin / interactive")
+    r.add_argument("-f", "--file", help="read input from a file")
+    _add_lang(r)
+    r.add_argument("--device", choices=["cpu", "cuda", "coreml"], default="cpu")
+    r.add_argument(
+        "--operating-point", default="balanced",
+        help="high_recall | balanced | high_precision, or comma-separated raw biases",
+    )
+    r.add_argument("--output-mode", choices=["typed", "redacted"], default="typed")
+    r.add_argument("--entity-types", help="comma-separated subset, e.g. PER,PHONE")
+    r.add_argument("--regex-only", action="store_true", help="skip the NER stage (no weights)")
+    r.add_argument("--cache-dir", help="local weight bundle directory (offline)")
+    r.add_argument("--revision", help="pin a HF Hub weight revision")
+    r.add_argument("--json", action="store_true", help="force JSON output")
+
+    # -- eval ---------------------------------------------------------------
+    e = sub.add_parser("eval", help="evaluate against a labelled JSONL dataset")
+    e.add_argument("dataset", help="JSONL with text + spans")
+    _add_lang(e)
+    e.add_argument("--eval-mode", choices=["typed", "untyped"], default="typed")
+    e.add_argument("--operating-point", default="balanced")
+    e.add_argument("--regex-only", action="store_true")
+    e.add_argument("--cache-dir")
+    e.add_argument("--limit", type=int, default=None)
+
+    # -- train --------------------------------------------------------------
+    t = sub.add_parser("train", help="fine-tune the NER model on custom JSONL")
+    t.add_argument("dataset", help="training JSONL")
+    _add_lang(t)
+    t.add_argument("--validation-dataset")
+    t.add_argument("--label-space-json", help="custom label map (FR-2.6 / §8.2)")
+    t.add_argument("--output-dir", required=True)
+    t.add_argument("--base-model", default=None)
+    t.add_argument("--epochs", type=int, default=3)
+    t.add_argument("--batch-size", type=int, default=16)
+    t.add_argument("--learning-rate", type=float, default=3e-5)
+    return p

privyscope/_cli/render.py

@@ -0,0 +1,65 @@
+"""Terminal rendering for the CLI (SRS FR-2.1 ANSI span previews)."""
+from __future__ import annotations
+
+import json
+import sys
+from typing import List
+
+from .._core.schema import RedactionResult
+
+# Per-label ANSI background colours; falls back to a default for custom labels.
+_COLORS = {
+    "PER": "\033[48;5;31m", "PHONE": "\033[48;5;130m", "ID_NUM": "\033[48;5;90m",
+    "EMAIL": "\033[48;5;28m", "LOC": "\033[48;5;24m", "BANK": "\033[48;5;94m",
+    "DATE": "\033[48;5;55m", "SECRET": "\033[48;5;124m",
+}
+_DEFAULT_COLOR = "\033[48;5;240m"
+_RESET = "\033[0m"
+_DIM = "\033[2m"
+_BOLD = "\033[1m"
+
+
+def supports_color(stream=sys.stdout) -> bool:
+    return hasattr(stream, "isatty") and stream.isatty()
+
+
+def highlight(result: RedactionResult, color: bool = True) -> str:
+    """Return the input with detected spans visually marked."""
+    text = result.text
+    out: List[str] = []
+    cursor = 0
+    for s in sorted(result.detected_spans, key=lambda d: d.start):
+        out.append(text[cursor : s.start])
+        chunk = text[s.start : s.end]
+        if color:
+            c = _COLORS.get(s.label, _DEFAULT_COLOR)
+            out.append(f"{c}{chunk}{_RESET}{_DIM}[{s.label}]{_RESET}")
+        else:
+            out.append(f"[{chunk}]({s.label})")
+        cursor = s.end
+    out.append(text[cursor:])
+    return "".join(out)
+
+
+def render_human(result: RedactionResult, color: bool = True) -> str:
+    lines = [
+        highlight(result, color),
+        "",
+        f"{_BOLD if color else ''}redacted:{_RESET if color else ''} {result.redacted_text}",
+        f"{_DIM if color else ''}spans: {result.summary['span_count']}  "
+        f"by_label: {result.summary['by_label']}"
+        + ("  (decode mismatch)" if result.decoded_mismatch else "")
+        + (_RESET if color else ""),
+    ]
+    return "\n".join(lines)
+
+
+def render_json(result: RedactionResult) -> str:
+    return json.dumps(result.to_dict(), ensure_ascii=False, indent=2)
+
+
+def print_result(result: RedactionResult, as_json: bool) -> None:
+    if as_json:
+        print(render_json(result))
+    else:
+        print(render_human(result, color=supports_color()))

privyscope/_core/__init__.py

@@ -0,0 +1,20 @@
+"""Runtime core: loading, regex/NER stages, decoding, span assembly."""
+from .bioes import Span, bioes_to_spans, spans_to_bioes
+from .pipeline import assemble, merge_union
+from .regex_filter import RegexFilter
+from .schema import DetectedSpan, RedactionResult, SCHEMA_VERSION
+from .transitions import OPERATING_POINTS, resolve_operating_point
+
+__all__ = [
+    "Span",
+    "bioes_to_spans",
+    "spans_to_bioes",
+    "assemble",
+    "merge_union",
+    "RegexFilter",
+    "DetectedSpan",
+    "RedactionResult",
+    "SCHEMA_VERSION",
+    "OPERATING_POINTS",
+    "resolve_operating_point",
+]