noirdoc 0.1.0__py3-none-any.whl

noirdoc/__init__.py

@@ -0,0 +1,12 @@
+"""noirdoc — local PII redaction and pseudonymization for documents."""
+
+from __future__ import annotations
+
+from noirdoc.sdk import RedactionResult, Redactor, redact
+
+try:
+    from noirdoc._version import __version__  # type: ignore[import-not-found]
+except ImportError:  # Source checkout without a build step (e.g. plain `pytest`).
+    __version__ = "0.0.0+unknown"
+
+__all__ = ["RedactionResult", "Redactor", "__version__", "redact"]

noirdoc/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

noirdoc/cli.py

@@ -0,0 +1,255 @@
+"""Command-line interface for noirdoc."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+
+from noirdoc import __version__
+from noirdoc.namespace import DEFAULT_NAMESPACE_ROOT, Namespace, list_namespaces
+from noirdoc.sdk import Redactor
+
+
+@click.group()
+@click.version_option(__version__, prog_name="noirdoc")
+def main() -> None:
+    """Local PII redaction and pseudonymization for documents."""
+
+
+@main.command()
+@click.argument("inputs", nargs=-1, type=click.Path(exists=True, path_type=Path), required=True)
+@click.option("-o", "--output", type=click.Path(path_type=Path), help="Output file (single input).")
+@click.option("--output-dir", type=click.Path(path_type=Path), help="Output directory (batch).")
+@click.option("--namespace", help="Persistent namespace for reversible pseudonyms.")
+@click.option(
+    "--language",
+    "-l",
+    default="de",
+    type=click.Choice(["de", "en"]),
+    show_default=True,
+)
+@click.option(
+    "--detector",
+    default="ensemble",
+    type=click.Choice(["presidio", "gliner", "ensemble"]),
+    show_default=True,
+)
+@click.option("--score-threshold", type=float, default=0.5, show_default=True)
+@click.option("-v", "--verbose", is_flag=True, help="Print per-file details.")
+def redact(
+    inputs: tuple[Path, ...],
+    output: Path | None,
+    output_dir: Path | None,
+    namespace: str | None,
+    language: str,
+    detector: str,
+    score_threshold: float,
+    verbose: bool,
+) -> None:
+    """Redact PII in one or more files."""
+    files = _expand_inputs(inputs)
+    if not files:
+        click.echo("No input files found.", err=True)
+        sys.exit(1)
+
+    if output and len(files) > 1:
+        click.echo("--output only works with a single input; use --output-dir.", err=True)
+        sys.exit(2)
+
+    r = Redactor(
+        namespace=namespace,
+        language=language,
+        detector=detector,  # type: ignore[arg-type]
+        score_threshold=score_threshold,
+    )
+
+    success = 0
+    for path in files:
+        try:
+            result = r.redact_file(path, language=language)
+        except Exception as exc:
+            click.echo(f"{path.name}: {exc}", err=True)
+            continue
+
+        out_path = _choose_output_path(
+            path,
+            output=output,
+            output_dir=output_dir,
+            reconstructed=result.reconstructed,
+        )
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_bytes(result.output_bytes)
+
+        if verbose:
+            types = ", ".join(f"{k}={v}" for k, v in sorted(result.entity_types.items()))
+            click.echo(f"{path.name}: {result.entity_count} entities [{types}] -> {out_path}")
+        else:
+            click.echo(f"{path.name}: {result.entity_count} entities -> {out_path}")
+        success += 1
+
+    if namespace:
+        click.echo(f"Namespace {namespace!r}: {r.mapper.entity_count} unique entities total")
+
+    sys.exit(0 if success else 1)
+
+
+@main.command()
+@click.argument("input", required=False, type=click.Path(exists=True, path_type=Path))
+@click.option("-o", "--output", type=click.Path(path_type=Path), help="Output file.")
+@click.option("--namespace", required=True, help="Namespace holding the mapping.")
+def reveal(input: Path | None, output: Path | None, namespace: str) -> None:
+    """Restore original values from pseudonyms using a namespace.
+
+    With no INPUT, reads text from stdin and writes to stdout.
+    With an INPUT file, writes reidentified bytes (DOCX/XLSX/plain) to OUTPUT.
+    PDF/PPTX/images are not roundtrippable; the command warns and copies the
+    original bytes.
+    """
+    ns = Namespace(namespace)
+    if not ns.exists():
+        click.echo(f"Namespace {namespace!r} does not exist.", err=True)
+        sys.exit(1)
+
+    r = Redactor(namespace=namespace)
+
+    if input is None:
+        text = sys.stdin.read()
+        sys.stdout.write(r.reveal_text(text))
+        return
+
+    revealed = r.reveal_file(input, output=output)
+    if revealed is None:
+        click.echo(
+            f"Cannot reveal {input.name}: format not supported for roundtrip. "
+            "Original bytes unchanged.",
+            err=True,
+        )
+        if output:
+            Path(output).write_bytes(input.read_bytes())
+        sys.exit(2)
+    click.echo(f"{input.name}: revealed -> {output or '(bytes returned)'}")
+
+
+@main.command()
+@click.argument("pseudonym")
+@click.option("--namespace", required=True)
+def lookup(pseudonym: str, namespace: str) -> None:
+    """Look up the original value behind a pseudonym token."""
+    r = Redactor(namespace=namespace)
+    original = r.lookup(pseudonym)
+    if original is None:
+        click.echo(f"{pseudonym}: not found", err=True)
+        sys.exit(1)
+    click.echo(original)
+
+
+@main.group()
+def ns() -> None:
+    """Manage persistent namespaces."""
+
+
+@ns.command("list")
+def ns_list() -> None:
+    """List all namespaces under ~/.noirdoc/namespaces/."""
+    names = list_namespaces()
+    if not names:
+        click.echo(f"No namespaces found in {DEFAULT_NAMESPACE_ROOT}")
+        return
+    for n in names:
+        click.echo(n)
+
+
+@ns.command("show")
+@click.argument("namespace")
+def ns_show(namespace: str) -> None:
+    """Print the mapping summary for NAMESPACE as JSON."""
+    ns_obj = Namespace(namespace)
+    if not ns_obj.exists():
+        click.echo(f"Namespace {namespace!r} does not exist.", err=True)
+        sys.exit(1)
+    mapper = ns_obj.load()
+    click.echo(json.dumps(mapper.get_mapping_summary(), indent=2, ensure_ascii=False))
+
+
+@ns.command("delete")
+@click.argument("namespace")
+@click.confirmation_option(prompt="Delete namespace and discard all mappings?")
+def ns_delete(namespace: str) -> None:
+    """Delete a namespace directory (irreversible)."""
+    ns_obj = Namespace(namespace)
+    if not ns_obj.exists():
+        click.echo(f"Namespace {namespace!r} does not exist.", err=True)
+        sys.exit(1)
+    ns_obj.delete()
+    click.echo(f"Deleted {namespace!r}")
+
+
+@main.group()
+def models() -> None:
+    """Manage local detection models."""
+
+
+@models.command("pull")
+@click.option("--language", "-l", multiple=True, default=["de", "en"], show_default=True)
+@click.option("--gliner/--no-gliner", default=True, show_default=True)
+@click.option(
+    "--gliner-model",
+    default="knowledgator/gliner-pii-edge-v1.0",
+    show_default=True,
+)
+def models_pull(language: tuple[str, ...], gliner: bool, gliner_model: str) -> None:
+    """Download spaCy models (and optionally GLiNER weights)."""
+    from noirdoc.detection.model_manager import ensure_spacy_models
+
+    click.echo(f"Downloading spaCy models for: {', '.join(language)}")
+    ensure_spacy_models(list(language))
+
+    if gliner:
+        click.echo(f"Loading GLiNER model: {gliner_model}")
+        try:
+            from noirdoc.detection.gliner_detector import GlinerDetector
+
+            GlinerDetector(model_name=gliner_model)
+        except ImportError:
+            click.echo(
+                "GLiNER not installed. Install with: pip install 'noirdoc[full]'",
+                err=True,
+            )
+            sys.exit(1)
+
+    click.echo("Models ready.")
+
+
+# -- helpers ------------------------------------------------------------------
+
+
+def _expand_inputs(inputs: tuple[Path, ...]) -> list[Path]:
+    files: list[Path] = []
+    for p in inputs:
+        if p.is_dir():
+            files.extend(sorted(q for q in p.iterdir() if q.is_file()))
+        elif p.is_file():
+            files.append(p)
+    return files
+
+
+def _choose_output_path(
+    input_path: Path,
+    *,
+    output: Path | None,
+    output_dir: Path | None,
+    reconstructed: bool,
+) -> Path:
+    if output:
+        return output
+    parent = output_dir or input_path.parent
+    if reconstructed:
+        return parent / f"{input_path.stem}_redacted{input_path.suffix}"
+    return parent / f"{input_path.stem}_redacted.txt"
+
+
+if __name__ == "__main__":
+    main()

noirdoc/detection/base.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel
+
+ENTITY_TYPES = {
+    "PERSON",
+    "EMAIL",
+    "PHONE",
+    "IBAN",
+    "CREDIT_CARD",
+    "LOCATION",
+    "DATE",
+    "ORGANIZATION",
+    "IP_ADDRESS",
+    "URL",
+    "MEDICAL_LICENSE",
+    "SVNR",
+    "STEUER_ID",
+    "CUSTOM",
+}
+
+
+class DetectedEntity(BaseModel):
+    """Eine erkannte PII-Entität mit Position im Text."""
+
+    entity_type: str
+    text: str
+    start: int
+    end: int
+    score: float
+    source: str
+
+
+class BaseDetector(ABC):
+    """Interface das jeder Detektor implementieren muss."""
+
+    @abstractmethod
+    async def detect(self, text: str, language: str = "de") -> list[DetectedEntity]: ...
+
+    @property
+    @abstractmethod
+    def name(self) -> str: ...

noirdoc/detection/ensemble.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import asyncio
+import re
+
+from noirdoc.detection.base import BaseDetector, DetectedEntity
+
+# Strong indicators: if ANY of these appear in a multi-word PERSON entity, reject it.
+_PERSON_STRONG_REJECT: set[str] = {
+    # verbs / participles commonly absorbed by spaCy NER
+    "wohnhaft",
+    "verheiratet",
+    "geschieden",
+    "ledig",
+    "verwitwet",
+    "überwiesen",
+    "geboren",
+    "verstorben",
+    "beschäftigt",
+    "gemeldet",
+    # common nouns that get falsely tagged
+    "euro",
+    "straße",
+    "strasse",
+    "nummer",
+    "datum",
+}
+
+# Weak indicators: reject only if entity text ENDS with one of these
+# (handles boundary absorption like "Hoffmann, und").
+_PERSON_TRAILING_REJECT: set[str] = {
+    "und",
+    "oder",
+    "mit",
+    "für",
+    "bei",
+    "nach",
+    "der",
+    "die",
+    "das",
+    "ein",
+    "eine",
+    "er",
+    "sie",
+    "es",
+}
+
+# Pattern for username-like strings (lowercase + digits, no spaces)
+_USERNAME_PATTERN = re.compile(r"^[a-z0-9._@\-]+$")
+
+
+def _validate_person(entity: DetectedEntity) -> bool:
+    """Return False if a PERSON entity is likely a false positive."""
+    if entity.entity_type != "PERSON":
+        return True
+
+    text = entity.text
+
+    # Reject entities containing commas or newlines (spaCy boundary absorption)
+    if "," in text or "\n" in text:
+        return False
+
+    # Reject single-character entities
+    if len(text) <= 1:
+        return False
+
+    # Reject username-like patterns (e.g. "cthwhdbfio0854", "al-kurdy8")
+    if _USERNAME_PATTERN.match(text):
+        return False
+
+    # Multi-word validation
+    words = text.lower().split()
+    if len(words) > 1:
+        if any(w in _PERSON_STRONG_REJECT for w in words):
+            return False
+        if words[-1] in _PERSON_TRAILING_REJECT:
+            return False
+
+    return True
+
+
+# Per-entity-type score thresholds (higher = stricter, fewer FPs).
+# Types not listed here use the global default.
+_TYPE_THRESHOLDS: dict[str, float] = {
+    "URL": 0.6,
+    "DATE": 0.7,
+}
+
+
+class EnsembleDetector:
+    """Kombiniert mehrere Detektoren und löst Überlappungen auf."""
+
+    def __init__(
+        self,
+        detectors: list[BaseDetector],
+        score_threshold: float = 0.5,
+    ) -> None:
+        self.detectors = detectors
+        self.score_threshold = score_threshold
+
+    async def detect(self, text: str, language: str = "de") -> list[DetectedEntity]:
+        if not text:
+            return []
+
+        results = await asyncio.gather(
+            *(d.detect(text, language) for d in self.detectors),
+            return_exceptions=True,
+        )
+
+        all_entities: list[DetectedEntity] = []
+        for result in results:
+            if isinstance(result, BaseException):
+                continue
+            all_entities.extend(result)
+
+        filtered = [
+            e
+            for e in all_entities
+            if e.score >= _TYPE_THRESHOLDS.get(e.entity_type, self.score_threshold)
+        ]
+        merged = self._merge_entities(filtered)
+        validated = [e for e in merged if _validate_person(e)]
+        return sorted(validated, key=lambda e: e.start)
+
+    def _merge_entities(self, entities: list[DetectedEntity]) -> list[DetectedEntity]:
+        """
+        Overlap Resolution:
+        1. Sort by start, then by span length descending (longer first)
+        2. For each entity, check overlap with already accepted entities
+        3. On overlap with SAME type: higher score wins; tie → longer span; tie → presidio wins
+        4. On overlap with DIFFERENT type: keep both (dual-type annotation)
+        5. No overlap: accept entity
+        """
+        sorted_ents = sorted(
+            entities,
+            key=lambda e: (e.start, -(e.end - e.start)),
+        )
+
+        accepted: list[DetectedEntity] = []
+        for candidate in sorted_ents:
+            same_type_idx = None
+            for i, existing in enumerate(accepted):
+                if candidate.start < existing.end and existing.start < candidate.end:
+                    if candidate.entity_type == existing.entity_type:
+                        same_type_idx = i
+                        break
+
+            if same_type_idx is None:
+                accepted.append(candidate)
+            else:
+                existing = accepted[same_type_idx]
+                winner = self._pick_winner(existing, candidate)
+                accepted[same_type_idx] = winner
+
+        return accepted
+
+    @staticmethod
+    def _pick_winner(a: DetectedEntity, b: DetectedEntity) -> DetectedEntity:
+        if a.score != b.score:
+            return a if a.score > b.score else b
+        len_a = a.end - a.start
+        len_b = b.end - b.start
+        if len_a != len_b:
+            return a if len_a > len_b else b
+        # Tie-break: presidio wins
+        if a.source == "presidio":
+            return a
+        if b.source == "presidio":
+            return b
+        return a

noirdoc/detection/flair_detector.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import asyncio
+
+from noirdoc.detection.base import BaseDetector, DetectedEntity
+
+_TAG_MAP: dict[str, str] = {
+    "PER": "PERSON",
+    "LOC": "LOCATION",
+    "ORG": "ORGANIZATION",
+}
+
+
+class FlairDetector(BaseDetector):
+    """Eigenständiger Flair-basierter NER-Detektor für deutsche Texte.
+
+    Nutzt flair/ner-german-large (XLM-R, F1=92.3% auf CoNLL-03 DE).
+    Robuster bei Lowercase-Text als spaCy.
+    """
+
+    def __init__(self, model_name: str = "flair/ner-german-large") -> None:
+        from flair.models import SequenceTagger
+
+        self._model = SequenceTagger.load(model_name)
+
+    async def detect(self, text: str, language: str = "de") -> list[DetectedEntity]:
+        if not text or language != "de":
+            return []
+
+        return await asyncio.to_thread(self._predict, text)
+
+    def _predict(self, text: str) -> list[DetectedEntity]:
+        from flair.data import Sentence
+
+        sentence = Sentence(text)
+        self._model.predict(sentence)
+
+        entities: list[DetectedEntity] = []
+        for span in sentence.get_spans("ner"):
+            entity_type = _TAG_MAP.get(span.tag)
+            if entity_type is None:
+                continue
+            entities.append(
+                DetectedEntity(
+                    entity_type=entity_type,
+                    text=span.text,
+                    start=span.start_position,
+                    end=span.end_position,
+                    score=round(span.score, 2),
+                    source="flair",
+                ),
+            )
+        return entities
+
+    @property
+    def name(self) -> str:
+        return "flair"

noirdoc/detection/flair_recognizer.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from presidio_analyzer import EntityRecognizer, RecognizerResult
+
+
+class FlairRecognizer(EntityRecognizer):
+    """Flair-basierter NER-Recognizer für deutsche Texte.
+
+    Nutzt flair/ner-german-large (XLM-R, F1=92.3% auf CoNLL-03 DE).
+    Robuster bei Lowercase-Text als spaCy.
+    """
+
+    PRESIDIO_EQUIVALENCES = {
+        "PER": "PERSON",
+        "LOC": "LOCATION",
+        "ORG": "ORGANIZATION",
+    }
+
+    def __init__(self, model_name: str = "flair/ner-german-large") -> None:
+        # Set attributes BEFORE super().__init__ because it calls self.load()
+        self._model_name = model_name
+        self._model = None
+        super().__init__(
+            supported_entities=list(self.PRESIDIO_EQUIVALENCES.values()),
+            supported_language="de",
+            name="Flair NER",
+        )
+
+    def _ensure_model(self):
+        if self._model is None:
+            from flair.models import SequenceTagger
+
+            self._model = SequenceTagger.load(self._model_name)
+
+    def load(self) -> None:
+        self._ensure_model()
+
+    def analyze(
+        self,
+        text: str,
+        entities: list[str] | None = None,
+        nlp_artifacts=None,
+    ) -> list[RecognizerResult]:
+        if not text:
+            return []
+
+        self._ensure_model()
+
+        from flair.data import Sentence
+
+        sentence = Sentence(text)
+        self._model.predict(sentence)  # type: ignore[attr-defined]
+
+        results: list[RecognizerResult] = []
+        for span in sentence.get_spans("ner"):
+            entity_type = self.PRESIDIO_EQUIVALENCES.get(span.tag)
+            if entity_type is None:
+                continue
+            if entities and entity_type not in entities:
+                continue
+            results.append(
+                RecognizerResult(
+                    entity_type=entity_type,
+                    start=span.start_position,
+                    end=span.end_position,
+                    score=round(span.score, 2),
+                ),
+            )
+        return results