carnaval 0.1.0__py3-none-any.whl

carnaval/__init__.py

@@ -0,0 +1,8 @@
+# Copyright 2026 Patrice AUBERT
+# SPDX-License-Identifier: Apache-2.0
+"""carnaval - PII anonymization framework with reversible vault.
+
+Apache License 2.0.
+"""
+
+__version__ = "0.1.0"

carnaval/cli/__init__.py

@@ -0,0 +1,6 @@
+# Copyright 2026 Patrice AUBERT
+# SPDX-License-Identifier: Apache-2.0
+"""Interfaces en ligne de commande de carnaval.
+
+Expose les points d'entree `carnaval-anonymize` et `carnaval-reinject`.
+"""

carnaval/cli/anonymize.py

@@ -0,0 +1,157 @@
+# Copyright 2026 Patrice AUBERT
+# SPDX-License-Identifier: Apache-2.0
+"""CLI carnaval : anonymise un fichier .txt vers les formats standards.
+
+Usage :
+    carnaval-anonymize inbox/doc.txt
+    carnaval-anonymize doc.txt --profile acknowledge
+    carnaval-anonymize doc.txt --profile acknowledge --private mon_profil
+    carnaval-anonymize doc.txt --no-gliner
+
+La variable d'environnement CARNAVAL_VAULT_PASSWORD doit etre definie.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from carnaval.core.logger import configure_logging
+from carnaval.pipeline import run_anonymization
+
+# Racine du depot (4 niveaux au-dessus : cli/ -> carnaval/ -> src/ -> repo).
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+
+DEMO_PASSWORD = "demo_password_change_me_in_prod"
+
+
+def main() -> int:
+    load_dotenv(_REPO_ROOT / ".env")
+
+    parser = argparse.ArgumentParser(
+        prog="carnaval-anonymize",
+        description=(
+            "Anonymise un fichier texte. Produit simultanement les sorties "
+            "TXT, JSON, JSONL, XML, CoNLL et HTML dans outbox/."
+        ),
+    )
+    parser.add_argument(
+        "input",
+        type=Path,
+        help="Chemin du fichier .txt a anonymiser",
+    )
+    parser.add_argument(
+        "--outbox",
+        type=Path,
+        default=_REPO_ROOT / "outbox",
+        help="Dossier de sortie (defaut : ./outbox)",
+    )
+    parser.add_argument(
+        "--profile",
+        type=str,
+        default=None,
+        help="Profil metier (acknowledge, invoice, email, ...)",
+    )
+    parser.add_argument(
+        "--private",
+        type=str,
+        default=None,
+        help="Profil prive sous profiles_private/",
+    )
+    parser.add_argument(
+        "--no-gliner",
+        action="store_true",
+        help="Desactive GLiNER (plus rapide, mais detecte moins de PERSON/LOCATION libres)",
+    )
+    parser.add_argument(
+        "--gliner-threshold",
+        type=float,
+        default=0.4,
+        help="Seuil de confiance GLiNER (defaut : 0.4)",
+    )
+    parser.add_argument(
+        "--cleanup-pipes",
+        action="store_true",
+        help="Retire les `|` parasites entre les mots (utile pour textes extracteur PDF defaillants)",
+    )
+    parser.add_argument(
+        "--language",
+        type=str,
+        default=None,
+        choices=("fr", "en", "de", "ja"),
+        help="Force la langue. Auto-detection si non specifie.",
+    )
+    parser.add_argument(
+        "--log-level",
+        type=str,
+        default="INFO",
+        choices=("DEBUG", "INFO", "WARNING", "ERROR"),
+        help="Niveau de log (defaut : INFO)",
+    )
+    parser.add_argument(
+        "--console",
+        action="store_true",
+        help="Logs en mode console lisible plutot que JSON",
+    )
+    args = parser.parse_args()
+
+    configure_logging(level=args.log_level, json_format=not args.console)
+
+    password = os.environ.get("CARNAVAL_VAULT_PASSWORD") or DEMO_PASSWORD
+    if password == DEMO_PASSWORD:
+        print(
+            "[AVERTISSEMENT] CARNAVAL_VAULT_PASSWORD non defini : password de demo utilise.",
+            file=sys.stderr,
+        )
+
+    if not args.input.exists():
+        print(f"[ERREUR] Fichier introuvable : {args.input}", file=sys.stderr)
+        return 2
+
+    try:
+        masked, written, cfg = run_anonymization(
+            input_path=args.input,
+            outbox_dir=args.outbox,
+            vault_password=password,
+            profile=args.profile,
+            private_profile=args.private,
+            use_gliner=not args.no_gliner,
+            gliner_threshold=args.gliner_threshold,
+            cleanup_pipes=args.cleanup_pipes,
+            language=args.language,
+            repo_root=_REPO_ROOT,
+        )
+    except Exception as e:
+        print(f"[ERREUR] {type(e).__name__}: {e}", file=sys.stderr)
+        return 1
+
+    # Resume console pour l'humain
+    print()
+    print("=" * 60)
+    print(f"  carnaval - {args.input.name}")
+    print("=" * 60)
+    print(f"  Langue          : {masked.language}")
+    print(f"  Spans masques   : {len(masked.spans)}")
+    print(f"  Par categorie   :")
+    for cat, n in sorted(masked.by_category.items()):
+        print(f"    {cat:18s} : {n}")
+    print()
+    print(f"  Fichiers produits :")
+    print(f"    TXT             : {written.txt_path}")
+    print(f"    JSON            : {written.json_path}")
+    print(f"    JSONL           : {written.jsonl_path}")
+    print(f"    XML             : {written.xml_path}")
+    print(f"    CoNLL           : {written.conll_path}")
+    print(f"    HTML            : {written.html_path}")
+    print(f"    Vault chiffre   : {written.vault_path}")
+    print(f"    Metadata        : {written.meta_path}")
+    print("=" * 60)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

carnaval/cli/reinject.py

@@ -0,0 +1,98 @@
+# Copyright 2026 Patrice AUBERT
+# SPDX-License-Identifier: Apache-2.0
+"""CLI carnaval : restaure les valeurs originales dans un JSON ou XML.
+
+Usage :
+    carnaval-reinject response.json --vault outbox/vault/doc_vault.enc
+    carnaval-reinject response.xml --vault outbox/vault/doc_vault.enc
+    carnaval-reinject response.json --vault ... --output final.json
+
+Auto-detection JSON vs XML par le contenu du fichier (premier caractere).
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+from carnaval.core.vault import Vault, VaultError
+from carnaval.stages.s7_reinject import reinject_string
+
+# Racine du depot (4 niveaux au-dessus : cli/ -> carnaval/ -> src/ -> repo).
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+
+DEMO_PASSWORD = "demo_password_change_me_in_prod"
+
+
+def main() -> int:
+    load_dotenv(_REPO_ROOT / ".env")
+
+    parser = argparse.ArgumentParser(
+        prog="carnaval-reinject",
+        description=(
+            "Restaure les valeurs originales dans un JSON ou XML produit par "
+            "un LLM downstream. Auto-detection du format."
+        ),
+    )
+    parser.add_argument(
+        "input",
+        type=Path,
+        help="Fichier JSON ou XML contenant des placeholders [TYPE_n]",
+    )
+    parser.add_argument(
+        "--vault",
+        type=Path,
+        required=True,
+        help="Chemin du vault chiffre produit par carnaval-anonymize",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=None,
+        help="Fichier de sortie (defaut : <input>_final.<ext>)",
+    )
+    args = parser.parse_args()
+
+    password = os.environ.get("CARNAVAL_VAULT_PASSWORD") or DEMO_PASSWORD
+    if password == DEMO_PASSWORD:
+        print(
+            "[AVERTISSEMENT] CARNAVAL_VAULT_PASSWORD non defini : demo utilise.",
+            file=sys.stderr,
+        )
+
+    if not args.input.exists():
+        print(f"[ERREUR] Fichier introuvable : {args.input}", file=sys.stderr)
+        return 2
+    if not args.vault.exists():
+        print(f"[ERREUR] Vault introuvable : {args.vault}", file=sys.stderr)
+        return 2
+
+    # Charger le vault
+    vault = Vault(password=password, path=args.vault)
+    try:
+        vault.load()
+    except VaultError as e:
+        print(f"[ERREUR] Vault : {e}", file=sys.stderr)
+        return 1
+
+    # Restaurer
+    content = args.input.read_text(encoding="utf-8")
+    restored = reinject_string(content, vault)
+
+    # Sortie
+    if args.output is None:
+        stem = args.input.stem
+        ext = args.input.suffix
+        args.output = args.input.parent / f"{stem}_final{ext}"
+
+    args.output.write_text(restored, encoding="utf-8")
+    print(f"[OK] Fichier restitue : {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

carnaval/core/__init__.py

@@ -0,0 +1,3 @@
+# Copyright 2026 Patrice AUBERT
+# SPDX-License-Identifier: Apache-2.0
+"""Primitives partagees par tous les etages."""

carnaval/core/config_loader.py

@@ -0,0 +1,215 @@
+# Copyright 2026 Patrice AUBERT
+# SPDX-License-Identifier: Apache-2.0
+"""Chargement de la configuration en couches.
+
+Strategie de merge :
+    1. base    -> config/pipeline.yaml + sous-fichiers config/*/*.yaml
+    2. profil  -> profiles/<type>/profile.yaml + sous-fichiers
+    3. prive   -> profiles_private/<custom>/profile.yaml + sous-fichiers (optionnel)
+
+Les listes sont CONCATENEES (deny_lists, allow_lists), les dicts sont MERGES
+profondement. Les scalaires de la couche superieure ecrasent la couche
+inferieure.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+@dataclass
+class Config:
+    """Configuration applicative resolue par couches."""
+
+    raw: dict[str, Any] = field(default_factory=dict)
+    layers: list[str] = field(default_factory=list)  # noms des couches mergees
+
+    # Acces rapides typiques
+    @property
+    def pipeline(self) -> dict[str, Any]:
+        return self.raw.get("pipeline", {})
+
+    @property
+    def patterns(self) -> dict[str, Any]:
+        return self.raw.get("patterns", {})
+
+    @property
+    def deny_lists(self) -> dict[str, list[str]]:
+        return self.raw.get("deny_lists", {})
+
+    @property
+    def allow_lists(self) -> dict[str, list[str]]:
+        return self.raw.get("allow_lists", {})
+
+    @property
+    def policies(self) -> dict[str, Any]:
+        return self.raw.get("policies", {})
+
+    @property
+    def ai_models(self) -> dict[str, Any]:
+        return self.raw.get("ai_models", {})
+
+    def get(self, dotted_key: str, default: Any = None) -> Any:
+        """Acces dotted-path : cfg.get('policies.priority_rules.DenylistRecognizer')."""
+        parts = dotted_key.split(".")
+        node: Any = self.raw
+        for p in parts:
+            if not isinstance(node, dict) or p not in node:
+                return default
+            node = node[p]
+        return node
+
+
+# ----------------------------------------------------------------------
+# Merge utilities
+# ----------------------------------------------------------------------
+
+
+def _deep_merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]:
+    """Merge profond.
+
+    Regles :
+    - dict + dict  -> merge cle a cle (recursif)
+    - list + list  -> concatenation (sans dedoublonnage : c'est le role du caller)
+    - scalaire + scalaire -> overlay gagne
+    - types mixtes -> overlay gagne (warning implicite)
+    """
+    result = dict(base)
+    for key, val in overlay.items():
+        if key in result:
+            existing = result[key]
+            if isinstance(existing, dict) and isinstance(val, dict):
+                result[key] = _deep_merge(existing, val)
+            elif isinstance(existing, list) and isinstance(val, list):
+                result[key] = existing + val
+            else:
+                result[key] = val
+        else:
+            result[key] = val
+    return result
+
+
+def _load_yaml(path: Path) -> dict[str, Any]:
+    """Charge un YAML, renvoie {} si fichier vide."""
+    if not path.exists():
+        return {}
+    with open(path, encoding="utf-8") as f:
+        data = yaml.safe_load(f)
+    return data if isinstance(data, dict) else {}
+
+
+def _load_directory_layer(dir_path: Path) -> dict[str, Any]:
+    """Charge tous les YAML d'un dossier en un dict structure par sous-dossier.
+
+    Pour un layout :
+        layer/
+            pipeline.yaml
+            patterns/
+                fiscal_fr.yaml      -> patterns.fiscal_fr.*
+            deny_lists/
+                organizations.yaml  -> deny_lists.organizations.*
+
+    Le contenu du fichier remplace le sous-namespace correspondant.
+    """
+    if not dir_path.exists() or not dir_path.is_dir():
+        return {}
+
+    out: dict[str, Any] = {}
+
+    # 1. fichiers .yaml directement a la racine de la couche (ex: pipeline.yaml, ai_models.yaml)
+    for yml in sorted(dir_path.glob("*.yaml")):
+        key = yml.stem
+        data = _load_yaml(yml)
+        # Si le YAML contient une cle racine du meme nom, on la fusionne
+        # Sinon on stocke le contenu sous la cle key.
+        if key in data and isinstance(data[key], dict):
+            out[key] = _deep_merge(out.get(key, {}), data[key])
+        else:
+            out[key] = (
+                _deep_merge(out.get(key, {}), data) if isinstance(data, dict) else data
+            )
+
+    # 2. sous-dossiers : patterns/, deny_lists/, allow_lists/, policies/
+    #    + recursion limitee pour places/{fr,de,...}.yaml
+    for sub in sorted(p for p in dir_path.iterdir() if p.is_dir()):
+        sub_content: dict[str, Any] = {}
+        # 2a. fichiers .yaml directs (ex: deny_lists/organizations.yaml)
+        for yml in sorted(sub.glob("*.yaml")):
+            sub_content[yml.stem] = _load_yaml(yml)
+        # 2b. sous-sous-dossiers (ex: deny_lists/places/fr.yaml)
+        #     -> deny_lists.places = {fr: [...], de: [...], ...}
+        for subsub in sorted(p for p in sub.iterdir() if p.is_dir()):
+            lang_dict: dict[str, Any] = {}
+            for yml in sorted(subsub.glob("*.yaml")):
+                lang_dict[yml.stem] = _load_yaml(yml)
+            if lang_dict:
+                sub_content[subsub.name] = _deep_merge(
+                    sub_content.get(subsub.name, {}), lang_dict
+                )
+        if sub_content:
+            out[sub.name] = _deep_merge(out.get(sub.name, {}), sub_content)
+
+    return out
+
+
+# ----------------------------------------------------------------------
+# API publique
+# ----------------------------------------------------------------------
+
+
+def load_config(
+    base_dir: Path | str | None = None,
+    profile: str | None = None,
+    private_profile: str | None = None,
+    repo_root: Path | str | None = None,
+) -> Config:
+    """Charge la config en cascade base -> profile -> private_profile.
+
+    Args:
+        base_dir: chemin du dossier `config/` (defaut : <repo>/config).
+        profile: nom du profil public a appliquer (ex: 'acknowledge').
+        private_profile: nom du profil prive (sous profiles_private/).
+        repo_root: racine du repo (auto-detection par defaut).
+
+    Returns:
+        Config resolu.
+    """
+    if repo_root is None:
+        # racine = parent de src/carnaval/core/
+        repo_root = Path(__file__).resolve().parents[3]
+    repo_root = Path(repo_root)
+
+    base_path = Path(base_dir) if base_dir else repo_root / "config"
+
+    layers_loaded: list[str] = []
+    merged: dict[str, Any] = {}
+
+    # Couche 1 : base
+    base_layer = _load_directory_layer(base_path)
+    if base_layer:
+        merged = _deep_merge(merged, base_layer)
+        layers_loaded.append(f"base:{base_path}")
+
+    # Couche 2 : profile public
+    if profile:
+        prof_path = repo_root / "profiles" / profile
+        prof_layer = _load_directory_layer(prof_path)
+        if not prof_layer:
+            raise FileNotFoundError(f"Profil introuvable : {prof_path}")
+        merged = _deep_merge(merged, prof_layer)
+        layers_loaded.append(f"profile:{profile}")
+
+    # Couche 3 : profil prive (optionnel)
+    if private_profile:
+        priv_path = repo_root / "profiles_private" / private_profile
+        priv_layer = _load_directory_layer(priv_path)
+        if not priv_layer:
+            raise FileNotFoundError(f"Profil prive introuvable : {priv_path}")
+        merged = _deep_merge(merged, priv_layer)
+        layers_loaded.append(f"private:{private_profile}")
+
+    return Config(raw=merged, layers=layers_loaded)

carnaval/core/language_detector.py

@@ -0,0 +1,52 @@
+# Copyright 2026 Patrice AUBERT
+# SPDX-License-Identifier: Apache-2.0
+"""Detection de langue via lingua-language-detector.
+
+Wrapper minimal. Renvoie un code ISO 'fr'/'en'/'de'/'ja' ou 'unknown'.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from lingua import Language, LanguageDetectorBuilder
+
+_SUPPORTED = (Language.FRENCH, Language.ENGLISH, Language.GERMAN, Language.JAPANESE)
+_DETECTOR: Optional[object] = None
+
+
+def _get_detector():
+    """Charge le detecteur paresseusement (premier appel = chargement modeles)."""
+    global _DETECTOR
+    if _DETECTOR is None:
+        _DETECTOR = (
+            LanguageDetectorBuilder.from_languages(*_SUPPORTED)
+            .with_preloaded_language_models()
+            .build()
+        )
+    return _DETECTOR
+
+
+def detect_language(text: str) -> str:
+    """Detecte la langue d'un texte.
+
+    Args:
+        text: contenu textuel a analyser.
+
+    Returns:
+        'fr', 'en', 'de', 'ja' ou 'unknown'.
+    """
+    if not text or not text.strip():
+        return "unknown"
+
+    sample = text[:5000]
+    lang = _get_detector().detect_language_of(sample)
+    if lang is None:
+        return "unknown"
+    mapping = {
+        Language.FRENCH: "fr",
+        Language.ENGLISH: "en",
+        Language.GERMAN: "de",
+        Language.JAPANESE: "ja",
+    }
+    return mapping.get(lang, "unknown")

carnaval/core/logger.py

@@ -0,0 +1,84 @@
+# Copyright 2026 Patrice AUBERT
+# SPDX-License-Identifier: Apache-2.0
+"""Logging structure avec garde-fou anti-fuite.
+
+Principe : un logger structlog standard, plus un filtre qui interdit toute
+journalisation contenant des cles a haut risque (`original`, `raw_text`, `mapping`).
+Si une telle cle apparait, sa valeur est remplacee par `<REDACTED>` avant emission.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from typing import Any
+
+import structlog
+
+# Cles dont la valeur ne doit JAMAIS apparaitre dans les logs.
+SENSITIVE_KEYS = frozenset(
+    {
+        "original",
+        "raw_text",
+        "raw",
+        "text",
+        "mapping",
+        "vault",
+        "vault_contents",
+        "password",
+        "secret",
+        "forward",
+        "backward",
+    }
+)
+
+
+def _redact_sensitive(
+    logger, method_name, event_dict: dict[str, Any]
+) -> dict[str, Any]:
+    """Processor structlog : remplace les valeurs sensibles par <REDACTED>."""
+    for key in list(event_dict.keys()):
+        if key.lower() in SENSITIVE_KEYS:
+            event_dict[key] = "<REDACTED>"
+    return event_dict
+
+
+def configure_logging(level: str = "INFO", json_format: bool = True) -> None:
+    """Configure structlog au niveau global.
+
+    Args:
+        level: 'DEBUG', 'INFO', 'WARNING', 'ERROR'.
+        json_format: True -> sortie JSON (prod). False -> sortie console lisible.
+    """
+    logging.basicConfig(
+        format="%(message)s",
+        stream=sys.stdout,
+        level=getattr(logging, level.upper(), logging.INFO),
+    )
+
+    processors: list[Any] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_log_level,
+        structlog.processors.TimeStamper(fmt="iso"),
+        _redact_sensitive,  # garde-fou anti-fuite
+    ]
+
+    if json_format:
+        processors.append(structlog.processors.JSONRenderer())
+    else:
+        processors.append(structlog.dev.ConsoleRenderer())
+
+    structlog.configure(
+        processors=processors,
+        wrapper_class=structlog.make_filtering_bound_logger(
+            getattr(logging, level.upper(), logging.INFO)
+        ),
+        context_class=dict,
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+
+
+def get_logger(name: str = "carnaval") -> structlog.BoundLogger:
+    """Obtient un logger structure (configure_logging() doit avoir ete appele)."""
+    return structlog.get_logger(name)