veu 0.4.0__py3-none-any.whl

veu/__init__.py

@@ -0,0 +1,78 @@
+"""Véu: esconde dados pessoais brasileiros antes de enviar textos para modelos de IA.
+
+Uso rápido:
+
+    import veu
+
+    resultado = veu.anonymize("Sou a Maria Souza, CPF 529.982.247-25")
+    resultado.text            # "Sou a [PESSOA_1], CPF [CPF_1]"
+    veu.restore("Olá, [PESSOA_1]!", resultado.vault)   # "Olá, Maria Souza!"
+"""
+
+from __future__ import annotations
+
+from veu.analyzer import Analyzer
+from veu.anonymizer import OPERATORS, AnonymizationResult, Anonymizer
+from veu.config import Config, ConfigError, load_config
+from veu.core import Veu
+from veu.entities import LABELS, Entity
+from veu.vault import StreamRestorer, Vault
+
+__version__ = "0.4.0"
+
+__all__ = [
+    "LABELS",
+    "OPERATORS",
+    "AnonymizationResult",
+    "Analyzer",
+    "Anonymizer",
+    "Config",
+    "ConfigError",
+    "Entity",
+    "StreamRestorer",
+    "Vault",
+    "Veu",
+    "__version__",
+    "analyze",
+    "anonymize",
+    "load_config",
+    "protect",
+    "restore",
+    "session",
+]
+
+_default: Veu | None = None
+
+
+def _instance() -> Veu:
+    global _default
+    if _default is None:
+        _default = Veu()
+    return _default
+
+
+def analyze(text: str) -> list[Entity]:
+    """Lista os dados pessoais encontrados no texto."""
+    return _instance().analyze(text)
+
+
+def anonymize(text: str, vault: Vault | None = None) -> AnonymizationResult:
+    """Troca os dados pessoais por marcadores. O cofre do resultado restaura a resposta do modelo."""
+    return _instance().anonymize(text, vault)
+
+
+def restore(text: str, vault: Vault) -> str:
+    """Devolve os dados reais no lugar dos marcadores."""
+    return vault.restore(text)
+
+
+def session(vault: Vault | None = None):
+    """Sessão com um cofre compartilhado: anonymize, anonymize_data, restore, restore_stream."""
+    return _instance().session(vault)
+
+
+def protect(func=None, **options):
+    """Decorador: anonimiza os argumentos e restaura o retorno (veja veu.integrations.decorator)."""
+    from veu.integrations.decorator import protect as decorator
+
+    return decorator(func, **options)

veu/__main__.py

@@ -0,0 +1,3 @@
+from veu.cli import main
+
+raise SystemExit(main())

veu/_text.py

@@ -0,0 +1,31 @@
+"""Funções de texto compartilhadas pelos detectores."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+_SPACES = re.compile(r"\s+")
+_NON_DIGITS = re.compile(r"\D")
+
+
+def strip_accents(text: str) -> str:
+    decomposed = unicodedata.normalize("NFKD", text)
+    return "".join(char for char in decomposed if not unicodedata.combining(char))
+
+
+def fold(text: str) -> str:
+    """Forma de comparação: minúsculas, sem acentos e com espaços simples."""
+    return _SPACES.sub(" ", strip_accents(text).casefold()).strip()
+
+
+def only_digits(text: str) -> str:
+    return _NON_DIGITS.sub("", text)
+
+
+def words_regex(words) -> re.Pattern | None:
+    """Regex que acha qualquer uma das palavras (já normalizadas com fold) como palavra inteira."""
+    terms = sorted({fold(word) for word in words if word}, key=len, reverse=True)
+    if not terms:
+        return None
+    return re.compile(r"(?<![\w/])(?:" + "|".join(re.escape(term) for term in terms) + r")(?![\w/])")

veu/analyzer.py

@@ -0,0 +1,121 @@
+"""Junta os detectores, filtra pela pontuação e resolve sobreposições."""
+
+from __future__ import annotations
+
+import bisect
+import re
+from collections.abc import Iterable
+
+from veu._text import fold, only_digits
+from veu.entities import Entity
+from veu.recognizers import (
+    AddressRecognizer,
+    Candidate,
+    PatternRecognizer,
+    PersonRecognizer,
+    Recognizer,
+    SecretRecognizer,
+)
+from veu.validators import normalize_cnpj
+
+DEFAULT_THRESHOLD = 0.5
+
+
+def default_recognizers() -> list[Recognizer]:
+    return [PatternRecognizer(), PersonRecognizer(), AddressRecognizer(), SecretRecognizer()]
+
+
+class Analyzer:
+    """Encontra dados pessoais em um texto.
+
+    threshold: pontuação mínima (0 a 1). O padrão 0,5 privilegia a privacidade: valores com formato
+    de documento, mas dígitos verificadores errados, também são escondidos.
+    labels: limita a detecção a alguns tipos (por padrão, todos).
+    allowlist: valores que nunca são escondidos, como o CNPJ ou o e-mail de contato da própria empresa.
+    """
+
+    def __init__(
+        self,
+        recognizers: Iterable[Recognizer] | None = None,
+        *,
+        threshold: float = DEFAULT_THRESHOLD,
+        labels: Iterable[str] | None = None,
+        allowlist: Iterable[str] = (),
+        allow_patterns: Iterable[str] = (),
+    ):
+        self.recognizers = list(default_recognizers() if recognizers is None else recognizers)
+        self.threshold = threshold
+        self.labels = {label.upper() for label in labels} if labels else None
+        allowlist = list(allowlist)
+        self._allow_text = {fold(value) for value in allowlist}
+        self._allow_codes = {only_digits(value) for value in allowlist if len(only_digits(value)) >= 6}
+        self._allow_codes |= {normalize_cnpj(value) for value in allowlist if len(normalize_cnpj(value)) == 14}
+        self._allow_patterns = [re.compile(pattern, re.IGNORECASE) for pattern in allow_patterns]
+        self._has_allowlist = bool(self._allow_text or self._allow_codes or self._allow_patterns)
+
+    def _allowed(self, value: str) -> bool:
+        if fold(value) in self._allow_text:
+            return True
+        if any(pattern.search(value) for pattern in self._allow_patterns):
+            return True
+        digits = only_digits(value)
+        return (len(digits) >= 6 and digits in self._allow_codes) or normalize_cnpj(value) in self._allow_codes
+
+    def candidates(self, text: str, threshold: float | None = None) -> list[Candidate]:
+        minimum = self.threshold if threshold is None else threshold
+        found = []
+        for recognizer in self.recognizers:
+            for candidate in recognizer.find(text):
+                if candidate.score < minimum:
+                    continue
+                if self.labels is not None and candidate.label not in self.labels:
+                    continue
+                if self._has_allowlist and self._allowed(text[candidate.start:candidate.end]):
+                    continue
+                found.append(candidate)
+        return found
+
+    def analyze(self, text: str, threshold: float | None = None) -> list[Entity]:
+        if not text:
+            return []
+        chosen = resolve_overlaps(self.candidates(text, threshold))
+        return [
+            Entity(c.label, c.start, c.end, c.score, text[c.start:c.end], c.recognizer)
+            for c in chosen
+        ]
+
+
+def resolve_overlaps(candidates: list[Candidate]) -> list[Candidate]:
+    """Escolhe um candidato por trecho.
+
+    1. Um candidato contido em outro maior é descartado (o endereço "Rua João Pessoa, 50" vence a pessoa
+       "João Pessoa"; o telefone com DDD vence o número local dentro dele).
+    2. Nas sobreposições parciais e nos trechos idênticos, vence a maior pontuação e, no empate, o mais longo.
+    """
+    ordered = sorted(candidates, key=lambda c: (c.start, -c.end, -c.score))
+    uncontained: list[Candidate] = []
+    container: Candidate | None = None
+    for candidate in ordered:
+        if (
+            container is not None
+            and candidate.end <= container.end
+            and (candidate.start, candidate.end) != (container.start, container.end)
+        ):
+            continue
+        uncontained.append(candidate)
+        if container is None or candidate.end > container.end:
+            container = candidate
+
+    starts: list[int] = []
+    ends: list[int] = []
+    chosen: list[Candidate] = []
+    for candidate in sorted(uncontained, key=lambda c: (-c.score, -(c.end - c.start), c.start)):
+        position = bisect.bisect_left(starts, candidate.start)
+        if position > 0 and ends[position - 1] > candidate.start:
+            continue
+        if position < len(starts) and starts[position] < candidate.end:
+            continue
+        starts.insert(position, candidate.start)
+        ends.insert(position, candidate.end)
+        chosen.append(candidate)
+    return sorted(chosen, key=lambda c: c.start)

veu/anonymizer.py

@@ -0,0 +1,209 @@
+"""Troca os dados pessoais encontrados por marcadores, substitutos ou máscaras."""
+
+from __future__ import annotations
+
+import bisect
+import re
+from collections import Counter
+from dataclasses import dataclass, field
+
+from veu._text import fold
+from veu.analyzer import Analyzer
+from veu.entities import SECRET_LABELS, Entity
+from veu.recognizers.base import folded_set
+from veu.recognizers.persons import CONNECTORS
+from veu.recognizers.secrets import secret_prefix
+from veu.surrogates import make_surrogate
+from veu.vault import Vault
+
+OPERATORS = {
+    "token": "marcador reversível, como [CPF_1] (padrão)",
+    "sintetico": "valor falso do mesmo tipo e formato, reversível (outro CPF válido, outro nome)",
+    "mascara": "mascara parte do valor, como ***.982.247-**; não é reversível",
+    "remover": "troca pelo tipo, como [CPF]; não é reversível",
+    "hash": "marcador estável entre requisições, como [CPF_3fa2c1d9]; reversível com o mesmo cofre",
+}
+OPERATOR_ALIASES = {"marcador": "token", "sintético": "sintetico", "máscara": "mascara", "synthetic": "sintetico",
+                    "mask": "mascara", "redact": "remover"}
+
+
+def normalize_operator(name: str) -> str:
+    key = OPERATOR_ALIASES.get(name.strip().lower(), name.strip().lower())
+    if key not in OPERATORS:
+        raise ValueError(f'Operador desconhecido: "{name}". Use um destes: {", ".join(OPERATORS)}.')
+    return key
+
+
+@dataclass(frozen=True, slots=True)
+class Replacement:
+    entity: Entity
+    replacement: str
+    operator: str
+
+    @property
+    def label(self) -> str:
+        return self.entity.label
+
+    def to_dict(self) -> dict:
+        data = self.entity.to_dict()
+        data.pop("texto")
+        data["substituto"] = self.replacement
+        data["operador"] = self.operator
+        return data
+
+
+@dataclass
+class AnonymizationResult:
+    text: str
+    replacements: list[Replacement]
+    vault: Vault = field(repr=False)
+
+    @property
+    def entities(self) -> list[Entity]:
+        return [item.entity for item in self.replacements]
+
+    def counts(self) -> Counter[str]:
+        return Counter(item.label for item in self.replacements)
+
+    def restore(self, text: str) -> str:
+        """Restaura um texto produzido a partir deste resultado (por exemplo, a resposta do modelo)."""
+        return self.vault.restore(text)
+
+
+def mask_value(label: str, value: str) -> str:
+    if label == "CPF":
+        chars = [char for char in value if char.isdigit()]
+        if len(chars) == 11:
+            visible = set(range(3, 9))
+            output, index = [], 0
+            for char in value:
+                if char.isdigit():
+                    output.append(char if index in visible else "*")
+                    index += 1
+                else:
+                    output.append(char)
+            return "".join(output)
+    if label == "EMAIL" and "@" in value:
+        local, domain = value.rsplit("@", 1)
+        return f"{local[:1]}***@{domain}"
+    if label == "PESSOA":
+        return " ".join(word if fold(word) in CONNECTORS else word[:1] + "***" for word in value.split())
+    if label in ("SENHA", "CHAVE_PRIVADA"):
+        return "********"
+    if label == "URL_CONEXAO" and "://" in value:
+        return value.split("://", 1)[0] + "://****"
+    if label in SECRET_LABELS:
+        return secret_prefix(value) + "****"
+    positions = [index for index, char in enumerate(value) if char.isalnum()]
+    keep = set(positions[-2:]) if len(positions) > 4 else set()
+    return "".join(char if (not char.isalnum() or index in keep) else "*" for index, char in enumerate(value))
+
+
+class Anonymizer:
+    """Anonimiza textos usando um Analyzer e um operador por tipo de dado.
+
+    operator: operador padrão ("token", "sintetico", "mascara", "remover" ou "hash").
+    operators: operador por tipo, como {"CARTAO_CREDITO": "remover"}.
+    link_names: aplica o marcador de um nome completo também às menções só ao primeiro nome ou ao
+    sobrenome ("João da Silva" ... "João" viram o mesmo [PESSOA_1]).
+    """
+
+    def __init__(
+        self,
+        analyzer: Analyzer | None = None,
+        *,
+        operator: str = "token",
+        operators: dict[str, str] | None = None,
+        link_names: bool = True,
+        secret: bytes | None = None,
+    ):
+        self.analyzer = analyzer or Analyzer()
+        self.operator = normalize_operator(operator)
+        self.operators = {label.upper(): normalize_operator(op) for label, op in (operators or {}).items()}
+        self.link_names = link_names
+        self.secret = secret
+        self._ambiguous = folded_set("ambiguous.txt")
+
+    def new_vault(self) -> Vault:
+        return Vault(secret=self.secret)
+
+    def anonymize(self, text: str, vault: Vault | None = None) -> AnonymizationResult:
+        vault = self.new_vault() if vault is None else vault
+        entities = self.analyzer.analyze(text)
+        canonical: dict[int, str] = {}
+        if self.link_names and entities:
+            entities, canonical = self._link_partial_names(text, entities, vault)
+
+        output: list[str] = []
+        replacements: list[Replacement] = []
+        position = 0
+        for entity in entities:
+            operator = self.operators.get(entity.label, self.operator)
+            replacement = self._replace(entity, operator, vault, canonical.get(entity.start))
+            output.append(text[position:entity.start])
+            output.append(replacement)
+            replacements.append(Replacement(entity, replacement, operator))
+            position = entity.end
+        output.append(text[position:])
+        return AnonymizationResult("".join(output), replacements, vault)
+
+    def replace_value(self, label: str, value: str, vault: Vault) -> str:
+        """Substitui um valor inteiro já sabido ser de um tipo (ex.: toda a coluna "cpf" de uma planilha)."""
+        label = label.upper()
+        operator = self.operators.get(label, self.operator)
+        return self._replace(Entity(label, 0, len(value), 1.0, value, "forcado"), operator, vault, None)
+
+    def _replace(self, entity: Entity, operator: str, vault: Vault, canonical: str | None) -> str:
+        value = entity.text
+        if canonical is not None and operator in ("token", "hash", "sintetico"):
+            # A menção parcial usa o marcador do nome completo, mesmo que ele apareça depois no texto.
+            full = Entity(entity.label, entity.start, entity.end, entity.score, canonical, entity.recognizer)
+            replacement = self._replace(full, operator, vault, None)
+            vault.alias(entity.label, value, replacement)
+            return replacement
+        if operator == "token":
+            return vault.placeholder(entity.label, value)
+        if operator == "hash":
+            return vault.placeholder(entity.label, value, hashed=True)
+        if operator == "sintetico":
+            return vault.surrogate(entity.label, value, lambda rng: make_surrogate(entity.label, value, rng))
+        if operator == "mascara":
+            return mask_value(entity.label, value)
+        return f"[{entity.label}]"
+
+    def _link_partial_names(self, text: str, entities: list[Entity], vault: Vault):
+        """Acha menções soltas ao primeiro nome ou ao sobrenome de uma pessoa já detectada."""
+        parts: dict[str, str] = {}
+        for entity in entities:
+            if entity.label != "PESSOA":
+                continue
+            words = [word for word in entity.text.split() if fold(word) not in CONNECTORS]
+            if len(words) < 2:
+                continue
+            for word in (words[0], words[-1]):
+                if len(word) >= 3 and fold(word) not in self._ambiguous and word[0].isupper():
+                    parts.setdefault(word, entity.text)
+        if not parts:
+            return entities, {}
+        alternatives = "|".join(re.escape(word) for word in sorted(parts, key=len, reverse=True))
+        pattern = re.compile(r"(?<![\w])(?:" + alternatives + r")(?![\w])")
+        added: list[Entity] = []
+        canonical: dict[int, str] = {}
+        # As entidades vêm ordenadas e sem sobreposição: busca binária mantém o custo linear em textos grandes.
+        starts = [e.start for e in entities]
+        ends = [e.end for e in entities]
+        for match in pattern.finditer(text):
+            position = bisect.bisect_right(starts, match.start())
+            if position > 0 and ends[position - 1] > match.start():
+                continue
+            if position < len(starts) and starts[position] < match.end():
+                continue
+            entity = Entity("PESSOA", match.start(), match.end(), 0.7, match.group(0), "vinculo")
+            added.append(entity)
+            canonical[match.start()] = parts[match.group(0)]
+        by_folded = {fold(word): full for word, full in parts.items()}
+        for entity in entities:
+            if entity.label == "PESSOA" and len(entity.text.split()) == 1 and fold(entity.text) in by_folded:
+                canonical[entity.start] = by_folded[fold(entity.text)]
+        merged = sorted(entities + added, key=lambda e: e.start)
+        return merged, canonical

veu/bench/__init__.py

@@ -0,0 +1 @@
+"""Benchmark do Véu: gerador de dados sintéticos anotados e avaliação."""

veu/bench/evaluate.py

@@ -0,0 +1,140 @@
+"""Avaliação do Véu: precisão, revocação e taxa de vazamento por tipo de dado."""
+
+from __future__ import annotations
+
+import time
+from collections import defaultdict
+from dataclasses import dataclass, field
+
+from veu.bench.generator import Sample
+from veu.core import Veu
+
+
+@dataclass
+class LabelReport:
+    true_positives: int = 0
+    false_positives: int = 0
+    false_negatives: int = 0
+    protected: int = 0
+    total: int = 0
+
+    @property
+    def precision(self) -> float:
+        found = self.true_positives + self.false_positives
+        return self.true_positives / found if found else 1.0
+
+    @property
+    def recall(self) -> float:
+        expected = self.true_positives + self.false_negatives
+        return self.true_positives / expected if expected else 1.0
+
+    @property
+    def f1(self) -> float:
+        p, r = self.precision, self.recall
+        return 2 * p * r / (p + r) if p + r else 0.0
+
+    @property
+    def leak_rate(self) -> float:
+        return 1 - self.protected / self.total if self.total else 0.0
+
+
+@dataclass
+class Report:
+    labels: dict[str, LabelReport] = field(default_factory=lambda: defaultdict(LabelReport))
+    samples: int = 0
+    characters: int = 0
+    seconds: float = 0.0
+    leaks: list[tuple[str, str, str]] = field(default_factory=list)
+    false_positives: list[tuple[str, str, str]] = field(default_factory=list)
+
+    @property
+    def overall(self) -> LabelReport:
+        total = LabelReport()
+        for report in self.labels.values():
+            total.true_positives += report.true_positives
+            total.false_positives += report.false_positives
+            total.false_negatives += report.false_negatives
+            total.protected += report.protected
+            total.total += report.total
+        return total
+
+    def to_dict(self) -> dict:
+        def row(report: LabelReport) -> dict:
+            return {
+                "ocorrencias": report.total,
+                "precisao": round(report.precision, 4),
+                "revocacao": round(report.recall, 4),
+                "f1": round(report.f1, 4),
+                "vazamento": round(report.leak_rate, 4),
+                "falsos_positivos": report.false_positives,
+            }
+
+        return {
+            "amostras": self.samples,
+            "caracteres_por_segundo": round(self.characters / self.seconds) if self.seconds else None,
+            "geral": row(self.overall),
+            "por_entidade": {label: row(report) for label, report in sorted(self.labels.items())},
+        }
+
+    def to_markdown(self) -> str:
+        lines = [
+            "| Entidade | Ocorrências | Precisão | Revocação | F1 | Vazamento |",
+            "|---|---:|---:|---:|---:|---:|",
+        ]
+        for label, report in sorted(self.labels.items(), key=lambda item: -item[1].total):
+            if not report.total and not report.false_positives:
+                continue
+            lines.append(
+                f"| {label} | {report.total} | {report.precision:.1%} | {report.recall:.1%} | {report.f1:.1%} | "
+                f"{report.leak_rate:.1%} |"
+            )
+        overall = self.overall
+        lines.append(
+            f"| **Geral** | **{overall.total}** | **{overall.precision:.1%}** | **{overall.recall:.1%}** | "
+            f"**{overall.f1:.1%}** | **{overall.leak_rate:.1%}** |"
+        )
+        return "\n".join(lines).replace(".", ",").replace("|---,", "|---.")
+
+
+def _covered(start: int, end: int, text: str, spans: list[tuple[int, int]]) -> bool:
+    """Todos os caracteres alfanuméricos do trecho ficaram escondidos?"""
+    return all(
+        not text[index].isalnum() or any(s <= index < e for s, e in spans) for index in range(start, end)
+    )
+
+
+def evaluate(samples: list[Sample], veu: Veu | None = None) -> Report:
+    veu = veu or Veu()
+    report = Report()
+    started = time.perf_counter()
+    for sample in samples:
+        predictions = veu.anonymize(sample.text).entities
+        report.samples += 1
+        report.characters += len(sample.text)
+        spans = [(entity.start, entity.end) for entity in predictions]
+        matched_predictions: set[int] = set()
+        for start, end, label in sample.entities:
+            stats = report.labels[label]
+            stats.total += 1
+            hit = next(
+                (i for i, e in enumerate(predictions) if e.label == label and e.start < end and start < e.end), None
+            )
+            if hit is None:
+                stats.false_negatives += 1
+            else:
+                stats.true_positives += 1
+                matched_predictions.add(hit)
+            if _covered(start, end, sample.text, spans):
+                stats.protected += 1
+            else:
+                report.leaks.append((label, sample.text[start:end], sample.text))
+        for index, entity in enumerate(predictions):
+            if index in matched_predictions:
+                continue
+            same = any(entity.label == label and entity.start < end and start < entity.end
+                       for start, end, label in sample.entities)
+            if not same:
+                report.labels[entity.label].false_positives += 1
+                report.false_positives.append((entity.label, entity.text, sample.text))
+    report.seconds = time.perf_counter() - started
+    return report