tarja 0.5.0__py3-none-any.whl

tarja/__init__.py

@@ -0,0 +1,65 @@
+# tarja/__init__.py
+# Public API: find() (search), mask() (replace), validate() (check one value), Vault (reversible
+#     tokens), residual() (second-pass check), register_entity() (your own entities).
+# author/autoria: https://github.com/macmaia
+
+# import the validator modules (one file per document type in validators/)
+from tarja.detect import Match, find, resolve_overlaps
+from tarja.entities import ENTITIES
+from tarja.mask import mask
+from tarja.registry import register_entity, unregister_entity
+from tarja.validators import cartao, cnj, cnpj, cns, cpf, nis
+from tarja.vault import (
+    ProtectedText,
+    Vault,
+    VaultCollisionError,
+    VaultConsumedError,
+    VaultError,
+    VaultExpiredError,
+    VaultScopeError,
+    residual,
+)
+
+# what "from tarja import *" exposes
+__all__ = [
+    "ENTITIES",
+    "Match",
+    "ProtectedText",
+    "Vault",
+    "VaultCollisionError",
+    "VaultConsumedError",
+    "VaultError",
+    "VaultExpiredError",
+    "VaultScopeError",
+    "cartao",
+    "cnj",
+    "cnpj",
+    "cns",
+    "cpf",
+    "find",
+    "mask",
+    "nis",
+    "register_entity",
+    "residual",
+    "resolve_overlaps",
+    "unregister_entity",
+    "validate",
+]
+
+# version read by hatch at build time (see pyproject.toml, [tool.hatch.version])
+__version__ = "0.5.0"
+
+
+def validate(entity: str, value: str) -> bool:
+    """EN: Check value against the entity's rule. E.g. validate("BR_CPF", "529.982.247-25").
+    PT: Valida value pela regra da entidade. Ex: validate("BR_CPF", "529.982.247-25").
+    """
+    # [VALIDATE] look the entity up in the registry (tarja/entities.py)
+    spec = ENTITIES.get(entity)
+    if spec is None:
+        # unknown entity -> explicit error, better than a silent False
+        raise ValueError(f"unknown entity / entidade desconhecida: {entity!r}")
+    # entities without a check digit can't be validated alone
+    if spec.validator is None:
+        raise ValueError(f"{entity} has no check digit / {entity} nao tem DV")
+    return spec.validator(value)

tarja/__main__.py

@@ -0,0 +1,5 @@
+# tarja/__main__.py
+# lets you run "python -m tarja scan file.txt"
+from tarja.cli import main
+
+raise SystemExit(main())

tarja/cli.py

@@ -0,0 +1,129 @@
+# tarja/cli.py
+# [CLI] command line. Examples:
+#   tarja scan contrato.txt                     -> JSON lines, one per finding (value hidden by default)
+#   tarja scan contrato.txt --format table      -> readable table
+#   tarja scan contrato.txt --show-values       -> include the raw identifier (careful)
+#   tarja mask contrato.txt > limpo.txt         -> masked copy, strategy redact
+#   tarja mask - --strategy pseudonym_stable --salt "$TARJA_SALT" < in.txt
+#   cat log.txt | tarja scan - --entities BR_CPF,BR_CNPJ --min-score 0.9
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+from tarja import __version__
+from tarja.detect import find
+from tarja.entities import ENTITIES
+from tarja.mask import STRATEGIES, STRATEGY_ALIASES, mask
+
+# [CLI-LIMIT] input cap, so a huge file or endless pipe can't exhaust memory (override with --max-mb)
+DEFAULT_MAX_MB = 50
+
+
+def _read(path: str, max_mb: float = DEFAULT_MAX_MB) -> str:
+    # [CLI-READ] "-" reads stdin, both capped at max_mb (read one char over the cap to detect it)
+    limit = int(max_mb * 1024 * 1024)
+    if path == "-":
+        text = sys.stdin.read(limit + 1)
+    else:
+        with open(path, encoding="utf-8") as fh:
+            text = fh.read(limit + 1)
+    if len(text) > limit:
+        raise ValueError(f"input larger than {max_mb:g} MB, use --max-mb or split the file")
+    return text
+
+
+def _entities(arg: str | None) -> list[str] | None:
+    # [CLI-ENTITIES] "BR_CPF,BR_CNPJ" -> list, None = all
+    if not arg:
+        return None
+    return [e.strip().upper() for e in arg.split(",") if e.strip()]
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    # [CLI-PARSER]
+    p = argparse.ArgumentParser(
+        prog="tarja",
+        description="Find and mask Brazilian personal identifiers. / Acha e mascara identificadores brasileiros.",
+    )
+    p.add_argument("--version", action="version", version=f"tarja {__version__}")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    # shared options
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("path", help="file or - for stdin / arquivo ou - p/ stdin")
+    common.add_argument(
+        "--entities",
+        help=f"comma-separated / separadas por virgula. Default: all / todas ({','.join(ENTITIES)})",
+    )
+    common.add_argument("--min-score", type=float, default=0.0, help="drop below this / descarta abaixo disso")
+    common.add_argument(
+        "--max-mb",
+        type=float,
+        default=DEFAULT_MAX_MB,
+        help=f"input size cap in MB, default {DEFAULT_MAX_MB} / limite de entrada em MB",
+    )
+
+    # [CLI-SCAN]
+    scan = sub.add_parser("scan", parents=[common], help="list findings / lista o q achou")
+    scan.add_argument("--format", choices=("jsonl", "table"), default="jsonl")
+    scan.add_argument(
+        "--show-values",
+        action="store_true",
+        help="print raw identifiers (personal data!) / mostra o valor cru (dado pessoal!)",
+    )
+
+    scan.add_argument(
+        "--suspect",
+        action="store_true",
+        help="also report ID-shaped values with a wrong check digit (score 0) / reporta tb formato de ID c/ DV errado",
+    )
+
+    # [CLI-MASK]
+    mk = sub.add_parser("mask", parents=[common], help="print masked text / imprime o texto mascarado")
+    mk.add_argument("--strategy", choices=(*STRATEGIES, *STRATEGY_ALIASES), default="redact")
+    mk.add_argument(
+        "--salt",
+        default=os.environ.get("TARJA_SALT"),
+        help="secret key for pseudonym_stable, or set TARJA_SALT / chave secreta, ou use TARJA_SALT",
+    )
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    """EN: Entry point (console script "tarja"). Returns an exit code.
+    PT: Ponto de entrada (script "tarja"). Devolve o exit code.
+    """
+    args = _build_parser().parse_args(argv)
+    try:
+        text = _read(args.path, args.max_mb)
+        # [CLI-SUSPECT] only scan reports suspects, mask never touches them
+        suspect = args.command == "scan" and args.suspect
+        found = find(text, entities=_entities(args.entities), min_score=args.min_score, report_invalid=suspect)
+        if args.command == "mask":
+            # [CLI-MASK-RUN]
+            sys.stdout.write(mask(text, strategy=args.strategy, salt=args.salt, matches=found))
+            return 0
+        # [CLI-SCAN-RUN]
+        if args.format == "jsonl":
+            for m in found:
+                sys.stdout.write(json.dumps(m.to_dict(include_value=args.show_values), ensure_ascii=False) + "\n")
+        else:
+            # simple fixed-width table
+            sys.stdout.write(f"{'entity':<10} {'start':>7} {'end':>7} {'score':>5}  value\n")
+            for m in found:
+                shown = m.value if args.show_values else "*" * len(m.value)
+                sys.stdout.write(f"{m.entity:<10} {m.start:>7} {m.end:>7} {m.score:>5.2f}  {shown}\n")
+        # exit 1 when something was found, handy in CI
+        return 1 if found else 0
+    except (OSError, ValueError, UnicodeDecodeError) as exc:
+        # [CLI-ERROR] clean message instead of a traceback
+        sys.stderr.write(f"tarja: {exc}\n")
+        return 2
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

tarja/detect.py

@@ -0,0 +1,187 @@
+# tarja/detect.py
+# [DETECT] detection engine. Pipeline per entity:
+#   1. normalise the text (same length, offsets kept)
+#   2. run every candidate regex
+#   3. validate the check digit (drop if wrong)
+#   4. look for context words around the match -> final score
+#   5. resolve overlaps between entities
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from functools import lru_cache
+
+from tarja.entities import ENTITIES, TIER_RANK, EntitySpec
+from tarja.normalise import fold, normalise_text
+
+
+@dataclass(frozen=True)
+class Match:
+    """EN: One detected identifier. start/end are offsets in the ORIGINAL text (text[start:end] == value).
+    PT: 1 identificador achado. start/end sao offsets no texto ORIGINAL (text[start:end] == value).
+    """
+
+    entity: str
+    start: int
+    end: int
+    value: str
+    score: float
+    tier: str
+    pattern: str
+    has_context: bool
+    # [DETECT-SUSPECT] False = right shape, WRONG check digit (only with find(report_invalid=True), score 0)
+    valid_dv: bool = True
+
+    def to_dict(self, include_value: bool = True) -> dict:
+        """EN: Plain dict for JSON output. include_value=False drops the raw identifier.
+        PT: Dict simples p/ saida JSON. include_value=False tira o identificador cru.
+        """
+        # [DETECT-DICT]
+        d = {
+            "entity": self.entity,
+            "start": self.start,
+            "end": self.end,
+            "score": self.score,
+            "tier": self.tier,
+            "pattern": self.pattern,
+            "has_context": self.has_context,
+            "valid_dv": self.valid_dv,
+        }
+        if include_value:
+            d["value"] = self.value
+        return d
+
+
+@lru_cache(maxsize=64)
+def _context_regex(words: tuple[str, ...]) -> re.Pattern[str]:
+    # [DETECT-CONTEXT-RE] one regex per word list, whole words only ("sus" must not hit "suspenso")
+    alternatives = "|".join(re.escape(w) for w in sorted(words, key=len, reverse=True))
+    return re.compile(rf"(?<![0-9a-z])(?:{alternatives})(?![0-9a-z])")
+
+
+@lru_cache(maxsize=64)
+def _adjacent_regexes(before: tuple[str, ...], after: tuple[str, ...], gap: int) -> tuple[re.Pattern[str], re.Pattern[str]]:
+    # [DETECT-ADJACENT-RE] word + up to gap non-alphanumerics (+ an optional "o" from a folded "nº") at the END of the
+    #   text before, and up to gap non-alphanumerics + one optional 1-3 letter linking word ("no", "do") + word at the
+    #   START of the text after
+    alt = lambda ws: "|".join(re.escape(w) for w in sorted(ws, key=len, reverse=True)) or "(?!)"  # noqa: E731
+    pre = re.compile(rf"(?<![0-9a-z])(?:{alt(before)})[^0-9a-z]{{0,{gap}}}(?:o[^0-9a-z]{{1,2}})?$")
+    post = re.compile(rf"^[^0-9a-z]{{0,{gap}}}(?:[a-z]{{1,3}}[^0-9a-z]{{1,2}})?(?:{alt(after)})(?![0-9a-z])")
+    return pre, post
+
+
+def _has_context(folded: str, start: int, end: int, spec: EntitySpec) -> bool:
+    # [DETECT-CONTEXT] search window before and after the match in the folded (lowercase, no accent) text
+    if spec.context_before or spec.context_after:
+        # [DETECT-ADJACENT] tight mode, see EntitySpec.context_gap
+        pre, post = _adjacent_regexes(spec.context_before, spec.context_after, spec.context_gap)
+        reach = max((len(w) for w in spec.context_before), default=0) + spec.context_gap + 3
+        return bool(pre.search(folded[max(0, start - reach) : start])) or bool(post.search(folded[end : end + 40]))
+    lo = max(0, start - spec.context_window)
+    hi = min(len(folded), end + spec.context_window)
+    # blank out the match itself so the number can't count as its own context
+    window = folded[lo:start] + " " + folded[end:hi]
+    return _context_regex(spec.context_words).search(window) is not None
+
+
+# [DETECT-SUSPECT-MIN] only "formatted" patterns (base score >= 0.5) can raise a suspect, so random digit
+#   runs don't flood the report
+SUSPECT_MIN_PATTERN_SCORE = 0.5
+
+
+def _candidates(norm: str, folded: str, text: str, spec: EntitySpec, report_invalid: bool = False) -> Iterable[Match]:
+    # [DETECT-CANDIDATES] every regex hit that passes the check digit, one Match per distinct span
+    seen: set[tuple[int, int]] = set()
+    for pat in spec.patterns:
+        for m in pat.regex.finditer(norm):
+            span = (m.start(), m.end())
+            if span in seen:
+                continue
+            seen.add(span)
+            # validate on the normalised slice (ASCII digits etc)
+            if spec.validator is not None and not spec.validator(m.group(0)):
+                # [DETECT-SUSPECT-EMIT] N1 look-alike with wrong DV, reported with score 0 when asked
+                if report_invalid and spec.tier == "N1" and pat.score >= SUSPECT_MIN_PATTERN_SCORE:
+                    ctx = _has_context(folded, m.start(), m.end(), spec)
+                    yield Match(spec.id, m.start(), m.end(), text[m.start() : m.end()], 0.0, spec.tier, pat.name, ctx, False)
+                continue
+            ctx = _has_context(folded, m.start(), m.end(), spec)
+            # entities that require context are dropped without it
+            if spec.context_required and not ctx:
+                continue
+            score = spec.score_with_context if ctx else spec.score_without_context
+            yield Match(
+                entity=spec.id,
+                start=m.start(),
+                end=m.end(),
+                value=text[m.start() : m.end()],
+                score=score,
+                tier=spec.tier,
+                pattern=pat.name,
+                has_context=ctx,
+            )
+
+
+def resolve_overlaps(matches: Iterable[Match], order: list[str] | None = None) -> list[Match]:
+    """EN: Keep the best match wherever spans overlap. Priority: stronger tier, then higher score,
+    then longer span, then registry order. Result is sorted by position.
+    PT: Fica c/ o melhor match onde os trechos se sobrepoem. Prioridade: nivel mais forte, dps score
+    maior, dps trecho mais longo, dps ordem do registro. Resultado ordenado por posicao.
+    """
+    # [DETECT-OVERLAP]
+    order = order or list(ENTITIES)
+    rank = {e: i for i, e in enumerate(order)}
+
+    def key(m: Match) -> tuple:
+        # sort key, best first
+        return (TIER_RANK.get(m.tier, 9), -m.score, -(m.end - m.start), rank.get(m.entity, 99), m.start)
+
+    kept: list[Match] = []
+    for m in sorted(matches, key=key):
+        # keep only if it doesn't touch anything already kept
+        if all(m.end <= k.start or m.start >= k.end for k in kept):
+            kept.append(m)
+    return sorted(kept, key=lambda m: (m.start, m.end))
+
+
+def find(
+    text: str,
+    entities: Iterable[str] | None = None,
+    min_score: float = 0.0,
+    resolve: bool = True,
+    report_invalid: bool = False,
+) -> list[Match]:
+    """EN: Find Brazilian identifiers in text.
+    entities: subset of ids (e.g. ["BR_CPF"]), default all. min_score: drop anything below.
+    resolve: resolve overlaps between entities (default True).
+    report_invalid: also return N1 look-alikes with a WRONG check digit (valid_dv=False, score 0), e.g. typos.
+    PT: Acha identificadores brasileiros no texto.
+    entities: subconjunto de ids (ex: ["BR_CPF"]), padrao todos. min_score: descarta abaixo disso.
+    resolve: resolve sobreposicao entre entidades (padrao True).
+    report_invalid: devolve tb parecidos N1 c/ DV ERRADO (valid_dv=False, score 0), ex: erro de digitacao.
+    """
+    # [FIND] type check
+    if not isinstance(text, str):
+        raise TypeError("text must be str / text precisa ser str")
+    ids = list(entities) if entities is not None else list(ENTITIES)
+    unknown = [e for e in ids if e not in ENTITIES]
+    if unknown:
+        raise ValueError(f"unknown entities / entidades desconhecidas: {unknown}")
+    # [FIND-PREP] normalise once, fold once
+    norm = normalise_text(text)
+    folded = fold(norm)
+    found: list[Match] = []
+    for eid in ids:
+        found.extend(_candidates(norm, folded, text, ENTITIES[eid], report_invalid))
+    # [FIND-SPLIT] suspects never compete with valid matches
+    suspects = [m for m in found if not m.valid_dv]
+    found = [m for m in found if m.valid_dv and m.score >= min_score]
+    # [FIND-RESOLVE]
+    kept = resolve_overlaps(found, ids) if resolve else sorted(found, key=lambda m: (m.start, m.end))
+    if suspects:
+        # a suspect survives only where no valid match sits
+        free = [x for x in suspects if all(x.end <= k.start or x.start >= k.end for k in kept)]
+        kept = sorted(kept + resolve_overlaps(free, ids), key=lambda m: (m.start, m.end))
+    return kept