anon-lib 0.1.0__py3-none-any.whl

anon_lib/__init__.py

@@ -0,0 +1,10 @@
+"""Deteccao e mascaramento de PII em texto livre (pt-br e en).
+
+Camada regex, deterministica e sem rede, opcionalmente somada a uma camada LLM
+via Ollama para entidades que dependem de contexto (nomes, lugares, idades).
+"""
+
+from anon_lib.anonymizer import Anonymizer
+
+__version__ = "0.1.0"
+__all__ = ["Anonymizer", "__version__"]

anon_lib/anonymizer.py

@@ -0,0 +1,160 @@
+import re
+import os
+import copy
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from anon_lib.patterns.patterns import _patterns_by_language
+
+class Anonymizer:
+    def __init__(self,
+            llm_model_name:str="hf.co/empero-ai/Qwen3.8-4B-Distill-GGUF:Q4_K_M",
+            llm_ipaddress_service:str="http://localhost:11434",
+            language="pt-br"
+        ):
+        self.llm_model_name=llm_model_name
+        self.llm_ipaddress_service=llm_ipaddress_service
+        self._llm_client=None  # criado sob demanda; veja _require_llm()
+        self.language=language
+        self._patterns_by_language=_patterns_by_language
+        self.list_of_undeterministics_entities = [
+            "PERSON_NAME", "LOCATION", "ORGANIZATION", 
+            "DATE", "AGE",
+            "TIME", "MONEY", "PERCENT", "FACILITY", "GPE"
+        ]
+
+        self.prompt = (
+            "Você é um detector de dados pessoais (PII). Devolva as entidades "
+            "encontradas no texto em JSON, seguindo o schema pedido.\n"
+            "Regras:\n"
+            "- Copie cada valor exatamente como aparece no texto, sem reescrever.\n"
+            "- Não devolva o texto mascarado: quem mascara é o código.\n"
+            "- Se não houver nenhuma entidade, devolva uma lista vazia.\n"
+            "- Capture TODOS os nomes de pessoas, incluindo terceiros mencionados "
+            "por vínculo familiar, profissional ou social "
+            "(ex: 'pai de', 'filha de', 'cônjuge de', 'advogado de').\n"
+            "- Capture idades expressas como número seguido de 'anos' "
+            "(ex: '35 anos', 'com 12 anos') como entidade do tipo AGE."
+        )
+
+    def _require_llm(self):
+        """Carrega as dependencias opcionais do modo LLM e devolve (client, schema).
+
+        Erro de import vira mensagem acionavel: sem isso o usuario receberia um
+        ModuleNotFoundError cru, sem pista de que existe um extra a instalar.
+        """
+        try:
+            import ollama
+            from anon_lib.schemas.models_schemas import AnonOutput
+        except ImportError as e:
+            raise ImportError(
+                "use_llm=True requer as dependencias opcionais do modo LLM: "
+                "pip install anon-lib[llm]"
+            ) from e
+        if self._llm_client is None:
+            self._llm_client = ollama.Client(host=self.llm_ipaddress_service)
+        return self._llm_client, AnonOutput
+
+    @property
+    def list_of_words_patterns(self):
+        try:
+            # Obtém a lista de padrões a serem procurados
+            lista_por_idioma=self._patterns_by_language.get(self.language)
+            if lista_por_idioma is None:
+                raise ValueError(f"Idioma '{self.language}' não suportado.")
+            return lista_por_idioma
+        except Exception as e:
+            raise ValueError(f"Erro ao obter a lista de padrões: {e}")
+        
+    def split_setences(self, sentence_text:str):
+        try:
+            text_splitter = RecursiveCharacterTextSplitter(
+                separators=["\n\n", "\n", ".", " ", ""],
+                chunk_size=1000,
+                chunk_overlap=100
+            )
+            return text_splitter.split_text(sentence_text)
+        except Exception as e:
+            raise ValueError(f"Erro ao realizar o split das sentenças textuais: {e}") from e
+
+    def find_keywords_and_replace(self, chunks_of_sentence_text:str=None):
+        if not chunks_of_sentence_text:
+            raise ValueError(f"Sem sentença a análisar.")
+
+        dict_found_keywords={}
+        valores_ja_capturados=set()
+        for chunk in chunks_of_sentence_text:
+            for pii_type, pattern in self.list_of_words_patterns.items():
+                for match in re.finditer(pattern, chunk):
+                    valor=match.group(0)
+                    if valor in valores_ja_capturados:
+                        continue
+                    valores_ja_capturados.add(valor)
+                    dict_found_keywords.setdefault(pii_type, set()).add(valor)
+        return dict_found_keywords
+    
+    def get_llm_response(self, prompt:str, format_schema:dict=None):
+        client, AnonOutput = self._require_llm()
+        try:
+            response = client.chat(
+                model=self.llm_model_name,
+                messages=[{"role": "user", "content": prompt}],
+                format=format_schema or AnonOutput.model_json_schema(),
+                options={"temperature": 0}
+            )
+            return response
+        except Exception as e:
+            raise ValueError(f"Erro ao obter resposta do LLM: {e}") from e
+        
+    def find_undeterministic_entities(self, sentence_text:str):
+        prompt = (
+            f"{self.prompt}\n\n"
+            f"Tipos de entidade a procurar:\n"
+            f"{', '.join(self.list_of_undeterministics_entities)}\n\n"
+            f"Ignore estes tipos, já tratados por regex:\n"
+            f"{', '.join(self.list_of_words_patterns)}\n\n"
+            f"Texto:\n{sentence_text.strip()}"
+        )
+
+        _, AnonOutput = self._require_llm()
+        schema = copy.deepcopy(AnonOutput.model_json_schema())
+        schema["$defs"]["PIIEntity"]["properties"]["tipo"]["enum"] = list(
+            self.list_of_undeterministics_entities
+        )
+        response = self.get_llm_response(prompt=prompt, format_schema=schema)
+
+        conteudo = getattr(response.message, "content", None)
+        if not conteudo:
+            raise ValueError("Resposta do LLM veio vazia.")
+        try:
+            resultado = AnonOutput.model_validate_json(conteudo)
+        except Exception as e:
+            raise ValueError(f"Resposta do LLM não bate com o schema: {e}") from e
+
+        entidades = {}
+        for entidade in resultado.anon_model_output:
+            if entidade.valor and entidade.valor in sentence_text:
+                entidades.setdefault(entidade.tipo, set()).add(entidade.valor)
+        return entidades
+            
+    def anonimize_text(self, sentence_text: str, use_llm: bool = False):
+        chunks = self.split_setences(sentence_text=sentence_text)
+        pii_words = self.find_keywords_and_replace(chunks_of_sentence_text=chunks)
+
+        if use_llm:
+            ja_capturados = {v for valores in pii_words.values() for v in valores}
+            for tipo, valores in self.find_undeterministic_entities(sentence_text).items():
+                novos = {v for v in valores
+                        if not any(v in j or j in v for j in ja_capturados)}
+                if novos:
+                    pii_words.setdefault(tipo, set()).update(novos)
+
+        tags = {
+            f"[{tipo}_{i}]": valor
+            for tipo, valores in pii_words.items()
+            for i, valor in enumerate(sorted(valores))
+        }
+
+        masked_text = sentence_text
+        for tag, valor in sorted(tags.items(), key=lambda kv: len(kv[1]), reverse=True):
+            masked_text = masked_text.replace(valor, tag)
+
+        return masked_text, {t: v for t, v in tags.items() if t in masked_text}

anon_lib/patterns/patterns.py

@@ -0,0 +1,36 @@
+_patterns_by_language={
+            "pt-br" : {
+                "EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b",
+                "CHAVE_PIX_ALEATORIA": r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b",
+                "CNPJ_NUMERICO_FORMATADO": r"\b\d{2}\.\d{3}\.\d{3}/\d{4}-\d{2}\b",
+                "CNPJ_ALFANUMERICO": r"\b[A-Z0-9]{2}\.[A-Z0-9]{3}\.[A-Z0-9]{3}/[A-Z0-9]{4}-\d{2}\b",
+                "RG": r"\b\d{2}\.\d{3}\.\d{3}-[0-9Xx]\b",
+                "PIS_PASEP": r"\b\d{3}\.\d{5}\.\d{2}-\d\b",
+                "CEP": r"(?<!\d)\d{5}-\d{3}(?!\d)",
+                "DATA": r"\b\d{2}/\d{2}/\d{4}\b",
+                "PLACA_VEICULO_MERCOSUL": r"\b[A-Z]{3}\d[A-Z]\d{2}\b",
+                "PLACA_VEICULO": r"\b[A-Z]{3}[- ]?\d{4}\b",
+                # --- Financeiro ---
+                "CARTAO_CREDITO_AMEX": r"(?<!\d)3[47]\d{2}[ .-]?\d{6}[ .-]?\d{5}(?!\d)",
+                "CARTAO_CREDITO": r"(?<!\d)(?:\d{4}[ .-]?){3}\d{4}(?!\d)",
+                "IP": r"(?<!\d)(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?!\d)",
+                "CNH": r"(?i)\bCNH\s*n?[º°]?\s*:?\s*\d{11}\b",
+                "AGENCIA": r"(?i)\bag(?:[êe]ncia)?\.?\s*n?[º°]?\s*:?\s*\d{4}(?:-\d)?\b",
+                "CONTA_BANCARIA": r"(?i)\b(?:c\/c|cc|conta)\.?\s*n?[º°]?\s*:?\s*\d{4,12}-?\d?\b",
+                # --- Numéricos crus (mais ambíguos, por último) ---
+                "TELEFONE_BR": r"(?<!\d)(?:\+55[-. ]?)?\(?\d{2}\)?[-. ]?9?\d{4}[-. ]?\d{4}(?!\d)",
+                "TITULO_ELEITOR": r"(?<!\d)\d{4}[ .]?\d{4}[ .]?\d{4}(?!\d)",
+                "CNPJ_NUMERICO": r"\b\d{14}\b",
+                "CNPJ_ALFANUMERICO_SEM_PONTUACAO": r"\b[A-Z0-9]{12}\d{2}\b",
+                "CPF": r"\b\d{3}\.\d{3}\.\d{3}[.-]\d{2}\b|(?<!\d)\d{11}(?!\d)",
+            },
+            "en": {
+                "EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b",
+                "SSN": r"(?<!\d)\d{3}-\d{2}-\d{4}(?!\d)",
+                "CREDIT_CARD_AMEX": r"(?<!\d)3[47]\d{2}[ .-]?\d{6}[ .-]?\d{5}(?!\d)",
+                "CREDIT_CARD": r"(?<!\d)(?:\d{4}[ .-]?){3}\d{4}(?!\d)",
+                "IP": r"(?<!\d)(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?!\d)",
+                "US_PHONE": r"(?<!\d)(?:\+?1[-.]?)?\(?([0-9]{3})\)?[-. ]?([0-9]{3})[-. ]?([0-9]{4})(?!\d)",
+                "US_ZIP": r"\b\d{5}(?:-\d{4})?\b"
+            }
+        }

anon_lib/schemas/models_schemas.py

@@ -0,0 +1,21 @@
+from typing import List
+
+from pydantic import BaseModel, Field
+
+
+class PIIEntity(BaseModel):
+    """Uma entidade PII localizada pelo LLM no texto."""
+
+    tipo: str = Field(description="Tipo da entidade: PERSON, AGE, ADDRESS, ...")
+    valor: str = Field(description="Trecho exato copiado do texto, sem reescrever")
+
+
+class AnonOutput(BaseModel):
+    """Saída estruturada esperada do LLM.
+
+    É uma lista de objetos, e não um dict {tipo: valor}, porque um dict não pode
+    repetir a mesma chave: um texto com duas pessoas perderia uma delas em
+    silêncio (é o mesmo defeito do exemplo de saída que está no README).
+    """
+
+    anon_model_output: List[PIIEntity]

anon_lib-0.1.0.dist-info/METADATA

@@ -0,0 +1,197 @@
+Metadata-Version: 2.4
+Name: anon-lib
+Version: 0.1.0
+Summary: PII detection and masking for Brazilian Portuguese and English
+Project-URL: Homepage, https://github.com/wyctorfogos/anon-lib
+Project-URL: Issues, https://github.com/wyctorfogos/anon-lib/issues
+Author: Wyctor Fogos da Rocha
+License-Expression: MIT
+License-File: LICENSE
+Keywords: anonymization,lgpd,ner,pii,portuguese
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: Natural Language :: Portuguese (Brazilian)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Security
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.8.13
+Requires-Dist: langchain-text-splitters>=0.2
+Provides-Extra: dev
+Requires-Dist: ollama>=0.3; extra == 'dev'
+Requires-Dist: pydantic>=2; extra == 'dev'
+Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: python-dotenv; extra == 'dev'
+Provides-Extra: llm
+Requires-Dist: ollama>=0.3; extra == 'llm'
+Requires-Dist: pydantic>=2; extra == 'llm'
+Description-Content-Type: text/markdown
+
+# Sentence-anonimizer
+
+Detects and masks personally identifiable information (PII) in free text, replacing
+each value with a stable tag and returning the tag → original value mapping.
+
+Supports Brazilian Portuguese (`pt-br`) and English (`en`).
+
+## How it works
+
+Two layers, and the split matters:
+
+| Layer | Finds | Guarantees |
+| --- | --- | --- |
+| **Regex** (always on) | Formatted data: CPF, CNPJ, e-mail, phone, credit card, CEP… | Deterministic. Same input, same output, no network. |
+| **LLM via Ollama** (opt-in) | Context-dependent data regex cannot express: names, places, organizations | Best effort. Recall varies between runs. |
+
+The LLM only **identifies** entities and returns them as structured JSON. The masking
+itself is always done in code, reusing the same tagging machinery as the regex layer.
+The model never rewrites your text, so it cannot silently drop or alter passages.
+
+## Install
+
+```bash
+pip install anon-lib            # regex layer only
+pip install "anon-lib[llm]"     # adds the optional LLM layer
+```
+
+The base install has a single dependency and never touches the network. `ollama` and
+`pydantic` come only with the `llm` extra and are imported lazily, so calling
+`use_llm=True` without them raises an actionable `ImportError` instead of a bare
+`ModuleNotFoundError`.
+
+The LLM layer also needs a running [Ollama](https://ollama.com) server with a model
+pulled:
+
+```bash
+ollama pull qwen3:0.6b
+```
+
+Point the library at it with the `llm_model_name` argument, or read it from your own
+config:
+
+```python
+Anonymizer(llm_model_name="qwen3:0.6b")
+```
+
+## Usage
+
+`anonimize_text` returns a `(masked_text, tags)` tuple. Every tag in the mapping is
+guaranteed to appear in the masked text.
+
+### Regex only (default)
+
+```python
+from anon_lib import Anonymizer
+
+anon = Anonymizer(language="pt-br")
+masked, tags = anon.anonimize_text(
+    "Contato: joao.silva@empresa.com.br, CPF 123.456.789-00, "
+    "CNPJ 12.345.678/0001-99, cel (11) 98765-4321."
+)
+```
+
+```
+Contato: [EMAIL_0], CPF [CPF_0], CNPJ [CNPJ_NUMERICO_FORMATADO_0], cel [TELEFONE_BR_0].
+
+{'[EMAIL_0]': 'joao.silva@empresa.com.br',
+ '[CPF_0]': '123.456.789-00',
+ '[CNPJ_NUMERICO_FORMATADO_0]': '12.345.678/0001-99',
+ '[TELEFONE_BR_0]': '(11) 98765-4321'}
+```
+
+### Regex + LLM
+
+```python
+masked, tags = anon.anonimize_text(
+    "Meu nome é Maria, moro na Rua das Flores, 123 e meu CPF é 111.222.333-44.",
+    use_llm=True,
+)
+```
+
+```
+Meu nome é [PERSON_NAME_0], moro na [LOCATION_0], 123 e meu CPF é [CPF_0].
+
+{'[CPF_0]': '111.222.333-44',
+ '[PERSON_NAME_0]': 'Maria',
+ '[LOCATION_0]': 'Rua das Flores'}
+```
+
+Note the house number `123` left unmasked: that is real output, and it illustrates why
+the regex layer is the one you rely on for guarantees.
+
+Run both layers end to end with `python examples/demo.py`.
+
+## Detected patterns
+
+**pt-br** — `CPF`, `CNPJ_NUMERICO_FORMATADO`, `CNPJ_ALFANUMERICO`, `CNPJ_NUMERICO`,
+`CNPJ_ALFANUMERICO_SEM_PONTUACAO`, `RG`, `PIS_PASEP`, `TITULO_ELEITOR`, `CNH`, `EMAIL`,
+`TELEFONE_BR`, `CEP`, `DATA`, `PLACA_VEICULO`, `PLACA_VEICULO_MERCOSUL`,
+`CARTAO_CREDITO`, `CARTAO_CREDITO_AMEX`, `CHAVE_PIX_ALEATORIA`, `AGENCIA`,
+`CONTA_BANCARIA`, `IP`
+
+**en** — `EMAIL`, `SSN`, `CREDIT_CARD`, `CREDIT_CARD_AMEX`, `US_PHONE`, `US_ZIP`, `IP`
+
+**LLM entity types** — `PERSON_NAME`, `LOCATION`, `ORGANIZATION`, `DATE`, `AGE`,
+`TIME`, `MONEY`, `PERCENT`, `FACILITY`, `GPE` (configurable via
+`list_of_undeterministics_entities`; the list is injected into the response schema as an
+enum, so the model cannot return a type outside it).
+
+## Design notes
+
+**Declaration order is priority.** Some values match several patterns — 11 bare digits
+are both a CPF and a mobile number. The first pattern declared wins, so formatted
+patterns come first and ambiguous bare numerics last.
+
+**Longest value is replaced first.** The leading 12 digits of a spaced credit card also
+match a voter ID. Replacing the shorter value first would corrupt the longer one.
+
+**Label-anchored patterns include the label.** `CNH`, `AGENCIA` and `CONTA_BANCARIA` mask
+the label too (`CNH: 12345678901` → `[CNH_0]`), because the bare number is
+indistinguishable from a CPF without it.
+
+**LLM values absent from the text are discarded.** A paraphrased or invented value would
+produce a no-op substitution while making the output *look* anonymized.
+
+## Known limitations
+
+- **No check-digit validation.** A well-formed but arithmetically invalid CPF is still
+  masked. For anonymization, over-detection is the safe side.
+- **`DATA` matches any `dd/mm/aaaa` date**, with no way to tell a birth date from any
+  other. Distinguishing them requires context, which is the LLM layer's job.
+- **`US_ZIP` matches any 5-digit number**, and bare 11-digit runs are inherently
+  ambiguous between CPF, CNH and mobile numbers.
+- **`MONEY`, `PERCENT` and `TIME` are not PII** and only add noise. Adjust
+  `list_of_undeterministics_entities` for your use case — the list is injected into the
+  response schema as an enum, so it is the effective control over what the model may
+  return.
+- **Tag numbering may skip an index.** When a short value is also a substring of a longer
+  masked one, it stays registered to catch standalone occurrences elsewhere. Dropping it
+  would leak PII.
+
+## Tests
+
+```bash
+pip install -e ".[dev]"   # needs Python 3.9+, as declared in pyproject.toml
+pytest        # 53 tests, hermetic: no server, no network (~2s)
+pytest -m llm # 3 prompt-regression tests against a real Ollama
+```
+
+The default suite stubs every LLM response, so it needs no server. The `llm` marker is
+deselected by default and holds the tests that call the model for real — they verify the
+prompt rules still take effect, which a stub cannot do, and skip themselves when Ollama
+or the model is unavailable.
+
+## Layout
+
+```text
+src/anon_lib/__init__.py                 public API: Anonymizer, __version__
+src/anon_lib/anonymizer.py               Anonymizer
+src/anon_lib/patterns/patterns.py        regex pattern tables, per language
+src/anon_lib/schemas/models_schemas.py   AnonOutput / PIIEntity (LLM response schema)
+examples/demo.py                         runnable demo of both layers
+tests/test_anonymizer.py                 test suite
+```

anon_lib-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+anon_lib/__init__.py,sha256=MPa-bPWPeL2OJtuBunMrSkJRaFnd3CsMvicS41OoIUU,331
+anon_lib/anonymizer.py,sha256=YsWf4rRFiUkrpQO9idnsTZ4U8et3q_lfix-rHJgd8rU,7006
+anon_lib/patterns/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+anon_lib/patterns/patterns.py,sha256=3ghBA6RX6mLXezC7kGgwy-a-j4nOgnyuVUzTENFmEC8,2447
+anon_lib/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+anon_lib/schemas/models_schemas.py,sha256=VzoIQxcYYbLxF8wbgHlPXwyX4FWSYKiChl4h2s7pfVQ,675
+anon_lib-0.1.0.dist-info/METADATA,sha256=AGfFEbt1vCLX8-PYvmyuY6W7dtCh88ZQD1ky_33pBS8,7524
+anon_lib-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+anon_lib-0.1.0.dist-info/licenses/LICENSE,sha256=8KxMCd_YyztEAdhGQW8sXglF3fqE84quMIk7caSNUaQ,1078
+anon_lib-0.1.0.dist-info/RECORD,,

anon_lib-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

anon_lib-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Wyctor Fogos da Rocha
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.