SqueakyCleanText 0.5.0__tar.gz → 0.5.2__tar.gz

{squeakycleantext-0.5.0 → squeakycleantext-0.5.2}/PKG-INFO

@@ -1,19 +1,29 @@
 Metadata-Version: 2.4
 Name: SqueakyCleanText
-Version: 0.5.0
-Summary: A comprehensive text cleaning and preprocessing pipeline.
+Version: 0.5.2
+Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
 Author: Rehan Fazal
 License: MIT
 Project-URL: Homepage, https://github.com/rhnfzl/SqueakyCleanText
-Keywords: text cleaning,text preprocessing,NLP,natural language processing
+Project-URL: Repository, https://github.com/rhnfzl/SqueakyCleanText
+Project-URL: Issues, https://github.com/rhnfzl/SqueakyCleanText/issues
+Project-URL: Changelog, https://github.com/rhnfzl/SqueakyCleanText/releases
+Project-URL: llms.txt, https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt
+Keywords: text cleaning,text preprocessing,NLP,natural language processing,named entity recognition,NER,anonymization,PII removal,data cleaning,machine learning,ONNX,language detection
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Natural Language :: English
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Classifier: Topic :: Software Development :: Libraries
-Classifier: Topic :: Text Processing
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -70,21 +80,25 @@ Dynamic: license-file
 A comprehensive text cleaning and preprocessing pipeline for machine learning and NLP tasks.
 </div>
 
+> **Using an AI coding assistant?** This repo includes an [`llms.txt`](https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt) with the full API surface, config reference, and Q&A - optimised for Claude, Cursor, Copilot, and ChatGPT.
+
 In the world of machine learning and natural language processing, clean and well-structured text data is crucial for building effective downstream models and managing token limits in language models.
 
-SqueakyCleanText simplifies the process by automatically addressing common text issues, ensuring your data is clean and well-structured with minimal effort on your part.
+SqueakyCleanText simplifies the process by automatically addressing common text issues - removing PII, anonymizing named entities (persons, organisations, locations), and ensuring your data is clean and well-structured for language models and classical ML pipelines with minimal effort on your part.
 
 ### Key Features
 
 - **Named Entity Recognition (NER)**:
   - Multi-backend: ONNX (default, torch-free), PyTorch, GLiNER, and ensemble modes
   - Zero-shot custom entities via GLiNER (e.g., PRODUCT, EVENT, SKILL)
-  - Multi-language support (English, Dutch, German, Spanish)
+  - Multi-language support (English, Dutch, German, Spanish, French, Portuguese, Italian)
   - Ensemble voting across backends for improved accuracy
   - Configurable confidence thresholds
   - Lazy model loading (models load on demand per language)
-  - Automatic text chunking for long documents
+  - Shared ONNX sessions across same-model languages (~600 MB RAM saved)
+  - Automatic text chunking for long documents (CJK/Arabic safe)
   - GPU acceleration support (CUDA for ONNX and PyTorch)
+  - Model warm-up API to pre-load on startup
 - **Text Normalization**:
   - Corrects text encoding problems and handles bad Unicode characters
   - Removes or replaces HTML tags and URLs with configurable tokens
@@ -99,7 +113,7 @@ SqueakyCleanText simplifies the process by automatically addressing common text
   - Smart case folding (preserves NER tokens like `<PERSON>`)
 - **Language Support**:
   - Automatic language detection (English, Dutch, German, Spanish)
-  - Language-specific NER models
+  - Language-specific NER models; French, Portuguese, Italian via multilingual model
   - Language-aware stopword removal
   - Extensible: add custom languages with stopwords, month names, and NER models
 - **Dual Output Formats**:
@@ -108,7 +122,9 @@ SqueakyCleanText simplifies the process by automatically addressing common text
 - **Performance**:
   - ONNX Runtime inference (torch-free base install, ~3-5x faster than PyTorch)
   - Thread-parallel batch processing via `ThreadPoolExecutor`
+  - Async batch processing (`aprocess_batch`) for FastAPI / aiohttp
   - Lazy model loading (only loads models as needed)
+  - Shared ONNX sessions for same-model languages (saves ~600 MB for FR/PT/IT)
   - Memory-efficient processing of large texts
   - GPU acceleration (CUDA) for both ONNX and PyTorch backends
 
@@ -141,7 +157,7 @@ SqueakyCleanText simplifies the process by automatically addressing common text
 pip install SqueakyCleanText
 ```
 
-The base install uses **ONNX Runtime** for NER inference — no PyTorch or Transformers required.
+The base install uses **ONNX Runtime** for NER inference - no PyTorch or Transformers required.
 
 ### Optional Extras
 
@@ -216,7 +232,7 @@ cfg = TextCleanerConfig(
     gliner_labels=('person', 'organization', 'location', 'product', 'event'),
     gliner_label_map={
         'person': 'PER', 'organization': 'ORG', 'location': 'LOC',
-        # 'product' and 'event' are unmapped — they become <PRODUCT>, <EVENT> tokens
+        # 'product' and 'event' are unmapped - they become <PRODUCT>, <EVENT> tokens
     },
     gliner_threshold=0.4,
 )
@@ -306,7 +322,7 @@ SqueakyCleanText supports five NER backends, selectable via the `ner_backend` co
 
 | Backend | Description | Dependencies | Best for |
 |---------|-------------|-------------|----------|
-| `onnx` (default) | ONNX Runtime inference with quantized XLM-RoBERTa models | Base install | Production — fast, torch-free |
+| `onnx` (default) | ONNX Runtime inference with quantized XLM-RoBERTa models | Base install | Production: fast, torch-free |
 | `torch` | PyTorch/Transformers pipeline with full XLM-RoBERTa models | `[torch]` extra | Compatibility with existing PyTorch workflows |
 | `gliner` | GLiNER zero-shot NER with custom entity labels | `[gliner]` or `[gliner2]` extra | Custom entity types (PRODUCT, SKILL, EVENT, etc.) |
 | `ensemble_onnx` | ONNX + GLiNER ensemble voting | `[gliner]` extra | Maximum recall with custom entities |
@@ -320,7 +336,8 @@ SqueakyCleanText supports five NER backends, selectable via the `ner_backend` co
 | Dutch | [`rhnfzl/xlm-roberta-large-conll02-dutch-onnx`](https://huggingface.co/rhnfzl/xlm-roberta-large-conll02-dutch-onnx) |
 | German | [`rhnfzl/xlm-roberta-large-conll03-german-onnx`](https://huggingface.co/rhnfzl/xlm-roberta-large-conll03-german-onnx) |
 | Spanish | [`rhnfzl/xlm-roberta-large-conll02-spanish-onnx`](https://huggingface.co/rhnfzl/xlm-roberta-large-conll02-spanish-onnx) |
-| Multilingual | [`rhnfzl/wikineural-multilingual-ner-onnx`](https://huggingface.co/rhnfzl/wikineural-multilingual-ner-onnx) |
+| French / Portuguese / Italian | [`rhnfzl/wikineural-multilingual-ner-onnx`](https://huggingface.co/rhnfzl/wikineural-multilingual-ner-onnx) (shared session) |
+| Multilingual (fallback) | [`rhnfzl/wikineural-multilingual-ner-onnx`](https://huggingface.co/rhnfzl/wikineural-multilingual-ner-onnx) |
 
 ### GLiNER Label Mapping
 
@@ -351,6 +368,28 @@ Processes the input text and returns a tuple containing:
 
 Processes multiple texts using thread-parallel execution. Each result follows the same format as `process()`.
 
+#### `aprocess_batch(texts: List[str], batch_size: int = None) -> List[Tuple[str, Optional[str], Optional[str]]]`
+
+Async version of `process_batch` for use with asyncio-based frameworks (FastAPI, aiohttp). Runs the batch in a thread-pool executor so it does not block the event loop:
+
+```python
+from sct import TextCleaner
+
+cleaner = TextCleaner()
+
+# In an async context (FastAPI route, aiohttp handler, etc.)
+results = await cleaner.aprocess_batch(texts)
+```
+
+#### `warmup(languages: Optional[List[str]] = None) -> None`
+
+Pre-loads NER models to avoid first-request latency. Call once during application startup:
+
+```python
+cleaner = TextCleaner()
+cleaner.warmup(['ENGLISH', 'DUTCH'])  # or warmup() for all supported languages
+```
+
 ### `TextCleanerConfig`
 
 Immutable (frozen) dataclass. Create modified copies with `dataclasses.replace()`:
@@ -413,6 +452,7 @@ new_cfg = dataclasses.replace(cfg, check_ner_process=False)
 | `ner_backend` | `'onnx'` | Backend: `onnx`, `torch`, `gliner`, `ensemble_onnx`, `ensemble_torch` |
 | `positional_tags` | `('PER', 'LOC', 'ORG', 'MISC')` | Entity types to recognize |
 | `ner_confidence_threshold` | `0.85` | Minimum confidence score |
+| `ner_batch_size` | `8` | Inference batch size (must be >= 1) |
 | `ner_models` | `None` | Language-keyed dict of ONNX model repo IDs |
 | `torch_ner_models` | `None` | Language-keyed dict of PyTorch model repo IDs |
 | `gliner_model` | `None` | GLiNER model ID (required for gliner/ensemble backends) |
@@ -421,6 +461,7 @@ new_cfg = dataclasses.replace(cfg, check_ner_process=False)
 | `gliner_label_map` | `None` | Maps GLiNER labels to NER tags |
 | `gliner_threshold` | `0.4` | GLiNER confidence threshold |
 | `fuzzy_date_score_cutoff` | `85` | Fuzzy matching threshold (0-100) for misspelled months |
+| `custom_pipeline_steps` | `()` | Tuple of `(text: str) -> str` callables appended after all built-in steps |
 
 **Language settings**:
 
@@ -466,44 +507,30 @@ Input Text
 (lm_text, stat_text, language)
 ```
 
-Each step is toggled by a `TextCleanerConfig` field. The pipeline is built once at initialization — disabled steps are skipped entirely (zero overhead).
-
-## What's New in v0.4.5
+Each step is toggled by a `TextCleanerConfig` field. The pipeline is built once at initialization; disabled steps are skipped entirely (zero overhead).
 
-Major release with architectural overhaul since v0.3.0:
+## What's New
 
-**Architecture**
-- Frozen `TextCleanerConfig` dataclass replaces global mutable config (thread-safe, per-instance)
-- ONNX-first NER inference — torch-free base install (~400MB models vs ~7GB)
-- Thread-parallel batch processing via `ThreadPoolExecutor` (ONNX releases the GIL)
+**v0.5.x**
+- `aprocess_batch()`: async batch processing for FastAPI / aiohttp integrations
+- `warmup(languages)`: pre-load NER models at startup to eliminate first-request latency
+- `custom_pipeline_steps`: attach arbitrary `(text: str) -> str` callables after the built-in pipeline
+- French, Portuguese, and Italian NER support via a shared multilingual ONNX session
+- Improved NER sentence boundary detection with abbreviation guard
 
-**NER**
-- 5 backends: `onnx`, `torch`, `gliner`, `ensemble_onnx`, `ensemble_torch`
+**v0.4.5**
+- Frozen `TextCleanerConfig` dataclass: immutable, thread-safe, per-instance configuration
+- ONNX-first NER inference: torch-free base install (~400 MB models vs ~7 GB)
+- Thread-parallel batch processing via `ThreadPoolExecutor`
+- Five NER backends: `onnx`, `torch`, `gliner`, `ensemble_onnx`, `ensemble_torch`
 - GLiNER zero-shot NER for custom entity types (PRODUCT, EVENT, SKILL, etc.)
 - Ensemble voting across backends for improved recall
-- Lazy per-language model loading (only loads models when needed)
-- Language-keyed model dict replaces fragile positional tuple
-- ONNX-quantized models hosted on [HuggingFace Hub](https://huggingface.co/rhnfzl)
-
-**Text Processing**
-- Multilingual date detection (ISO 8601, European formats, month names in EN/NL/DE/ES)
-- Fuzzy date matching for misspelled months (via rapidfuzz, empirically calibrated threshold)
-- Configurable emoji removal
-- Configurable bracket/brace content removal
-- Smart case folding (preserves NER replacement tokens)
-- Custom stopwords and month names per language
-
-**Dependencies**
-- `stop-words` package replaces NLTK (50KB bundled vs 30MB download)
-- PyTorch/Transformers moved to optional `[torch]` extra
-- New optional extras: `[gpu]`, `[fuzzy]`, `[gliner]`, `[gliner2]`, `[all-ner]`
-- Migrated from `setup.py` to `pyproject.toml` (PEP 517)
-
-**Quality**
-- Python 3.11–3.13 support
-- `ruff` linter (replaces flake8)
-- hypothesis-based property testing with pytest-timeout
-- Collision-safe NER entity keys
+- Lazy per-language model loading
+- Multilingual date detection and fuzzy date matching
+- Configurable emoji removal, bracket/brace content removal, and smart case folding
+- `stop-words` replaces NLTK (50 KB bundled vs 30 MB download)
+- PyTorch and Transformers moved to optional extras
+- Migrated to `pyproject.toml` (PEP 517), Python 3.11-3.13, ruff linter
 
 ## Contributing
 

{squeakycleantext-0.5.0 → squeakycleantext-0.5.2}/README.md

@@ -11,21 +11,25 @@
 A comprehensive text cleaning and preprocessing pipeline for machine learning and NLP tasks.
 </div>
 
+> **Using an AI coding assistant?** This repo includes an [`llms.txt`](https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt) with the full API surface, config reference, and Q&A - optimised for Claude, Cursor, Copilot, and ChatGPT.
+
 In the world of machine learning and natural language processing, clean and well-structured text data is crucial for building effective downstream models and managing token limits in language models.
 
-SqueakyCleanText simplifies the process by automatically addressing common text issues, ensuring your data is clean and well-structured with minimal effort on your part.
+SqueakyCleanText simplifies the process by automatically addressing common text issues - removing PII, anonymizing named entities (persons, organisations, locations), and ensuring your data is clean and well-structured for language models and classical ML pipelines with minimal effort on your part.
 
 ### Key Features
 
 - **Named Entity Recognition (NER)**:
   - Multi-backend: ONNX (default, torch-free), PyTorch, GLiNER, and ensemble modes
   - Zero-shot custom entities via GLiNER (e.g., PRODUCT, EVENT, SKILL)
-  - Multi-language support (English, Dutch, German, Spanish)
+  - Multi-language support (English, Dutch, German, Spanish, French, Portuguese, Italian)
   - Ensemble voting across backends for improved accuracy
   - Configurable confidence thresholds
   - Lazy model loading (models load on demand per language)
-  - Automatic text chunking for long documents
+  - Shared ONNX sessions across same-model languages (~600 MB RAM saved)
+  - Automatic text chunking for long documents (CJK/Arabic safe)
   - GPU acceleration support (CUDA for ONNX and PyTorch)
+  - Model warm-up API to pre-load on startup
 - **Text Normalization**:
   - Corrects text encoding problems and handles bad Unicode characters
   - Removes or replaces HTML tags and URLs with configurable tokens
@@ -40,7 +44,7 @@ SqueakyCleanText simplifies the process by automatically addressing common text
   - Smart case folding (preserves NER tokens like `<PERSON>`)
 - **Language Support**:
   - Automatic language detection (English, Dutch, German, Spanish)
-  - Language-specific NER models
+  - Language-specific NER models; French, Portuguese, Italian via multilingual model
   - Language-aware stopword removal
   - Extensible: add custom languages with stopwords, month names, and NER models
 - **Dual Output Formats**:
@@ -49,7 +53,9 @@ SqueakyCleanText simplifies the process by automatically addressing common text
 - **Performance**:
   - ONNX Runtime inference (torch-free base install, ~3-5x faster than PyTorch)
   - Thread-parallel batch processing via `ThreadPoolExecutor`
+  - Async batch processing (`aprocess_batch`) for FastAPI / aiohttp
   - Lazy model loading (only loads models as needed)
+  - Shared ONNX sessions for same-model languages (saves ~600 MB for FR/PT/IT)
   - Memory-efficient processing of large texts
   - GPU acceleration (CUDA) for both ONNX and PyTorch backends
 
@@ -82,7 +88,7 @@ SqueakyCleanText simplifies the process by automatically addressing common text
 pip install SqueakyCleanText
 ```
 
-The base install uses **ONNX Runtime** for NER inference — no PyTorch or Transformers required.
+The base install uses **ONNX Runtime** for NER inference - no PyTorch or Transformers required.
 
 ### Optional Extras
 
@@ -157,7 +163,7 @@ cfg = TextCleanerConfig(
     gliner_labels=('person', 'organization', 'location', 'product', 'event'),
     gliner_label_map={
         'person': 'PER', 'organization': 'ORG', 'location': 'LOC',
-        # 'product' and 'event' are unmapped — they become <PRODUCT>, <EVENT> tokens
+        # 'product' and 'event' are unmapped - they become <PRODUCT>, <EVENT> tokens
     },
     gliner_threshold=0.4,
 )
@@ -247,7 +253,7 @@ SqueakyCleanText supports five NER backends, selectable via the `ner_backend` co
 
 | Backend | Description | Dependencies | Best for |
 |---------|-------------|-------------|----------|
-| `onnx` (default) | ONNX Runtime inference with quantized XLM-RoBERTa models | Base install | Production — fast, torch-free |
+| `onnx` (default) | ONNX Runtime inference with quantized XLM-RoBERTa models | Base install | Production: fast, torch-free |
 | `torch` | PyTorch/Transformers pipeline with full XLM-RoBERTa models | `[torch]` extra | Compatibility with existing PyTorch workflows |
 | `gliner` | GLiNER zero-shot NER with custom entity labels | `[gliner]` or `[gliner2]` extra | Custom entity types (PRODUCT, SKILL, EVENT, etc.) |
 | `ensemble_onnx` | ONNX + GLiNER ensemble voting | `[gliner]` extra | Maximum recall with custom entities |
@@ -261,7 +267,8 @@ SqueakyCleanText supports five NER backends, selectable via the `ner_backend` co
 | Dutch | [`rhnfzl/xlm-roberta-large-conll02-dutch-onnx`](https://huggingface.co/rhnfzl/xlm-roberta-large-conll02-dutch-onnx) |
 | German | [`rhnfzl/xlm-roberta-large-conll03-german-onnx`](https://huggingface.co/rhnfzl/xlm-roberta-large-conll03-german-onnx) |
 | Spanish | [`rhnfzl/xlm-roberta-large-conll02-spanish-onnx`](https://huggingface.co/rhnfzl/xlm-roberta-large-conll02-spanish-onnx) |
-| Multilingual | [`rhnfzl/wikineural-multilingual-ner-onnx`](https://huggingface.co/rhnfzl/wikineural-multilingual-ner-onnx) |
+| French / Portuguese / Italian | [`rhnfzl/wikineural-multilingual-ner-onnx`](https://huggingface.co/rhnfzl/wikineural-multilingual-ner-onnx) (shared session) |
+| Multilingual (fallback) | [`rhnfzl/wikineural-multilingual-ner-onnx`](https://huggingface.co/rhnfzl/wikineural-multilingual-ner-onnx) |
 
 ### GLiNER Label Mapping
 
@@ -292,6 +299,28 @@ Processes the input text and returns a tuple containing:
 
 Processes multiple texts using thread-parallel execution. Each result follows the same format as `process()`.
 
+#### `aprocess_batch(texts: List[str], batch_size: int = None) -> List[Tuple[str, Optional[str], Optional[str]]]`
+
+Async version of `process_batch` for use with asyncio-based frameworks (FastAPI, aiohttp). Runs the batch in a thread-pool executor so it does not block the event loop:
+
+```python
+from sct import TextCleaner
+
+cleaner = TextCleaner()
+
+# In an async context (FastAPI route, aiohttp handler, etc.)
+results = await cleaner.aprocess_batch(texts)
+```
+
+#### `warmup(languages: Optional[List[str]] = None) -> None`
+
+Pre-loads NER models to avoid first-request latency. Call once during application startup:
+
+```python
+cleaner = TextCleaner()
+cleaner.warmup(['ENGLISH', 'DUTCH'])  # or warmup() for all supported languages
+```
+
 ### `TextCleanerConfig`
 
 Immutable (frozen) dataclass. Create modified copies with `dataclasses.replace()`:
@@ -354,6 +383,7 @@ new_cfg = dataclasses.replace(cfg, check_ner_process=False)
 | `ner_backend` | `'onnx'` | Backend: `onnx`, `torch`, `gliner`, `ensemble_onnx`, `ensemble_torch` |
 | `positional_tags` | `('PER', 'LOC', 'ORG', 'MISC')` | Entity types to recognize |
 | `ner_confidence_threshold` | `0.85` | Minimum confidence score |
+| `ner_batch_size` | `8` | Inference batch size (must be >= 1) |
 | `ner_models` | `None` | Language-keyed dict of ONNX model repo IDs |
 | `torch_ner_models` | `None` | Language-keyed dict of PyTorch model repo IDs |
 | `gliner_model` | `None` | GLiNER model ID (required for gliner/ensemble backends) |
@@ -362,6 +392,7 @@ new_cfg = dataclasses.replace(cfg, check_ner_process=False)
 | `gliner_label_map` | `None` | Maps GLiNER labels to NER tags |
 | `gliner_threshold` | `0.4` | GLiNER confidence threshold |
 | `fuzzy_date_score_cutoff` | `85` | Fuzzy matching threshold (0-100) for misspelled months |
+| `custom_pipeline_steps` | `()` | Tuple of `(text: str) -> str` callables appended after all built-in steps |
 
 **Language settings**:
 
@@ -407,44 +438,30 @@ Input Text
 (lm_text, stat_text, language)
 ```
 
-Each step is toggled by a `TextCleanerConfig` field. The pipeline is built once at initialization — disabled steps are skipped entirely (zero overhead).
-
-## What's New in v0.4.5
+Each step is toggled by a `TextCleanerConfig` field. The pipeline is built once at initialization; disabled steps are skipped entirely (zero overhead).
 
-Major release with architectural overhaul since v0.3.0:
+## What's New
 
-**Architecture**
-- Frozen `TextCleanerConfig` dataclass replaces global mutable config (thread-safe, per-instance)
-- ONNX-first NER inference — torch-free base install (~400MB models vs ~7GB)
-- Thread-parallel batch processing via `ThreadPoolExecutor` (ONNX releases the GIL)
+**v0.5.x**
+- `aprocess_batch()`: async batch processing for FastAPI / aiohttp integrations
+- `warmup(languages)`: pre-load NER models at startup to eliminate first-request latency
+- `custom_pipeline_steps`: attach arbitrary `(text: str) -> str` callables after the built-in pipeline
+- French, Portuguese, and Italian NER support via a shared multilingual ONNX session
+- Improved NER sentence boundary detection with abbreviation guard
 
-**NER**
-- 5 backends: `onnx`, `torch`, `gliner`, `ensemble_onnx`, `ensemble_torch`
+**v0.4.5**
+- Frozen `TextCleanerConfig` dataclass: immutable, thread-safe, per-instance configuration
+- ONNX-first NER inference: torch-free base install (~400 MB models vs ~7 GB)
+- Thread-parallel batch processing via `ThreadPoolExecutor`
+- Five NER backends: `onnx`, `torch`, `gliner`, `ensemble_onnx`, `ensemble_torch`
 - GLiNER zero-shot NER for custom entity types (PRODUCT, EVENT, SKILL, etc.)
 - Ensemble voting across backends for improved recall
-- Lazy per-language model loading (only loads models when needed)
-- Language-keyed model dict replaces fragile positional tuple
-- ONNX-quantized models hosted on [HuggingFace Hub](https://huggingface.co/rhnfzl)
-
-**Text Processing**
-- Multilingual date detection (ISO 8601, European formats, month names in EN/NL/DE/ES)
-- Fuzzy date matching for misspelled months (via rapidfuzz, empirically calibrated threshold)
-- Configurable emoji removal
-- Configurable bracket/brace content removal
-- Smart case folding (preserves NER replacement tokens)
-- Custom stopwords and month names per language
-
-**Dependencies**
-- `stop-words` package replaces NLTK (50KB bundled vs 30MB download)
-- PyTorch/Transformers moved to optional `[torch]` extra
-- New optional extras: `[gpu]`, `[fuzzy]`, `[gliner]`, `[gliner2]`, `[all-ner]`
-- Migrated from `setup.py` to `pyproject.toml` (PEP 517)
-
-**Quality**
-- Python 3.11–3.13 support
-- `ruff` linter (replaces flake8)
-- hypothesis-based property testing with pytest-timeout
-- Collision-safe NER entity keys
+- Lazy per-language model loading
+- Multilingual date detection and fuzzy date matching
+- Configurable emoji removal, bracket/brace content removal, and smart case folding
+- `stop-words` replaces NLTK (50 KB bundled vs 30 MB download)
+- PyTorch and Transformers moved to optional extras
+- Migrated to `pyproject.toml` (PEP 517), Python 3.11-3.13, ruff linter
 
 ## Contributing
 

{squeakycleantext-0.5.0 → squeakycleantext-0.5.2}/SqueakyCleanText.egg-info/PKG-INFO

@@ -1,19 +1,29 @@
 Metadata-Version: 2.4
 Name: SqueakyCleanText
-Version: 0.5.0
-Summary: A comprehensive text cleaning and preprocessing pipeline.
+Version: 0.5.2
+Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
 Author: Rehan Fazal
 License: MIT
 Project-URL: Homepage, https://github.com/rhnfzl/SqueakyCleanText
-Keywords: text cleaning,text preprocessing,NLP,natural language processing
+Project-URL: Repository, https://github.com/rhnfzl/SqueakyCleanText
+Project-URL: Issues, https://github.com/rhnfzl/SqueakyCleanText/issues
+Project-URL: Changelog, https://github.com/rhnfzl/SqueakyCleanText/releases
+Project-URL: llms.txt, https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt
+Keywords: text cleaning,text preprocessing,NLP,natural language processing,named entity recognition,NER,anonymization,PII removal,data cleaning,machine learning,ONNX,language detection
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Natural Language :: English
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Classifier: Topic :: Software Development :: Libraries
-Classifier: Topic :: Text Processing
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -70,21 +80,25 @@ Dynamic: license-file
 A comprehensive text cleaning and preprocessing pipeline for machine learning and NLP tasks.
 </div>
 
+> **Using an AI coding assistant?** This repo includes an [`llms.txt`](https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt) with the full API surface, config reference, and Q&A - optimised for Claude, Cursor, Copilot, and ChatGPT.
+
 In the world of machine learning and natural language processing, clean and well-structured text data is crucial for building effective downstream models and managing token limits in language models.
 
-SqueakyCleanText simplifies the process by automatically addressing common text issues, ensuring your data is clean and well-structured with minimal effort on your part.
+SqueakyCleanText simplifies the process by automatically addressing common text issues - removing PII, anonymizing named entities (persons, organisations, locations), and ensuring your data is clean and well-structured for language models and classical ML pipelines with minimal effort on your part.
 
 ### Key Features
 
 - **Named Entity Recognition (NER)**:
   - Multi-backend: ONNX (default, torch-free), PyTorch, GLiNER, and ensemble modes
   - Zero-shot custom entities via GLiNER (e.g., PRODUCT, EVENT, SKILL)
-  - Multi-language support (English, Dutch, German, Spanish)
+  - Multi-language support (English, Dutch, German, Spanish, French, Portuguese, Italian)
   - Ensemble voting across backends for improved accuracy
   - Configurable confidence thresholds
   - Lazy model loading (models load on demand per language)
-  - Automatic text chunking for long documents
+  - Shared ONNX sessions across same-model languages (~600 MB RAM saved)
+  - Automatic text chunking for long documents (CJK/Arabic safe)
   - GPU acceleration support (CUDA for ONNX and PyTorch)
+  - Model warm-up API to pre-load on startup
 - **Text Normalization**:
   - Corrects text encoding problems and handles bad Unicode characters
   - Removes or replaces HTML tags and URLs with configurable tokens
@@ -99,7 +113,7 @@ SqueakyCleanText simplifies the process by automatically addressing common text
   - Smart case folding (preserves NER tokens like `<PERSON>`)
 - **Language Support**:
   - Automatic language detection (English, Dutch, German, Spanish)
-  - Language-specific NER models
+  - Language-specific NER models; French, Portuguese, Italian via multilingual model
   - Language-aware stopword removal
   - Extensible: add custom languages with stopwords, month names, and NER models
 - **Dual Output Formats**:
@@ -108,7 +122,9 @@ SqueakyCleanText simplifies the process by automatically addressing common text
 - **Performance**:
   - ONNX Runtime inference (torch-free base install, ~3-5x faster than PyTorch)
   - Thread-parallel batch processing via `ThreadPoolExecutor`
+  - Async batch processing (`aprocess_batch`) for FastAPI / aiohttp
   - Lazy model loading (only loads models as needed)
+  - Shared ONNX sessions for same-model languages (saves ~600 MB for FR/PT/IT)
   - Memory-efficient processing of large texts
   - GPU acceleration (CUDA) for both ONNX and PyTorch backends
 
@@ -141,7 +157,7 @@ SqueakyCleanText simplifies the process by automatically addressing common text
 pip install SqueakyCleanText
 ```
 
-The base install uses **ONNX Runtime** for NER inference — no PyTorch or Transformers required.
+The base install uses **ONNX Runtime** for NER inference - no PyTorch or Transformers required.
 
 ### Optional Extras
 
@@ -216,7 +232,7 @@ cfg = TextCleanerConfig(
     gliner_labels=('person', 'organization', 'location', 'product', 'event'),
     gliner_label_map={
         'person': 'PER', 'organization': 'ORG', 'location': 'LOC',
-        # 'product' and 'event' are unmapped — they become <PRODUCT>, <EVENT> tokens
+        # 'product' and 'event' are unmapped - they become <PRODUCT>, <EVENT> tokens
     },
     gliner_threshold=0.4,
 )
@@ -306,7 +322,7 @@ SqueakyCleanText supports five NER backends, selectable via the `ner_backend` co
 
 | Backend | Description | Dependencies | Best for |
 |---------|-------------|-------------|----------|
-| `onnx` (default) | ONNX Runtime inference with quantized XLM-RoBERTa models | Base install | Production — fast, torch-free |
+| `onnx` (default) | ONNX Runtime inference with quantized XLM-RoBERTa models | Base install | Production: fast, torch-free |
 | `torch` | PyTorch/Transformers pipeline with full XLM-RoBERTa models | `[torch]` extra | Compatibility with existing PyTorch workflows |
 | `gliner` | GLiNER zero-shot NER with custom entity labels | `[gliner]` or `[gliner2]` extra | Custom entity types (PRODUCT, SKILL, EVENT, etc.) |
 | `ensemble_onnx` | ONNX + GLiNER ensemble voting | `[gliner]` extra | Maximum recall with custom entities |
@@ -320,7 +336,8 @@ SqueakyCleanText supports five NER backends, selectable via the `ner_backend` co
 | Dutch | [`rhnfzl/xlm-roberta-large-conll02-dutch-onnx`](https://huggingface.co/rhnfzl/xlm-roberta-large-conll02-dutch-onnx) |
 | German | [`rhnfzl/xlm-roberta-large-conll03-german-onnx`](https://huggingface.co/rhnfzl/xlm-roberta-large-conll03-german-onnx) |
 | Spanish | [`rhnfzl/xlm-roberta-large-conll02-spanish-onnx`](https://huggingface.co/rhnfzl/xlm-roberta-large-conll02-spanish-onnx) |
-| Multilingual | [`rhnfzl/wikineural-multilingual-ner-onnx`](https://huggingface.co/rhnfzl/wikineural-multilingual-ner-onnx) |
+| French / Portuguese / Italian | [`rhnfzl/wikineural-multilingual-ner-onnx`](https://huggingface.co/rhnfzl/wikineural-multilingual-ner-onnx) (shared session) |
+| Multilingual (fallback) | [`rhnfzl/wikineural-multilingual-ner-onnx`](https://huggingface.co/rhnfzl/wikineural-multilingual-ner-onnx) |
 
 ### GLiNER Label Mapping
 
@@ -351,6 +368,28 @@ Processes the input text and returns a tuple containing:
 
 Processes multiple texts using thread-parallel execution. Each result follows the same format as `process()`.
 
+#### `aprocess_batch(texts: List[str], batch_size: int = None) -> List[Tuple[str, Optional[str], Optional[str]]]`
+
+Async version of `process_batch` for use with asyncio-based frameworks (FastAPI, aiohttp). Runs the batch in a thread-pool executor so it does not block the event loop:
+
+```python
+from sct import TextCleaner
+
+cleaner = TextCleaner()
+
+# In an async context (FastAPI route, aiohttp handler, etc.)
+results = await cleaner.aprocess_batch(texts)
+```
+
+#### `warmup(languages: Optional[List[str]] = None) -> None`
+
+Pre-loads NER models to avoid first-request latency. Call once during application startup:
+
+```python
+cleaner = TextCleaner()
+cleaner.warmup(['ENGLISH', 'DUTCH'])  # or warmup() for all supported languages
+```
+
 ### `TextCleanerConfig`
 
 Immutable (frozen) dataclass. Create modified copies with `dataclasses.replace()`:
@@ -413,6 +452,7 @@ new_cfg = dataclasses.replace(cfg, check_ner_process=False)
 | `ner_backend` | `'onnx'` | Backend: `onnx`, `torch`, `gliner`, `ensemble_onnx`, `ensemble_torch` |
 | `positional_tags` | `('PER', 'LOC', 'ORG', 'MISC')` | Entity types to recognize |
 | `ner_confidence_threshold` | `0.85` | Minimum confidence score |
+| `ner_batch_size` | `8` | Inference batch size (must be >= 1) |
 | `ner_models` | `None` | Language-keyed dict of ONNX model repo IDs |
 | `torch_ner_models` | `None` | Language-keyed dict of PyTorch model repo IDs |
 | `gliner_model` | `None` | GLiNER model ID (required for gliner/ensemble backends) |
@@ -421,6 +461,7 @@ new_cfg = dataclasses.replace(cfg, check_ner_process=False)
 | `gliner_label_map` | `None` | Maps GLiNER labels to NER tags |
 | `gliner_threshold` | `0.4` | GLiNER confidence threshold |
 | `fuzzy_date_score_cutoff` | `85` | Fuzzy matching threshold (0-100) for misspelled months |
+| `custom_pipeline_steps` | `()` | Tuple of `(text: str) -> str` callables appended after all built-in steps |
 
 **Language settings**:
 
@@ -466,44 +507,30 @@ Input Text
 (lm_text, stat_text, language)
 ```
 
-Each step is toggled by a `TextCleanerConfig` field. The pipeline is built once at initialization — disabled steps are skipped entirely (zero overhead).
-
-## What's New in v0.4.5
+Each step is toggled by a `TextCleanerConfig` field. The pipeline is built once at initialization; disabled steps are skipped entirely (zero overhead).
 
-Major release with architectural overhaul since v0.3.0:
+## What's New
 
-**Architecture**
-- Frozen `TextCleanerConfig` dataclass replaces global mutable config (thread-safe, per-instance)
-- ONNX-first NER inference — torch-free base install (~400MB models vs ~7GB)
-- Thread-parallel batch processing via `ThreadPoolExecutor` (ONNX releases the GIL)
+**v0.5.x**
+- `aprocess_batch()`: async batch processing for FastAPI / aiohttp integrations
+- `warmup(languages)`: pre-load NER models at startup to eliminate first-request latency
+- `custom_pipeline_steps`: attach arbitrary `(text: str) -> str` callables after the built-in pipeline
+- French, Portuguese, and Italian NER support via a shared multilingual ONNX session
+- Improved NER sentence boundary detection with abbreviation guard
 
-**NER**
-- 5 backends: `onnx`, `torch`, `gliner`, `ensemble_onnx`, `ensemble_torch`
+**v0.4.5**
+- Frozen `TextCleanerConfig` dataclass: immutable, thread-safe, per-instance configuration
+- ONNX-first NER inference: torch-free base install (~400 MB models vs ~7 GB)
+- Thread-parallel batch processing via `ThreadPoolExecutor`
+- Five NER backends: `onnx`, `torch`, `gliner`, `ensemble_onnx`, `ensemble_torch`
 - GLiNER zero-shot NER for custom entity types (PRODUCT, EVENT, SKILL, etc.)
 - Ensemble voting across backends for improved recall
-- Lazy per-language model loading (only loads models when needed)
-- Language-keyed model dict replaces fragile positional tuple
-- ONNX-quantized models hosted on [HuggingFace Hub](https://huggingface.co/rhnfzl)
-
-**Text Processing**
-- Multilingual date detection (ISO 8601, European formats, month names in EN/NL/DE/ES)
-- Fuzzy date matching for misspelled months (via rapidfuzz, empirically calibrated threshold)
-- Configurable emoji removal
-- Configurable bracket/brace content removal
-- Smart case folding (preserves NER replacement tokens)
-- Custom stopwords and month names per language
-
-**Dependencies**
-- `stop-words` package replaces NLTK (50KB bundled vs 30MB download)
-- PyTorch/Transformers moved to optional `[torch]` extra
-- New optional extras: `[gpu]`, `[fuzzy]`, `[gliner]`, `[gliner2]`, `[all-ner]`
-- Migrated from `setup.py` to `pyproject.toml` (PEP 517)
-
-**Quality**
-- Python 3.11–3.13 support
-- `ruff` linter (replaces flake8)
-- hypothesis-based property testing with pytest-timeout
-- Collision-safe NER entity keys
+- Lazy per-language model loading
+- Multilingual date detection and fuzzy date matching
+- Configurable emoji removal, bracket/brace content removal, and smart case folding
+- `stop-words` replaces NLTK (50 KB bundled vs 30 MB download)
+- PyTorch and Transformers moved to optional extras
+- Migrated to `pyproject.toml` (PEP 517), Python 3.11-3.13, ruff linter
 
 ## Contributing
 

{squeakycleantext-0.5.0 → squeakycleantext-0.5.2}/pyproject.toml

@@ -4,22 +4,32 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "SqueakyCleanText"
-version = "0.5.0"
-description = "A comprehensive text cleaning and preprocessing pipeline."
+dynamic = ["version"]
+description = "Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement."
 readme = "README.md"
 license = {text = "MIT"}
 authors = [{name = "Rehan Fazal"}]
 requires-python = ">=3.11"
-keywords = ["text cleaning", "text preprocessing", "NLP", "natural language processing"]
+keywords = [
+    "text cleaning", "text preprocessing", "NLP", "natural language processing",
+    "named entity recognition", "NER", "anonymization", "PII removal",
+    "data cleaning", "machine learning", "ONNX", "language detection",
+]
 classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Topic :: Text Processing :: Linguistic",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Natural Language :: English",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
-    "Topic :: Software Development :: Libraries",
-    "Topic :: Text Processing",
 ]
 dependencies = [
     "lingua-language-detector>=2.0.2",
@@ -38,6 +48,10 @@ dependencies = [
 
 [project.urls]
 Homepage = "https://github.com/rhnfzl/SqueakyCleanText"
+Repository = "https://github.com/rhnfzl/SqueakyCleanText"
+Issues = "https://github.com/rhnfzl/SqueakyCleanText/issues"
+Changelog = "https://github.com/rhnfzl/SqueakyCleanText/releases"
+"llms.txt" = "https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt"
 
 [project.optional-dependencies]
 gpu = [
@@ -75,6 +89,9 @@ test = [
     "pytest-cov>=4.1",
 ]
 
+[tool.setuptools.dynamic]
+version = {attr = "sct.__version__"}
+
 [tool.setuptools.packages.find]
 include = ["sct*"]
 exclude = ["tests*"]

{squeakycleantext-0.5.0 → squeakycleantext-0.5.2}/sct/__init__.py

@@ -3,4 +3,5 @@
 from sct.config import TextCleanerConfig
 from sct.sct import TextCleaner
 
+__version__ = "0.5.2"
 __all__ = ["TextCleaner", "TextCleanerConfig"]