lexiredact 0.1.0__tar.gz

lexiredact-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Baihela Abid Hussain
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

lexiredact-0.1.0/MANIFEST.in

@@ -0,0 +1,19 @@
+include README.md
+include LICENSE*
+
+recursive-include lexiredact *.py
+recursive-include lexiredact *.typed
+
+prune backend
+prune benchmarks
+prune data
+prune dist
+prune .tmp-build
+prune lexiredact_data
+prune venv
+
+global-exclude *.py[cod]
+global-exclude __pycache__
+global-exclude *.so
+exclude mlflow.db
+exclude requirements-backend.txt

lexiredact-0.1.0/PKG-INFO

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: lexiredact
+Version: 0.1.0
+Summary: Privacy-First Vector Database for Sensitive Data
+Author-email: Shwetan Londhe <shwetan.college@gmail.com>, Varad Limbkar <varadlimbkar@gmail.com>, Baihela Husain <baihelahusain@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/lexiredact/lexiredact
+Project-URL: Repository, https://github.com/lexiredact/lexiredact
+Project-URL: Documentation, https://github.com/lexiredact/lexiredact#documentation
+Project-URL: Issues, https://github.com/lexiredact/lexiredact/issues
+Keywords: pii,privacy,vector-database,embedding,rag
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: presidio-analyzer>=2.2.0
+Requires-Dist: presidio-anonymizer>=2.2.0
+Requires-Dist: fastembed>=0.2.0
+Requires-Dist: chromadb>=0.4.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: numpy>=1.24.0
+Provides-Extra: pdf
+Requires-Dist: pypdf>=4.0.0; extra == "pdf"
+Provides-Extra: redis
+Requires-Dist: redis[async]>=5.0.0; extra == "redis"
+Provides-Extra: mlflow
+Requires-Dist: mlflow>=2.10.0; extra == "mlflow"
+Provides-Extra: all
+Requires-Dist: pypdf>=4.0.0; extra == "all"
+Requires-Dist: redis[async]>=5.0.0; extra == "all"
+Requires-Dist: mlflow>=2.10.0; extra == "all"
+Dynamic: license-file
+
+# LexiRedact
+
+LexiRedact is a Python package for privacy-first document ingestion in RAG and vector database workflows. It detects PII, redacts sensitive text before storage, and preserves retrieval quality by generating embeddings from the original text while storing only sanitized content.
+
+## Install
+
+```bash
+pip install lexiredact
+```
+
+Optional extras:
+
+```bash
+pip install "lexiredact[pdf]"
+pip install "lexiredact[redis]"
+pip install "lexiredact[mlflow]"
+pip install "lexiredact[all]"
+```
+
+## What It Focuses On
+
+- PII detection with Presidio
+- safe redaction before vector-store persistence
+- configurable ingestion pipeline components
+- operational metrics for privacy and latency
+- optional retrieval evaluation helpers for model comparison
+
+## Quick Start
+
+```python
+import asyncio
+import lexiredact as lr
+
+
+async def main() -> None:
+    pipeline = lr.IngestionPipeline()
+    await pipeline.initialize()
+
+    result = await pipeline.process_document(
+        lr.Document(
+            id="doc-1",
+            text="Contact Jane Doe at jane@example.com or 555-0101",
+            metadata={"source": "demo"},
+        )
+    )
+
+    print(result.clean_text)
+    print(result.pii_entities)
+
+    await pipeline.shutdown()
+
+
+asyncio.run(main())
+```
+
+## Docs And Examples
+
+- docs: [`docs/`](./docs)
+- examples: [`examples/`](./examples)

lexiredact-0.1.0/README.md

@@ -0,0 +1,59 @@
+# LexiRedact
+
+LexiRedact is a Python package for privacy-first document ingestion in RAG and vector database workflows. It detects PII, redacts sensitive text before storage, and preserves retrieval quality by generating embeddings from the original text while storing only sanitized content.
+
+## Install
+
+```bash
+pip install lexiredact
+```
+
+Optional extras:
+
+```bash
+pip install "lexiredact[pdf]"
+pip install "lexiredact[redis]"
+pip install "lexiredact[mlflow]"
+pip install "lexiredact[all]"
+```
+
+## What It Focuses On
+
+- PII detection with Presidio
+- safe redaction before vector-store persistence
+- configurable ingestion pipeline components
+- operational metrics for privacy and latency
+- optional retrieval evaluation helpers for model comparison
+
+## Quick Start
+
+```python
+import asyncio
+import lexiredact as lr
+
+
+async def main() -> None:
+    pipeline = lr.IngestionPipeline()
+    await pipeline.initialize()
+
+    result = await pipeline.process_document(
+        lr.Document(
+            id="doc-1",
+            text="Contact Jane Doe at jane@example.com or 555-0101",
+            metadata={"source": "demo"},
+        )
+    )
+
+    print(result.clean_text)
+    print(result.pii_entities)
+
+    await pipeline.shutdown()
+
+
+asyncio.run(main())
+```
+
+## Docs And Examples
+
+- docs: [`docs/`](./docs)
+- examples: [`examples/`](./examples)

lexiredact-0.1.0/lexiredact/__init__.py

@@ -0,0 +1,139 @@
+"""
+LexiRedact - Privacy-Preserving RAG Middleware
+
+A Python SDK for protecting PII in vector databases while maintaining
+semantic search quality through intelligent embedding and redaction.
+
+Key Features:
+- Automatic PII detection and redaction using Microsoft Presidio
+- Embedding generation from original text (Shadow Mode architecture)
+- Only sanitized text stored in vector databases
+- Redis caching for performance optimization
+- Pluggable architecture via dependency injection
+- Comprehensive metrics and tracking
+
+Basic Usage:
+    >>> import lexiredact as vs
+    >>> 
+    >>> # Create pipeline with defaults
+    >>> pipeline = vs.IngestionPipeline()
+    >>> await pipeline.initialize()
+    >>> 
+    >>> # Process documents
+    >>> doc = vs.Document(id="1", text="Contact John at john@example.com")
+    >>> result = await pipeline.process_document(doc)
+    >>> 
+    >>> print(result.clean_text)  # "Contact <PERSON> at <EMAIL_ADDRESS>"
+    >>> print(result.pii_entities)  # ["PERSON", "EMAIL_ADDRESS"]
+    >>> 
+    >>> await pipeline.shutdown()
+
+Custom Configuration:
+    >>> from lexiredact import IngestionPipeline, load_config
+    >>> 
+    >>> config = load_config(config_dict={
+    ...     "embedding_model": "BAAI/bge-base-en-v1.5",
+    ...     "cache_backend": "redis",
+    ...     "redis_host": "localhost"
+    ... })
+    >>> 
+    >>> pipeline = IngestionPipeline(config=config)
+
+Custom Components:
+    >>> from lexiredact import IngestionPipeline
+    >>> from lexiredact.interfaces import Embedder
+    >>> 
+    >>> class MyEmbedder(Embedder):
+    ...     # Custom implementation
+    ...     pass
+    >>> 
+    >>> pipeline = IngestionPipeline(embedder=MyEmbedder())
+"""
+
+__version__ = "0.1.0"
+
+# Core pipeline
+from .pipeline import IngestionPipeline, Document, ProcessedDocument
+
+# Configuration
+from .config import load_config, get_default_config, save_config_to_yaml
+
+# Privacy components
+from .privacy import PIIDetector, PIIRedactor, PIIPolicy
+
+# Interfaces (for custom implementations)
+from .interfaces import CacheBackend, Embedder, VectorStore, Tracker
+
+# Default implementations
+from .implementations import (
+    MemoryCache,  
+    RedisCache,
+    GenericCache,
+    FastEmbedEmbedder,
+    GenericEmbedder,  
+    ChromaVectorStore,
+    GenericVectorStore, 
+    MLflowTracker,
+)
+
+# Metrics
+from .metrics import (
+    MetricsCollector,
+    AggregateStats,
+    RetrievalAggregateStats,
+    RetrievalMetricsEvaluator,
+    RetrievalQueryMetrics,
+)
+
+# Utilities
+from .utils import hash_text, generate_cache_key, Timer
+
+__all__ = [
+    # Version
+    "__version__",
+    
+    # Core
+    "IngestionPipeline",
+    "Document",
+    "ProcessedDocument",
+    
+    # Configuration
+    "load_config",
+    "get_default_config",
+    "save_config_to_yaml",
+    
+    # Privacy
+    "PIIDetector",
+    "PIIRedactor",
+    "PIIPolicy",
+    
+    # Interfaces
+    "CacheBackend",
+    "Embedder",
+    "VectorStore",
+    "Tracker",
+    
+    #custom models
+    "GenericCache",
+    "GenericEmbedder", 
+    "GenericVectorStore",
+    
+    # Implementations
+    "MemoryCache",
+    "RedisCache",
+    "FastEmbedEmbedder",
+    "ChromaVectorStore",
+    "MLflowTracker",
+    
+    # Metrics
+    "MetricsCollector",
+    "AggregateStats",
+    "RetrievalAggregateStats",
+    "RetrievalMetricsEvaluator",
+    "RetrievalQueryMetrics",
+    
+    # Utils
+    "hash_text",
+    "generate_cache_key",
+    "Timer",
+]

lexiredact-0.1.0/lexiredact/chunking/__init__.py

@@ -0,0 +1,18 @@
+"""
+Document chunking module for LexiRedact.
+
+Converts PDFs and large text documents into manageable chunks
+suitable for embedding and PII detection.
+"""
+
+from .chunker import Chunk, DocumentChunker, ChunkingStrategy
+from .json_exporter import JSONExporter
+from .pdf_loader import PDFLoader
+
+__all__ = [
+    "Chunk",
+    "DocumentChunker",
+    "ChunkingStrategy",
+    "PDFLoader",
+    "JSONExporter",
+]

lexiredact-0.1.0/lexiredact/chunking/chunker.py

@@ -0,0 +1,321 @@
+"""
+Core document chunking logic.
+Splits large documents into smaller chunks with overlap.
+"""
+
+from typing import List, Dict, Any, Optional, Literal
+from dataclasses import dataclass
+import uuid
+from enum import Enum
+import re
+
+
+class ChunkingStrategy(str, Enum):
+    """Chunking strategies."""
+    FIXED_SIZE = "fixed_size"          # Fixed token/char chunks
+    SENTENCE = "sentence"               # Split by sentences
+    PARAGRAPH = "paragraph"             # Split by paragraphs
+    HYBRID = "hybrid"                   # Sentences grouped into chunks
+
+
+@dataclass
+class Chunk:
+    """Single chunk of text."""
+    id: str
+    text: str
+    chunk_index: int
+    start_char: int
+    end_char: int
+    metadata: Dict[str, Any]
+
+
+class DocumentChunker:
+    """
+    Convert large documents into LexiRedact-compatible chunks.
+
+    Support for:
+    - Fixed-size chunking (tokens or characters)
+    - Sentence-based chunking
+    - Paragraph-based chunking
+    - Overlap between chunks
+    """
+
+    def __init__(
+        self,
+        chunk_size: int = 512,              # Max characters per chunk
+        overlap: int = 100,                 # Overlap between chunks (chars)
+        strategy: ChunkingStrategy = ChunkingStrategy.FIXED_SIZE,
+        preserve_sentences: bool = True,    # Don't split mid-sentence
+    ):
+        """
+        Initialize chunker.
+
+        Args:
+            chunk_size: Target chunk size in characters
+            overlap: Overlap between chunks (to preserve context)
+            strategy: Chunking strategy to use
+            preserve_sentences: Don't split in middle of sentence
+        """
+        if chunk_size <= 0:
+            raise ValueError("chunk_size must be greater than 0")
+        if overlap < 0:
+            raise ValueError("overlap must be greater than or equal to 0")
+        if overlap >= chunk_size:
+            raise ValueError("overlap must be smaller than chunk_size")
+
+        self.chunk_size = chunk_size
+        self.overlap = overlap
+        self.strategy = strategy
+        self.preserve_sentences = preserve_sentences
+
+    def chunk_text(
+        self,
+        text: str,
+        doc_id: str,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> List[Chunk]:
+        """
+        Chunk a document.
+
+        Args:
+            text: Document text to chunk
+            doc_id: Original document ID (source)
+            metadata: Optional metadata to attach to chunks
+
+        Returns:
+            List of Chunk objects
+        """
+        metadata = metadata or {}
+
+        if self.strategy == ChunkingStrategy.FIXED_SIZE:
+            return self._chunk_fixed_size(text, doc_id, metadata)
+        elif self.strategy == ChunkingStrategy.SENTENCE:
+            return self._chunk_by_sentence(text, doc_id, metadata)
+        elif self.strategy == ChunkingStrategy.PARAGRAPH:
+            return self._chunk_by_paragraph(text, doc_id, metadata)
+        elif self.strategy == ChunkingStrategy.HYBRID:
+            return self._chunk_hybrid(text, doc_id, metadata)
+        else:
+            raise ValueError(f"Unknown strategy: {self.strategy}")
+
+    def _chunk_fixed_size(
+        self,
+        text: str,
+        doc_id: str,
+        metadata: Dict[str, Any]
+    ) -> List[Chunk]:
+        """Split into fixed-size chunks with overlap."""
+        chunks = []
+        chunk_index = 0
+        start = 0
+
+        while start < len(text):
+            end = min(start + self.chunk_size, len(text))
+
+            # If preserve_sentences, adjust end to not split mid-sentence
+            if self.preserve_sentences and end < len(text):
+                window = text[start:end]
+                boundaries = list(re.finditer(r"[.!?](?=\s|$)|\n", window))
+                if boundaries:
+                    end = start + boundaries[-1].end()
+
+            chunk_text = text[start:end].strip()
+
+            if chunk_text:  # Skip empty chunks
+                chunk = Chunk(
+                    id=f"{doc_id}_chunk_{chunk_index}",
+                    text=chunk_text,
+                    chunk_index=chunk_index,
+                    start_char=start,
+                    end_char=end,
+                    metadata={
+                        **metadata,
+                        "source_doc_id": doc_id,
+                        "chunk_number": chunk_index,
+                        "strategy": self.strategy.value,
+                    }
+                )
+                chunks.append(chunk)
+                chunk_index += 1
+
+            if end >= len(text):
+                break
+
+            # Move start position (with overlap)
+            start = end - self.overlap
+
+        return chunks
+
+    def _chunk_by_sentence(
+        self,
+        text: str,
+        doc_id: str,
+        metadata: Dict[str, Any]
+    ) -> List[Chunk]:
+        """Split by sentences while preserving punctuation-heavy tokens."""
+        sentences = self._split_sentences(text)
+        chunks = []
+        chunk_index = 0
+
+        current_chunk = []
+        current_size = 0
+        start_char = 0
+
+        for sentence in sentences:
+            sentence_size = len(sentence)
+
+            # If adding this sentence exceeds chunk_size, save current chunk
+            if current_size + sentence_size > self.chunk_size and current_chunk:
+                chunk_text = ' '.join(current_chunk).strip()
+                if chunk_text:
+                    chunk = Chunk(
+                        id=f"{doc_id}_chunk_{chunk_index}",
+                        text=chunk_text,
+                        chunk_index=chunk_index,
+                        start_char=start_char,
+                        end_char=start_char + len(chunk_text),
+                        metadata={
+                            **metadata,
+                            "source_doc_id": doc_id,
+                            "chunk_number": chunk_index,
+                            "strategy": self.strategy.value,
+                        }
+                    )
+                    chunks.append(chunk)
+                    chunk_index += 1
+
+                start_char += len(chunk_text) + 1
+                current_chunk = [sentence]
+                current_size = sentence_size
+            else:
+                current_chunk.append(sentence)
+                current_size += sentence_size
+
+        # Add remaining chunk
+        if current_chunk:
+            chunk_text = ' '.join(current_chunk).strip()
+            chunk = Chunk(
+                id=f"{doc_id}_chunk_{chunk_index}",
+                text=chunk_text,
+                chunk_index=chunk_index,
+                start_char=start_char,
+                end_char=start_char + len(chunk_text),
+                metadata={
+                    **metadata,
+                    "source_doc_id": doc_id,
+                    "chunk_number": chunk_index,
+                    "strategy": self.strategy.value,
+                }
+            )
+            chunks.append(chunk)
+
+        return chunks
+
+    def _chunk_by_paragraph(
+        self,
+        text: str,
+        doc_id: str,
+        metadata: Dict[str, Any]
+    ) -> List[Chunk]:
+        """Split by paragraphs (double newline)."""
+        paragraphs = text.split('\n\n')
+        chunks = []
+        chunk_index = 0
+        start_char = 0
+
+        for para in paragraphs:
+            para = para.strip()
+            if para:
+                chunk = Chunk(
+                    id=f"{doc_id}_chunk_{chunk_index}",
+                    text=para,
+                    chunk_index=chunk_index,
+                    start_char=start_char,
+                    end_char=start_char + len(para),
+                    metadata={
+                        **metadata,
+                        "source_doc_id": doc_id,
+                        "chunk_number": chunk_index,
+                        "strategy": self.strategy.value,
+                    }
+                )
+                chunks.append(chunk)
+                chunk_index += 1
+                start_char += len(para) + 2  # +2 for '\n\n'
+
+        return chunks
+
+    def _chunk_hybrid(
+        self,
+        text: str,
+        doc_id: str,
+        metadata: Dict[str, Any]
+    ) -> List[Chunk]:
+        """
+        Hybrid: Group sentences into chunks of target size.
+        Better than fixed_size because sentences stay together.
+        """
+        sentences = self._split_sentences(text)
+        chunks = []
+        chunk_index = 0
+        start_char = 0
+
+        current_chunk = []
+        current_size = 0
+
+        for sentence in sentences:
+            # If adding sentence exceeds size and we have content, save chunk
+            if current_size + len(sentence) > self.chunk_size and current_chunk:
+                chunk_text = ' '.join(current_chunk).strip()
+                chunk = Chunk(
+                    id=f"{doc_id}_chunk_{chunk_index}",
+                    text=chunk_text,
+                    chunk_index=chunk_index,
+                    start_char=start_char,
+                    end_char=start_char + len(chunk_text),
+                    metadata={
+                        **metadata,
+                        "source_doc_id": doc_id,
+                        "chunk_number": chunk_index,
+                        "strategy": self.strategy.value,
+                    }
+                )
+                chunks.append(chunk)
+                chunk_index += 1
+                start_char += len(chunk_text) + 1
+                current_chunk = [sentence]
+                current_size = len(sentence)
+            else:
+                current_chunk.append(sentence)
+                current_size += len(sentence)
+
+        # Add final chunk
+        if current_chunk:
+            chunk_text = ' '.join(current_chunk).strip()
+            chunk = Chunk(
+                id=f"{doc_id}_chunk_{chunk_index}",
+                text=chunk_text,
+                chunk_index=chunk_index,
+                start_char=start_char,
+                end_char=start_char + len(chunk_text),
+                metadata={
+                    **metadata,
+                    "source_doc_id": doc_id,
+                    "chunk_number": chunk_index,
+                    "strategy": self.strategy.value,
+                }
+            )
+            chunks.append(chunk)
+
+        return chunks
+
+    def _split_sentences(self, text: str) -> List[str]:
+        """
+        Split text on sentence boundaries without breaking emails or domains.
+        """
+        normalized = text.strip()
+        if not normalized:
+            return []
+
+        parts = re.split(r"(?<=[.!?])\s+(?=[A-Z0-9])", normalized)
+        return [part.strip() for part in parts if part.strip()]