lexiredact 0.0.2__tar.gz

lexiredact-0.0.2/.gitignore

@@ -0,0 +1,13 @@
+.venv/
+venv/
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+build/
+dist/
+chroma_db/
+.env
+.DS_Store

lexiredact-0.0.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Baihela Abid Hussain, Shwetan Bharat Londhe, Varad Limbkar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

lexiredact-0.0.2/PKG-INFO

@@ -0,0 +1,95 @@
+Metadata-Version: 2.4
+Name: lexiredact
+Version: 0.0.2
+Summary: Privacy-preserving RAG ingestion middleware with dual-pipeline processing
+Project-URL: Homepage, https://github.com/baihelahusain/lexiredact
+Project-URL: Repository, https://github.com/baihelahusain/LexiRedact
+Project-URL: Issues, https://github.com/baihelahusain/LexiRedact/issues
+Project-URL: Documentation, https://github.com/baihelahusain/LexiRedact
+Author: Baihela Abid Hussain, Shwetan Bharat Londhe, Varad Limbkar
+License: MIT
+License-File: LICENSE
+Keywords: embeddings,llm,pii,privacy,rag,redaction,retrieval-augmented-generation,security,vector-database
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Security
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Requires-Dist: pydantic<3.0,>=2.0
+Requires-Dist: pyyaml<7.0,>=6.0
+Provides-Extra: all
+Requires-Dist: chromadb>=0.4; extra == 'all'
+Requires-Dist: click>=8.0; extra == 'all'
+Requires-Dist: presidio-analyzer; extra == 'all'
+Requires-Dist: presidio-anonymizer; extra == 'all'
+Requires-Dist: redis>=5.0; extra == 'all'
+Requires-Dist: sentence-transformers>=2.0; extra == 'all'
+Requires-Dist: spacy; extra == 'all'
+Provides-Extra: cache
+Requires-Dist: redis>=5.0; extra == 'cache'
+Provides-Extra: cli
+Requires-Dist: click>=8.0; extra == 'cli'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: embed
+Requires-Dist: sentence-transformers>=2.0; extra == 'embed'
+Provides-Extra: pii
+Requires-Dist: presidio-analyzer; extra == 'pii'
+Requires-Dist: presidio-anonymizer; extra == 'pii'
+Requires-Dist: spacy; extra == 'pii'
+Provides-Extra: store
+Requires-Dist: chromadb>=0.4; extra == 'store'
+Description-Content-Type: text/markdown
+
+# LexiRedact
+
+LexiRedact is a privacy-preserving RAG ingestion middleware package for detecting and redacting PII before content is embedded and written to a vector store.
+
+## Installation
+
+```bash
+pip install lexiredact
+```
+
+Optional dependency groups are available for specific integrations:
+
+```bash
+pip install "lexiredact[pii,embed,store,cache,cli]"
+```
+
+## Quick Start
+
+```python
+from lexiredact import LexiredactPipeline, load_config
+
+config = load_config("lexiredact_config.yaml")
+pipeline = LexiredactPipeline(config)
+```
+
+## CLI
+
+```bash
+lexiredact --help
+```
+
+## Release
+
+This project publishes to PyPI from GitHub Actions when a version tag is pushed:
+
+```bash
+git tag v0.0.2
+git push origin v0.0.2
+```
+
+The PyPI project must be configured for Trusted Publishing with the `pypi` environment and the `.github/workflows/publish.yml` workflow.

lexiredact-0.0.2/README.md

@@ -0,0 +1,41 @@
+# LexiRedact
+
+LexiRedact is a privacy-preserving RAG ingestion middleware package for detecting and redacting PII before content is embedded and written to a vector store.
+
+## Installation
+
+```bash
+pip install lexiredact
+```
+
+Optional dependency groups are available for specific integrations:
+
+```bash
+pip install "lexiredact[pii,embed,store,cache,cli]"
+```
+
+## Quick Start
+
+```python
+from lexiredact import LexiredactPipeline, load_config
+
+config = load_config("lexiredact_config.yaml")
+pipeline = LexiredactPipeline(config)
+```
+
+## CLI
+
+```bash
+lexiredact --help
+```
+
+## Release
+
+This project publishes to PyPI from GitHub Actions when a version tag is pushed:
+
+```bash
+git tag v0.0.2
+git push origin v0.0.2
+```
+
+The PyPI project must be configured for Trusted Publishing with the `pypi` environment and the `.github/workflows/publish.yml` workflow.

lexiredact-0.0.2/lexiredact/README.md

@@ -0,0 +1,9 @@
+# Lexiredact
+
+Privacy-preserving RAG ingestion middleware with dual-pipeline processing.
+
+```bash
+pip install -e ".[dev]"          # core + dev tools
+pip install -e ".[pii,embed]"    # add Presidio + sentence-transformers
+pip install -e ".[all]"          # everything
+```

lexiredact-0.0.2/lexiredact/__init__.py

@@ -0,0 +1,41 @@
+"""
+LexiRedact — Privacy-preserving RAG ingestion middleware with dual-pipeline processing.
+
+Import everything you need from here:
+    from lexiredact import load_config, ProcessingResult, configure_logging
+"""
+
+from __future__ import annotations
+
+from lexiredact.config.loader import load_config
+from lexiredact.pipeline_api import LexiredactPipeline
+from lexiredact.config.schema import LexiredactConfig
+from lexiredact.exceptions import (
+    LexiredactCacheError,
+    LexiredactConfigError,
+    LexiredactError,
+    LexiredactInputError,
+    LexiredactStorageError,
+)
+from lexiredact.app_logging import configure_logging, get_logger
+from lexiredact.models.chunk import Chunk
+from lexiredact.models.result import DetectedEntity, ProcessingResult
+
+__version__ = "0.0.2"
+
+__all__ = [
+    "LexiredactConfig",
+    "LexiredactPipeline",
+    "load_config",
+    "ProcessingResult",
+    "DetectedEntity",
+    "Chunk",
+    "LexiredactError",
+    "LexiredactConfigError",
+    "LexiredactInputError",
+    "LexiredactStorageError",
+    "LexiredactCacheError",
+    "configure_logging",
+    "get_logger",
+    "__version__",
+]

lexiredact-0.0.2/lexiredact/adapters/__init__.py

@@ -0,0 +1,5 @@
+"""
+adapters — Input adapters that translate caller-supplied dicts to internal models.
+
+- chunk_adapter.py — ChunkAdapter: maps raw dicts → Chunk using InputSchemaConfig.
+"""

lexiredact-0.0.2/lexiredact/adapters/chunk_adapter.py

@@ -0,0 +1,93 @@
+"""
+adapters/chunk_adapter.py — Maps raw user input dicts to internal Chunk objects.
+
+Field names are never hardcoded here; they are always read from InputSchemaConfig.
+This is the only place in the codebase that performs per-chunk input validation.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from lexiredact.config.schema import InputSchemaConfig
+from lexiredact.exceptions import LexiredactInputError
+from lexiredact.app_logging import get_logger
+from lexiredact.models.chunk import Chunk
+
+logger = get_logger(__name__)
+
+
+class ChunkAdapter:
+    """Converts arbitrary user input dicts into validated Chunk objects.
+
+    Args:
+        config: Field-name mapping configuration from InputSchemaConfig.
+    """
+
+    def __init__(self, config: InputSchemaConfig) -> None:
+        self._config = config
+
+    def adapt(self, raw: dict[str, Any]) -> Chunk:
+        """Convert a single raw input dict into a Chunk.
+
+        Raises:
+            LexiredactInputError: If id_field missing, text_field missing,
+                or text is empty after stripping whitespace.
+        """
+        cfg = self._config
+
+        if cfg.id_field not in raw:
+            raise LexiredactInputError(
+                f"Missing required id field '{cfg.id_field}' in input dict.",
+                context={
+                    "expected_id_field": cfg.id_field,
+                    "available_keys": list(raw.keys()),
+                },
+            )
+
+        chunk_id = str(raw[cfg.id_field])
+
+        if cfg.text_field not in raw:
+            raise LexiredactInputError(
+                f"Missing required text field '{cfg.text_field}' in chunk '{chunk_id}'.",
+                context={
+                    "expected_text_field": cfg.text_field,
+                    "available_keys": list(raw.keys()),
+                    "chunk_id": chunk_id,
+                },
+            )
+
+        text: str = raw[cfg.text_field]
+        metadata: dict[str, Any] = {
+            key: raw[key] for key in cfg.metadata_fields if key in raw
+        }
+
+        # Chunk.__post_init__ raises LexiredactInputError if text is empty.
+        return Chunk(id=chunk_id, text=text, metadata=metadata)
+
+    def adapt_batch(
+        self, raws: list[dict[str, Any]]
+    ) -> tuple[list[Chunk], list[dict[str, Any]]]:
+        """Convert a list of raw dicts, collecting — not raising — per-item errors.
+
+        Returns:
+            (successful_chunks, failed_items)
+            failed_items entries: {"index": int, "error": str, "raw": dict}
+        """
+        successful_chunks: list[Chunk] = []
+        failed_items: list[dict[str, Any]] = []
+
+        for index, raw in enumerate(raws):
+            try:
+                successful_chunks.append(self.adapt(raw))
+            except LexiredactInputError as exc:
+                logger.warning("Chunk at index %d failed adaptation: %s", index, exc)
+                failed_items.append({"index": index, "error": str(exc), "raw": raw})
+
+        if failed_items:
+            logger.warning(
+                "adapt_batch: %d/%d chunks failed. Proceeding with %d successful.",
+                len(failed_items), len(raws), len(successful_chunks),
+            )
+
+        return successful_chunks, failed_items

lexiredact-0.0.2/lexiredact/app_logging.py

@@ -0,0 +1,57 @@
+"""
+logging.py — Centralised logging configuration for lexiredact.
+
+Registers a NullHandler at import time so that library consumers who have
+not configured logging do not receive "No handlers could be found" warnings.
+
+Usage inside library modules:
+    from lexiredact.logging import get_logger
+    logger = get_logger(__name__)
+"""
+
+from __future__ import annotations
+
+import logging
+
+# Register NullHandler at module-import time so LexiRedact is a well-behaved library.
+logging.getLogger("lexiredact").addHandler(logging.NullHandler())
+
+_LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
+
+
+def configure_logging(level: str = "INFO") -> None:
+    """Configure the root ``LexiRedact`` logger with a StreamHandler.
+
+    Adds a single StreamHandler pointing to stderr with the standard format.
+    Safe to call multiple times — will not add duplicate handlers.
+
+    Args:
+        level: Logging level string, e.g. "DEBUG", "INFO", "WARNING".
+    """
+    root_logger = logging.getLogger("lexiredact")
+    numeric_level = getattr(logging, level.upper(), logging.INFO)
+    root_logger.setLevel(numeric_level)
+
+    for handler in root_logger.handlers:
+        if isinstance(handler, logging.StreamHandler) and not isinstance(
+            handler, logging.FileHandler
+        ):
+            return
+
+    handler = logging.StreamHandler()
+    handler.setLevel(numeric_level)
+    handler.setFormatter(logging.Formatter(_LOG_FORMAT))
+    root_logger.addHandler(handler)
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Return a child logger scoped under the ``LexiRedact`` namespace.
+
+    Args:
+        name: Typically __name__ of the calling module. The "lexiredact."
+              prefix is added automatically.
+
+    Returns:
+        A logging.Logger instance namespaced under lexiredact.
+    """
+    return logging.getLogger(f"lexiredact.{name}")

lexiredact-0.0.2/lexiredact/cache/__init__.py

@@ -0,0 +1,11 @@
+"""
+cache — Redis-backed embedding cache for lexiredact.
+
+  EmbeddingCache — transparent cache layer keyed on SHA-256 of chunk text.
+                   Any Redis failure is caught silently; callers never see exceptions.
+                   Disabled entirely when CacheConfig.enabled=False.
+"""
+
+from lexiredact.cache.redis_cache import EmbeddingCache
+
+__all__ = ["EmbeddingCache"]

lexiredact-0.0.2/lexiredact/cache/redis_cache.py

@@ -0,0 +1,98 @@
+"""
+cache/redis_cache.py — Redis-backed embedding cache.
+
+Failure contract (critical):
+  - LexiredactCacheError is raised internally but NEVER propagated to callers.
+  - Any Redis error, JSON decode error, or connection failure results in a cache
+    miss (get → None) or a silent no-op (set → return). The pipeline continues.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+
+from lexiredact.config.schema import CacheConfig
+from lexiredact.exceptions import LexiredactCacheError
+from lexiredact.app_logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class EmbeddingCache:
+    """Redis-backed embedding cache. Completely transparent to callers.
+
+    When config.enabled is False, every method is a no-op and get always
+    returns None. No Redis connection is attempted.
+
+    Args:
+        config: Cache configuration (redis_url, ttl_seconds, key_prefix, enabled).
+    """
+
+    def __init__(self, config: CacheConfig) -> None:
+        self._config = config
+        self._client = None  # redis.Redis instance; created lazily on first use.
+        logger.debug("EmbeddingCache initialized (enabled=%s)", config.enabled)
+
+    def get(self, text: str) -> list[float] | None:
+        """Return cached embedding for text or None on any miss/error."""
+        if not self._config.enabled:
+            return None
+        key = self._make_key(text)
+        try:
+            self._ensure_connected()
+            raw = self._client.get(key)  # type: ignore[union-attr]
+            if raw is None:
+                return None
+            vector: list[float] = json.loads(raw)
+            if not isinstance(vector, list):
+                raise LexiredactCacheError(
+                    "Cached value is not a list",
+                    context={"key": key, "got_type": type(vector).__name__},
+                )
+            return vector
+        except LexiredactCacheError as exc:
+            logger.warning("Cache get failed for key '%s': %s", key, exc)
+            return None
+        except json.JSONDecodeError as exc:
+            logger.warning("Cache get: JSON decode error for key '%s': %s", key, exc)
+            return None
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Cache get: Redis error for key '%s': %s", key, exc)
+            return None
+
+    def set(self, text: str, vector: list[float]) -> None:
+        """Store embedding in Redis with configured TTL. Silent on any error."""
+        if not self._config.enabled:
+            return
+        key = self._make_key(text)
+        try:
+            self._ensure_connected()
+            self._client.setex(key, self._config.ttl_seconds, json.dumps(vector))  # type: ignore[union-attr]
+            logger.debug("Cache set: stored %d-dim vector at key '%s'.", len(vector), key)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Cache set failed for key '%s': %s", key, exc)
+
+    def _make_key(self, text: str) -> str:
+        """Build Redis key: {prefix}:emb:{sha256(text)[:16]}"""
+        hash_fragment = hashlib.sha256(text.encode()).hexdigest()[:16]
+        return f"{self._config.key_prefix}:emb:{hash_fragment}"
+
+    def _ensure_connected(self) -> None:
+        """Lazily initialise the Redis client. Raises LexiredactCacheError on failure."""
+        if self._client is not None:
+            return
+        try:
+            import redis  # type: ignore[import-untyped]
+            self._client = redis.Redis.from_url(
+                self._config.redis_url,
+                decode_responses=True,
+                socket_connect_timeout=2,
+                socket_timeout=2,
+            )
+            logger.debug("Redis client connected to %s.", self._config.redis_url)
+        except Exception as exc:
+            raise LexiredactCacheError(
+                "Failed to initialise Redis client",
+                context={"redis_url": self._config.redis_url, "error": str(exc)},
+            ) from exc