lexiredact 0.0.2__py3-none-any.whl

lexiredact/README.md

@@ -0,0 +1,9 @@
+# Lexiredact
+
+Privacy-preserving RAG ingestion middleware with dual-pipeline processing.
+
+```bash
+pip install -e ".[dev]"          # core + dev tools
+pip install -e ".[pii,embed]"    # add Presidio + sentence-transformers
+pip install -e ".[all]"          # everything
+```

lexiredact/__init__.py

@@ -0,0 +1,41 @@
+"""
+LexiRedact — Privacy-preserving RAG ingestion middleware with dual-pipeline processing.
+
+Import everything you need from here:
+    from lexiredact import load_config, ProcessingResult, configure_logging
+"""
+
+from __future__ import annotations
+
+from lexiredact.config.loader import load_config
+from lexiredact.pipeline_api import LexiredactPipeline
+from lexiredact.config.schema import LexiredactConfig
+from lexiredact.exceptions import (
+    LexiredactCacheError,
+    LexiredactConfigError,
+    LexiredactError,
+    LexiredactInputError,
+    LexiredactStorageError,
+)
+from lexiredact.app_logging import configure_logging, get_logger
+from lexiredact.models.chunk import Chunk
+from lexiredact.models.result import DetectedEntity, ProcessingResult
+
+__version__ = "0.0.2"
+
+__all__ = [
+    "LexiredactConfig",
+    "LexiredactPipeline",
+    "load_config",
+    "ProcessingResult",
+    "DetectedEntity",
+    "Chunk",
+    "LexiredactError",
+    "LexiredactConfigError",
+    "LexiredactInputError",
+    "LexiredactStorageError",
+    "LexiredactCacheError",
+    "configure_logging",
+    "get_logger",
+    "__version__",
+]

lexiredact/adapters/__init__.py

@@ -0,0 +1,5 @@
+"""
+adapters — Input adapters that translate caller-supplied dicts to internal models.
+
+- chunk_adapter.py — ChunkAdapter: maps raw dicts → Chunk using InputSchemaConfig.
+"""

lexiredact/adapters/chunk_adapter.py

@@ -0,0 +1,93 @@
+"""
+adapters/chunk_adapter.py — Maps raw user input dicts to internal Chunk objects.
+
+Field names are never hardcoded here; they are always read from InputSchemaConfig.
+This is the only place in the codebase that performs per-chunk input validation.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from lexiredact.config.schema import InputSchemaConfig
+from lexiredact.exceptions import LexiredactInputError
+from lexiredact.app_logging import get_logger
+from lexiredact.models.chunk import Chunk
+
+logger = get_logger(__name__)
+
+
+class ChunkAdapter:
+    """Converts arbitrary user input dicts into validated Chunk objects.
+
+    Args:
+        config: Field-name mapping configuration from InputSchemaConfig.
+    """
+
+    def __init__(self, config: InputSchemaConfig) -> None:
+        self._config = config
+
+    def adapt(self, raw: dict[str, Any]) -> Chunk:
+        """Convert a single raw input dict into a Chunk.
+
+        Raises:
+            LexiredactInputError: If id_field missing, text_field missing,
+                or text is empty after stripping whitespace.
+        """
+        cfg = self._config
+
+        if cfg.id_field not in raw:
+            raise LexiredactInputError(
+                f"Missing required id field '{cfg.id_field}' in input dict.",
+                context={
+                    "expected_id_field": cfg.id_field,
+                    "available_keys": list(raw.keys()),
+                },
+            )
+
+        chunk_id = str(raw[cfg.id_field])
+
+        if cfg.text_field not in raw:
+            raise LexiredactInputError(
+                f"Missing required text field '{cfg.text_field}' in chunk '{chunk_id}'.",
+                context={
+                    "expected_text_field": cfg.text_field,
+                    "available_keys": list(raw.keys()),
+                    "chunk_id": chunk_id,
+                },
+            )
+
+        text: str = raw[cfg.text_field]
+        metadata: dict[str, Any] = {
+            key: raw[key] for key in cfg.metadata_fields if key in raw
+        }
+
+        # Chunk.__post_init__ raises LexiredactInputError if text is empty.
+        return Chunk(id=chunk_id, text=text, metadata=metadata)
+
+    def adapt_batch(
+        self, raws: list[dict[str, Any]]
+    ) -> tuple[list[Chunk], list[dict[str, Any]]]:
+        """Convert a list of raw dicts, collecting — not raising — per-item errors.
+
+        Returns:
+            (successful_chunks, failed_items)
+            failed_items entries: {"index": int, "error": str, "raw": dict}
+        """
+        successful_chunks: list[Chunk] = []
+        failed_items: list[dict[str, Any]] = []
+
+        for index, raw in enumerate(raws):
+            try:
+                successful_chunks.append(self.adapt(raw))
+            except LexiredactInputError as exc:
+                logger.warning("Chunk at index %d failed adaptation: %s", index, exc)
+                failed_items.append({"index": index, "error": str(exc), "raw": raw})
+
+        if failed_items:
+            logger.warning(
+                "adapt_batch: %d/%d chunks failed. Proceeding with %d successful.",
+                len(failed_items), len(raws), len(successful_chunks),
+            )
+
+        return successful_chunks, failed_items

lexiredact/app_logging.py

@@ -0,0 +1,57 @@
+"""
+logging.py — Centralised logging configuration for lexiredact.
+
+Registers a NullHandler at import time so that library consumers who have
+not configured logging do not receive "No handlers could be found" warnings.
+
+Usage inside library modules:
+    from lexiredact.logging import get_logger
+    logger = get_logger(__name__)
+"""
+
+from __future__ import annotations
+
+import logging
+
+# Register NullHandler at module-import time so LexiRedact is a well-behaved library.
+logging.getLogger("lexiredact").addHandler(logging.NullHandler())
+
+_LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s"
+
+
+def configure_logging(level: str = "INFO") -> None:
+    """Configure the root ``LexiRedact`` logger with a StreamHandler.
+
+    Adds a single StreamHandler pointing to stderr with the standard format.
+    Safe to call multiple times — will not add duplicate handlers.
+
+    Args:
+        level: Logging level string, e.g. "DEBUG", "INFO", "WARNING".
+    """
+    root_logger = logging.getLogger("lexiredact")
+    numeric_level = getattr(logging, level.upper(), logging.INFO)
+    root_logger.setLevel(numeric_level)
+
+    for handler in root_logger.handlers:
+        if isinstance(handler, logging.StreamHandler) and not isinstance(
+            handler, logging.FileHandler
+        ):
+            return
+
+    handler = logging.StreamHandler()
+    handler.setLevel(numeric_level)
+    handler.setFormatter(logging.Formatter(_LOG_FORMAT))
+    root_logger.addHandler(handler)
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Return a child logger scoped under the ``LexiRedact`` namespace.
+
+    Args:
+        name: Typically __name__ of the calling module. The "lexiredact."
+              prefix is added automatically.
+
+    Returns:
+        A logging.Logger instance namespaced under lexiredact.
+    """
+    return logging.getLogger(f"lexiredact.{name}")

lexiredact/cache/__init__.py

@@ -0,0 +1,11 @@
+"""
+cache — Redis-backed embedding cache for lexiredact.
+
+  EmbeddingCache — transparent cache layer keyed on SHA-256 of chunk text.
+                   Any Redis failure is caught silently; callers never see exceptions.
+                   Disabled entirely when CacheConfig.enabled=False.
+"""
+
+from lexiredact.cache.redis_cache import EmbeddingCache
+
+__all__ = ["EmbeddingCache"]

lexiredact/cache/redis_cache.py

@@ -0,0 +1,98 @@
+"""
+cache/redis_cache.py — Redis-backed embedding cache.
+
+Failure contract (critical):
+  - LexiredactCacheError is raised internally but NEVER propagated to callers.
+  - Any Redis error, JSON decode error, or connection failure results in a cache
+    miss (get → None) or a silent no-op (set → return). The pipeline continues.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+
+from lexiredact.config.schema import CacheConfig
+from lexiredact.exceptions import LexiredactCacheError
+from lexiredact.app_logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class EmbeddingCache:
+    """Redis-backed embedding cache. Completely transparent to callers.
+
+    When config.enabled is False, every method is a no-op and get always
+    returns None. No Redis connection is attempted.
+
+    Args:
+        config: Cache configuration (redis_url, ttl_seconds, key_prefix, enabled).
+    """
+
+    def __init__(self, config: CacheConfig) -> None:
+        self._config = config
+        self._client = None  # redis.Redis instance; created lazily on first use.
+        logger.debug("EmbeddingCache initialized (enabled=%s)", config.enabled)
+
+    def get(self, text: str) -> list[float] | None:
+        """Return cached embedding for text or None on any miss/error."""
+        if not self._config.enabled:
+            return None
+        key = self._make_key(text)
+        try:
+            self._ensure_connected()
+            raw = self._client.get(key)  # type: ignore[union-attr]
+            if raw is None:
+                return None
+            vector: list[float] = json.loads(raw)
+            if not isinstance(vector, list):
+                raise LexiredactCacheError(
+                    "Cached value is not a list",
+                    context={"key": key, "got_type": type(vector).__name__},
+                )
+            return vector
+        except LexiredactCacheError as exc:
+            logger.warning("Cache get failed for key '%s': %s", key, exc)
+            return None
+        except json.JSONDecodeError as exc:
+            logger.warning("Cache get: JSON decode error for key '%s': %s", key, exc)
+            return None
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Cache get: Redis error for key '%s': %s", key, exc)
+            return None
+
+    def set(self, text: str, vector: list[float]) -> None:
+        """Store embedding in Redis with configured TTL. Silent on any error."""
+        if not self._config.enabled:
+            return
+        key = self._make_key(text)
+        try:
+            self._ensure_connected()
+            self._client.setex(key, self._config.ttl_seconds, json.dumps(vector))  # type: ignore[union-attr]
+            logger.debug("Cache set: stored %d-dim vector at key '%s'.", len(vector), key)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Cache set failed for key '%s': %s", key, exc)
+
+    def _make_key(self, text: str) -> str:
+        """Build Redis key: {prefix}:emb:{sha256(text)[:16]}"""
+        hash_fragment = hashlib.sha256(text.encode()).hexdigest()[:16]
+        return f"{self._config.key_prefix}:emb:{hash_fragment}"
+
+    def _ensure_connected(self) -> None:
+        """Lazily initialise the Redis client. Raises LexiredactCacheError on failure."""
+        if self._client is not None:
+            return
+        try:
+            import redis  # type: ignore[import-untyped]
+            self._client = redis.Redis.from_url(
+                self._config.redis_url,
+                decode_responses=True,
+                socket_connect_timeout=2,
+                socket_timeout=2,
+            )
+            logger.debug("Redis client connected to %s.", self._config.redis_url)
+        except Exception as exc:
+            raise LexiredactCacheError(
+                "Failed to initialise Redis client",
+                context={"redis_url": self._config.redis_url, "error": str(exc)},
+            ) from exc