daytashield 0.1.1__py3-none-any.whl

daytashield/core/router.py

@@ -0,0 +1,217 @@
+"""Data routing based on validation results."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Callable
+
+from pydantic import BaseModel, Field
+
+from daytashield.core.result import ValidationResult, ValidationStatus
+
+
+class RouteAction(str, Enum):
+    """Actions that can be taken for routed data."""
+
+    PASS = "pass"  # Send to destination
+    REVIEW = "review"  # Send to review queue
+    QUARANTINE = "quarantine"  # Isolate failed data
+    RETRY = "retry"  # Attempt reprocessing
+    DROP = "drop"  # Discard the data
+
+
+class Route(BaseModel):
+    """A routing rule with condition and action."""
+
+    name: str = Field(..., description="Route name for identification")
+    action: RouteAction = Field(..., description="Action to take")
+    condition: Callable[[ValidationResult], bool] | None = Field(
+        None, description="Custom condition function"
+    )
+    destination: str | None = Field(None, description="Destination identifier")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Additional route metadata")
+
+    model_config = {"arbitrary_types_allowed": True}
+
+
+class RoutingDecision(BaseModel):
+    """The result of routing a validation result."""
+
+    route: Route = Field(..., description="The matched route")
+    result: ValidationResult = Field(..., description="The validation result")
+    reason: str = Field(..., description="Why this route was selected")
+
+    model_config = {"arbitrary_types_allowed": True}
+
+
+class RouterConfig(BaseModel):
+    """Configuration for the data router."""
+
+    default_action: RouteAction = Field(
+        RouteAction.QUARANTINE, description="Default action when no route matches"
+    )
+    include_warnings_in_review: bool = Field(
+        True, description="Route warnings to review queue"
+    )
+
+
+class DataRouter:
+    """Routes data based on validation results.
+
+    The router examines validation results and determines where data should
+    go: pass to destination, send to review, quarantine, etc.
+
+    Default routing logic:
+    - PASSED → PASS (send to destination)
+    - WARNING → REVIEW (send to review queue) if configured
+    - FAILED → QUARANTINE (isolate for investigation)
+    - ERROR → QUARANTINE
+
+    Example:
+        >>> router = DataRouter()
+        >>> result = pipeline.validate(data)
+        >>> decision = router.route(result)
+        >>> if decision.route.action == RouteAction.PASS:
+        ...     send_to_destination(result.data)
+        >>> elif decision.route.action == RouteAction.QUARANTINE:
+        ...     quarantine(result.data, decision.reason)
+    """
+
+    def __init__(
+        self,
+        routes: list[Route] | None = None,
+        config: RouterConfig | dict[str, Any] | None = None,
+    ):
+        """Initialize the router.
+
+        Args:
+            routes: Custom routes (checked before defaults)
+            config: Router configuration
+        """
+        self.custom_routes: list[Route] = routes or []
+        
+        if config is None:
+            self.config = RouterConfig()
+        elif isinstance(config, dict):
+            self.config = RouterConfig(**config)
+        else:
+            self.config = config
+
+        # Set up default routes
+        self._default_routes = self._create_default_routes()
+
+    def _create_default_routes(self) -> list[Route]:
+        """Create the default routing rules."""
+        routes = [
+            Route(
+                name="pass_valid",
+                action=RouteAction.PASS,
+                condition=lambda r: r.status == ValidationStatus.PASSED,
+            ),
+            Route(
+                name="quarantine_failed",
+                action=RouteAction.QUARANTINE,
+                condition=lambda r: r.status == ValidationStatus.FAILED,
+            ),
+            Route(
+                name="quarantine_error",
+                action=RouteAction.QUARANTINE,
+                condition=lambda r: r.status == ValidationStatus.ERROR,
+            ),
+        ]
+
+        # Add warning route based on config
+        if self.config.include_warnings_in_review:
+            routes.insert(
+                1,
+                Route(
+                    name="review_warnings",
+                    action=RouteAction.REVIEW,
+                    condition=lambda r: r.status == ValidationStatus.WARNING,
+                ),
+            )
+        else:
+            routes.insert(
+                1,
+                Route(
+                    name="pass_warnings",
+                    action=RouteAction.PASS,
+                    condition=lambda r: r.status == ValidationStatus.WARNING,
+                ),
+            )
+
+        return routes
+
+    def add_route(self, route: Route) -> DataRouter:
+        """Add a custom route.
+
+        Custom routes are checked before default routes.
+
+        Args:
+            route: The route to add
+
+        Returns:
+            Self for method chaining
+        """
+        self.custom_routes.append(route)
+        return self
+
+    def route(self, result: ValidationResult) -> RoutingDecision:
+        """Determine where to route the data.
+
+        Args:
+            result: The validation result to route
+
+        Returns:
+            RoutingDecision with the selected route
+        """
+        # Check custom routes first
+        for route in self.custom_routes:
+            if route.condition and route.condition(result):
+                return RoutingDecision(
+                    route=route,
+                    result=result,
+                    reason=f"Matched custom route: {route.name}",
+                )
+
+        # Check default routes
+        for route in self._default_routes:
+            if route.condition and route.condition(result):
+                return RoutingDecision(
+                    route=route,
+                    result=result,
+                    reason=f"Matched default route: {route.name}",
+                )
+
+        # Fallback to default action
+        fallback_route = Route(
+            name="fallback",
+            action=self.config.default_action,
+        )
+        return RoutingDecision(
+            route=fallback_route,
+            result=result,
+            reason=f"No route matched, using default action: {self.config.default_action.value}",
+        )
+
+    def route_batch(self, results: list[ValidationResult]) -> dict[RouteAction, list[RoutingDecision]]:
+        """Route multiple results and group by action.
+
+        Args:
+            results: List of validation results
+
+        Returns:
+            Dict mapping actions to lists of routing decisions
+        """
+        grouped: dict[RouteAction, list[RoutingDecision]] = {
+            action: [] for action in RouteAction
+        }
+
+        for result in results:
+            decision = self.route(result)
+            grouped[decision.route.action].append(decision)
+
+        return grouped
+
+    def __repr__(self) -> str:
+        return f"DataRouter(custom_routes={len(self.custom_routes)}, config={self.config})"

daytashield/integrations/__init__.py

@@ -0,0 +1,7 @@
+"""DaytaShield integrations with external frameworks."""
+
+from daytashield.integrations.langchain import ValidatedRetriever
+
+__all__ = [
+    "ValidatedRetriever",
+]

daytashield/integrations/langchain.py

@@ -0,0 +1,391 @@
+"""LangChain integration for validated retrieval."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any, Callable, Literal
+
+from pydantic import Field
+
+from daytashield.core.pipeline import ValidationPipeline
+from daytashield.core.result import ValidationResult, ValidationStatus, create_result
+
+if TYPE_CHECKING:
+    from langchain_core.callbacks import CallbackManagerForRetrieverRun
+    from langchain_core.documents import Document
+    from langchain_core.retrievers import BaseRetriever
+
+logger = logging.getLogger(__name__)
+
+
+OnFailAction = Literal["filter", "raise", "warn", "tag"]
+
+
+class ValidatedRetriever:
+    """A LangChain retriever wrapper that validates documents before returning.
+
+    Wraps any LangChain retriever and applies DaytaShield validation to
+    retrieved documents. Invalid documents can be filtered, flagged, or
+    cause exceptions based on configuration.
+
+    Example:
+        >>> from langchain_community.vectorstores import FAISS
+        >>> from daytashield import SchemaValidator, FreshnessValidator
+        >>> from daytashield.integrations.langchain import ValidatedRetriever
+        >>>
+        >>> # Create base retriever
+        >>> vectorstore = FAISS.from_texts(texts, embeddings)
+        >>> base_retriever = vectorstore.as_retriever()
+        >>>
+        >>> # Wrap with validation
+        >>> retriever = ValidatedRetriever(
+        ...     base_retriever=base_retriever,
+        ...     validators=[
+        ...         SchemaValidator(schema={"type": "object"}),
+        ...         FreshnessValidator(max_age="7d"),
+        ...     ],
+        ...     on_fail="filter",
+        ... )
+        >>>
+        >>> # Use like any retriever
+        >>> docs = retriever.invoke("search query")
+
+    Actions on validation failure:
+    - "filter": Remove invalid documents from results
+    - "raise": Raise ValidationError for any invalid document
+    - "warn": Log warning but include document
+    - "tag": Add validation metadata to document
+    """
+
+    def __init__(
+        self,
+        base_retriever: BaseRetriever,
+        validators: list[Any] | None = None,
+        pipeline: ValidationPipeline | None = None,
+        on_fail: OnFailAction = "filter",
+        validate_metadata: bool = True,
+        validate_content: bool = True,
+        min_confidence: float = 0.0,
+    ):
+        """Initialize the validated retriever.
+
+        Args:
+            base_retriever: The underlying LangChain retriever
+            validators: List of validators to apply (creates pipeline)
+            pipeline: Pre-configured ValidationPipeline (alternative to validators)
+            on_fail: Action when validation fails
+            validate_metadata: Validate document metadata
+            validate_content: Validate document content
+            min_confidence: Minimum confidence score to pass (for semantic validation)
+        """
+        self.base_retriever = base_retriever
+        self.on_fail = on_fail
+        self.validate_metadata = validate_metadata
+        self.validate_content = validate_content
+        self.min_confidence = min_confidence
+
+        # Set up validation pipeline
+        if pipeline is not None:
+            self.pipeline = pipeline
+        elif validators:
+            self.pipeline = ValidationPipeline(validators=validators)
+        else:
+            self.pipeline = ValidationPipeline()
+
+        # Statistics
+        self._stats = {
+            "total_retrieved": 0,
+            "total_validated": 0,
+            "total_passed": 0,
+            "total_filtered": 0,
+        }
+
+    def invoke(
+        self,
+        input: str,  # noqa: A002
+        config: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> list[Document]:
+        """Retrieve and validate documents.
+
+        Args:
+            input: The search query
+            config: Optional config for the retriever
+            **kwargs: Additional arguments for base retriever
+
+        Returns:
+            List of validated Document objects
+        """
+        # Retrieve documents from base retriever
+        docs = self.base_retriever.invoke(input, config=config, **kwargs)
+        self._stats["total_retrieved"] += len(docs)
+
+        # Validate each document
+        validated_docs: list[Document] = []
+
+        for doc in docs:
+            result = self._validate_document(doc)
+            self._stats["total_validated"] += 1
+
+            if result.passed:
+                self._stats["total_passed"] += 1
+                validated_docs.append(self._tag_document(doc, result))
+            else:
+                self._handle_failed_validation(doc, result, validated_docs)
+
+        return validated_docs
+
+    def _validate_document(self, doc: Document) -> ValidationResult:
+        """Validate a single document.
+
+        Args:
+            doc: LangChain Document to validate
+
+        Returns:
+            ValidationResult
+        """
+        # Build data dict from document
+        data: dict[str, Any] = {}
+
+        if self.validate_content:
+            data["content"] = doc.page_content
+
+        if self.validate_metadata and doc.metadata:
+            data["metadata"] = doc.metadata
+            # Also add metadata fields at top level for validators
+            data.update(doc.metadata)
+
+        # Run validation
+        result = self.pipeline.validate(data)
+
+        # Check confidence threshold
+        confidence = result.metadata.get("semantic_confidence", 1.0)
+        if confidence < self.min_confidence:
+            result.add_message(
+                code="LOW_CONFIDENCE",
+                message=f"Confidence {confidence:.2f} below threshold {self.min_confidence}",
+                severity=ValidationStatus.FAILED,
+                validator="langchain_retriever",
+            )
+            result.status = ValidationStatus.FAILED
+
+        return result
+
+    def _handle_failed_validation(
+        self,
+        doc: Document,
+        result: ValidationResult,
+        validated_docs: list[Document],
+    ) -> None:
+        """Handle a document that failed validation.
+
+        Args:
+            doc: The failed document
+            result: Validation result
+            validated_docs: List to potentially add document to
+        """
+        if self.on_fail == "filter":
+            self._stats["total_filtered"] += 1
+            logger.debug(f"Filtered document due to validation failure: {result}")
+
+        elif self.on_fail == "raise":
+            error_messages = [str(m) for m in result.errors]
+            raise ValidationError(
+                f"Document validation failed: {'; '.join(error_messages)}",
+                result=result,
+            )
+
+        elif self.on_fail == "warn":
+            logger.warning(f"Document validation warning: {result}")
+            validated_docs.append(self._tag_document(doc, result))
+
+        elif self.on_fail == "tag":
+            validated_docs.append(self._tag_document(doc, result))
+
+    def _tag_document(self, doc: Document, result: ValidationResult) -> Document:
+        """Add validation metadata to document.
+
+        Args:
+            doc: Document to tag
+            result: Validation result
+
+        Returns:
+            Tagged document (same object, modified metadata)
+        """
+        doc.metadata["_daytashield_status"] = result.status.value
+        doc.metadata["_daytashield_passed"] = result.passed
+        doc.metadata["_daytashield_message_count"] = len(result.messages)
+
+        if result.errors:
+            doc.metadata["_daytashield_errors"] = [str(e) for e in result.errors[:5]]
+
+        if "semantic_confidence" in result.metadata:
+            doc.metadata["_daytashield_confidence"] = result.metadata["semantic_confidence"]
+
+        return doc
+
+    @property
+    def stats(self) -> dict[str, int]:
+        """Get retrieval and validation statistics."""
+        return dict(self._stats)
+
+    def reset_stats(self) -> None:
+        """Reset statistics counters."""
+        for key in self._stats:
+            self._stats[key] = 0
+
+    # LangChain Retriever interface methods
+
+    def get_relevant_documents(
+        self,
+        query: str,
+        *,
+        callbacks: CallbackManagerForRetrieverRun | None = None,
+        tags: list[str] | None = None,
+        metadata: dict[str, Any] | None = None,
+        run_name: str | None = None,
+        **kwargs: Any,
+    ) -> list[Document]:
+        """Legacy method for LangChain compatibility."""
+        return self.invoke(query, **kwargs)
+
+    async def ainvoke(
+        self,
+        input: str,  # noqa: A002
+        config: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> list[Document]:
+        """Async version of invoke."""
+        # For now, just call sync version
+        # Could be optimized with async validation in the future
+        return self.invoke(input, config=config, **kwargs)
+
+    def __repr__(self) -> str:
+        return (
+            f"ValidatedRetriever(base={self.base_retriever.__class__.__name__}, "
+            f"validators={len(self.pipeline.validators)}, on_fail={self.on_fail!r})"
+        )
+
+
+class ValidationError(Exception):
+    """Raised when document validation fails and on_fail='raise'."""
+
+    def __init__(self, message: str, result: ValidationResult | None = None):
+        super().__init__(message)
+        self.result = result
+
+
+class ValidatedDocumentLoader:
+    """A document loader wrapper that validates documents during loading.
+
+    Wraps any LangChain document loader and validates documents as they're
+    loaded. Useful for validating data at ingestion time.
+
+    Example:
+        >>> from langchain_community.document_loaders import PyPDFLoader
+        >>> from daytashield.integrations.langchain import ValidatedDocumentLoader
+        >>>
+        >>> loader = ValidatedDocumentLoader(
+        ...     base_loader=PyPDFLoader("document.pdf"),
+        ...     validators=[ComplianceValidator(rules=["hipaa"])],
+        ...     on_fail="filter",
+        ... )
+        >>> docs = loader.load()  # Only compliant documents returned
+    """
+
+    def __init__(
+        self,
+        base_loader: Any,
+        validators: list[Any] | None = None,
+        pipeline: ValidationPipeline | None = None,
+        on_fail: OnFailAction = "filter",
+        transform: Callable[[Document], dict[str, Any]] | None = None,
+    ):
+        """Initialize the validated loader.
+
+        Args:
+            base_loader: The underlying LangChain document loader
+            validators: List of validators to apply
+            pipeline: Pre-configured ValidationPipeline
+            on_fail: Action when validation fails
+            transform: Optional function to transform document to validation input
+        """
+        self.base_loader = base_loader
+        self.on_fail = on_fail
+        self.transform = transform
+
+        if pipeline is not None:
+            self.pipeline = pipeline
+        elif validators:
+            self.pipeline = ValidationPipeline(validators=validators)
+        else:
+            self.pipeline = ValidationPipeline()
+
+    def load(self) -> list[Document]:
+        """Load and validate documents.
+
+        Returns:
+            List of validated documents
+        """
+        docs = self.base_loader.load()
+        return self._validate_docs(docs)
+
+    def lazy_load(self) -> Any:
+        """Lazily load and validate documents.
+
+        Yields:
+            Validated documents one at a time
+        """
+        for doc in self.base_loader.lazy_load():
+            data = self._doc_to_data(doc)
+            result = self.pipeline.validate(data)
+
+            if result.passed:
+                yield doc
+            elif self.on_fail == "warn":
+                logger.warning(f"Document validation warning: {result}")
+                yield doc
+            elif self.on_fail == "tag":
+                doc.metadata["_daytashield_validation"] = result.to_dict()
+                yield doc
+            elif self.on_fail == "raise":
+                raise ValidationError(f"Document validation failed: {result}", result)
+            # "filter" - just don't yield
+
+    def _validate_docs(self, docs: list[Document]) -> list[Document]:
+        """Validate a list of documents."""
+        validated = []
+        for doc in docs:
+            data = self._doc_to_data(doc)
+            result = self.pipeline.validate(data)
+
+            if result.passed:
+                validated.append(doc)
+            elif self.on_fail == "warn":
+                logger.warning(f"Document validation warning: {result}")
+                validated.append(doc)
+            elif self.on_fail == "tag":
+                doc.metadata["_daytashield_validation"] = result.to_dict()
+                validated.append(doc)
+            elif self.on_fail == "raise":
+                raise ValidationError(f"Document validation failed: {result}", result)
+
+        return validated
+
+    def _doc_to_data(self, doc: Document) -> dict[str, Any]:
+        """Convert document to validation data."""
+        if self.transform:
+            return self.transform(doc)
+
+        return {
+            "content": doc.page_content,
+            "metadata": doc.metadata,
+            **doc.metadata,
+        }
+
+
+# Type alias for the Document type (avoid import at module level)
+try:
+    from langchain_core.documents import Document
+except ImportError:
+    Document = Any  # type: ignore[misc, assignment]

daytashield/processors/__init__.py

@@ -0,0 +1,13 @@
+"""DaytaShield processors for extracting and transforming data."""
+
+from daytashield.processors.base import BaseProcessor
+from daytashield.processors.csv import CSVProcessor
+from daytashield.processors.json import JSONProcessor
+from daytashield.processors.pdf import PDFProcessor
+
+__all__ = [
+    "BaseProcessor",
+    "PDFProcessor",
+    "CSVProcessor",
+    "JSONProcessor",
+]