daytashield 0.1.1__py3-none-any.whl

daytashield/core/audit.py

@@ -0,0 +1,275 @@
+"""Immutable audit trail for validation operations."""
+
+from __future__ import annotations
+
+import gzip
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterator
+from uuid import UUID
+
+import orjson
+from pydantic import BaseModel, Field
+
+from daytashield.core.result import ValidationResult, ValidationStatus
+
+
+class AuditEntry(BaseModel):
+    """A single audit log entry."""
+
+    timestamp: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="When the entry was created",
+    )
+    result_id: UUID = Field(..., description="ID of the validation result")
+    status: ValidationStatus = Field(..., description="Validation status")
+    validators_run: list[str] = Field(default_factory=list, description="Validators executed")
+    message_count: int = Field(0, description="Number of validation messages")
+    error_count: int = Field(0, description="Number of errors")
+    warning_count: int = Field(0, description="Number of warnings")
+    duration_ms: float | None = Field(None, description="Validation duration")
+    source_id: str | None = Field(None, description="Source identifier")
+    source_path: str | None = Field(None, description="Source file path")
+    checksum: str | None = Field(None, description="Data checksum")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
+
+    @classmethod
+    def from_result(cls, result: ValidationResult, metadata: dict[str, Any] | None = None) -> AuditEntry:
+        """Create an audit entry from a validation result.
+
+        Args:
+            result: The validation result to log
+            metadata: Additional metadata to include
+
+        Returns:
+            New AuditEntry
+        """
+        return cls(
+            result_id=result.id,
+            status=result.status,
+            validators_run=result.validators_run,
+            message_count=len(result.messages),
+            error_count=len(result.errors),
+            warning_count=len(result.warnings),
+            duration_ms=result.duration_ms,
+            source_id=result.provenance.source_id if result.provenance else None,
+            source_path=result.provenance.source_path if result.provenance else None,
+            checksum=result.provenance.checksum if result.provenance else None,
+            metadata=metadata or {},
+        )
+
+    def to_jsonl(self) -> bytes:
+        """Serialize to JSON Lines format (single line)."""
+        return orjson.dumps(self.model_dump(mode="json")) + b"\n"
+
+
+class AuditTrailConfig(BaseModel):
+    """Configuration for the audit trail."""
+
+    path: Path = Field(
+        default_factory=lambda: Path("./audit.jsonl"),
+        description="Path to the audit log file",
+    )
+    compress: bool = Field(False, description="Use gzip compression")
+    max_size_bytes: int = Field(100 * 1024 * 1024, description="Max file size before rotation")
+    include_data: bool = Field(False, description="Include actual data in audit (careful!)")
+    buffer_size: int = Field(100, description="Number of entries to buffer before flush")
+
+
+class AuditTrail:
+    """Immutable audit trail using JSON Lines format.
+
+    The audit trail provides an append-only log of all validation operations.
+    It uses JSON Lines format for easy querying and analysis.
+
+    Example:
+        >>> audit = AuditTrail(path="./validation_audit.jsonl")
+        >>> result = pipeline.validate(data)
+        >>> audit.log(result)
+        >>> # Later, query the audit trail
+        >>> for entry in audit.query(status=ValidationStatus.FAILED):
+        ...     print(f"Failed: {entry.source_id} at {entry.timestamp}")
+
+    Features:
+    - Append-only (immutable)
+    - JSON Lines format (one JSON object per line)
+    - Optional gzip compression
+    - File rotation support
+    - Query by status, date range, source
+    """
+
+    def __init__(self, config: AuditTrailConfig | dict[str, Any] | Path | str | None = None):
+        """Initialize the audit trail.
+
+        Args:
+            config: Configuration (AuditTrailConfig, dict, or path string)
+        """
+        if config is None:
+            self.config = AuditTrailConfig()
+        elif isinstance(config, (str, Path)):
+            self.config = AuditTrailConfig(path=Path(config))
+        elif isinstance(config, dict):
+            self.config = AuditTrailConfig(**config)
+        else:
+            self.config = config
+
+        self._buffer: list[AuditEntry] = []
+        self._ensure_path()
+
+    def _ensure_path(self) -> None:
+        """Ensure the audit file directory exists."""
+        self.config.path.parent.mkdir(parents=True, exist_ok=True)
+
+    def _get_file_path(self) -> Path:
+        """Get the current audit file path."""
+        if self.config.compress:
+            return self.config.path.with_suffix(".jsonl.gz")
+        return self.config.path
+
+    def log(self, result: ValidationResult, metadata: dict[str, Any] | None = None) -> AuditEntry:
+        """Log a validation result to the audit trail.
+
+        Args:
+            result: The validation result to log
+            metadata: Additional metadata to include
+
+        Returns:
+            The created AuditEntry
+        """
+        entry = AuditEntry.from_result(result, metadata)
+        self._buffer.append(entry)
+
+        if len(self._buffer) >= self.config.buffer_size:
+            self.flush()
+
+        return entry
+
+    def flush(self) -> int:
+        """Flush buffered entries to disk.
+
+        Returns:
+            Number of entries flushed
+        """
+        if not self._buffer:
+            return 0
+
+        count = len(self._buffer)
+        data = b"".join(entry.to_jsonl() for entry in self._buffer)
+
+        file_path = self._get_file_path()
+
+        if self.config.compress:
+            with gzip.open(file_path, "ab") as f:
+                f.write(data)
+        else:
+            with open(file_path, "ab") as f:
+                f.write(data)
+
+        self._buffer.clear()
+        return count
+
+    def log_batch(self, results: list[ValidationResult]) -> list[AuditEntry]:
+        """Log multiple validation results.
+
+        Args:
+            results: List of validation results
+
+        Returns:
+            List of created AuditEntries
+        """
+        entries = [self.log(result) for result in results]
+        self.flush()
+        return entries
+
+    def query(
+        self,
+        status: ValidationStatus | None = None,
+        start_time: datetime | None = None,
+        end_time: datetime | None = None,
+        source_id: str | None = None,
+        limit: int | None = None,
+    ) -> Iterator[AuditEntry]:
+        """Query the audit trail.
+
+        Args:
+            status: Filter by validation status
+            start_time: Filter entries after this time
+            end_time: Filter entries before this time
+            source_id: Filter by source identifier
+            limit: Maximum number of entries to return
+
+        Yields:
+            Matching AuditEntry objects
+        """
+        self.flush()  # Ensure all entries are on disk
+
+        file_path = self._get_file_path()
+        if not file_path.exists():
+            return
+
+        count = 0
+        opener = gzip.open if self.config.compress else open
+
+        with opener(file_path, "rb") as f:  # type: ignore[operator]
+            for line in f:
+                if not line.strip():
+                    continue
+
+                data = orjson.loads(line)
+                entry = AuditEntry(**data)
+
+                # Apply filters
+                if status is not None and entry.status != status:
+                    continue
+                if start_time is not None and entry.timestamp < start_time:
+                    continue
+                if end_time is not None and entry.timestamp > end_time:
+                    continue
+                if source_id is not None and entry.source_id != source_id:
+                    continue
+
+                yield entry
+                count += 1
+
+                if limit is not None and count >= limit:
+                    return
+
+    def stats(self) -> dict[str, Any]:
+        """Get statistics from the audit trail.
+
+        Returns:
+            Dict with count by status, total count, etc.
+        """
+        self.flush()
+
+        stats: dict[str, Any] = {
+            "total": 0,
+            "by_status": {status.value: 0 for status in ValidationStatus},
+            "avg_duration_ms": 0.0,
+            "total_errors": 0,
+            "total_warnings": 0,
+        }
+
+        durations: list[float] = []
+
+        for entry in self.query():
+            stats["total"] += 1
+            stats["by_status"][entry.status.value] += 1
+            stats["total_errors"] += entry.error_count
+            stats["total_warnings"] += entry.warning_count
+            if entry.duration_ms is not None:
+                durations.append(entry.duration_ms)
+
+        if durations:
+            stats["avg_duration_ms"] = sum(durations) / len(durations)
+
+        return stats
+
+    def __enter__(self) -> AuditTrail:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.flush()
+
+    def __repr__(self) -> str:
+        return f"AuditTrail(path={self.config.path})"

daytashield/core/pipeline.py

@@ -0,0 +1,240 @@
+"""Validation pipeline orchestrator."""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, BinaryIO
+
+from pydantic import BaseModel, Field
+
+from daytashield.core.result import ValidationResult, ValidationStatus, create_result
+
+if TYPE_CHECKING:
+    from daytashield.processors.base import BaseProcessor
+    from daytashield.validators.base import BaseValidator
+
+
+class PipelineConfig(BaseModel):
+    """Configuration for the validation pipeline."""
+
+    fail_fast: bool = Field(False, description="Stop on first validation failure")
+    parallel: bool = Field(False, description="Run validators in parallel (async)")
+    include_original_data: bool = Field(True, description="Include original data in result")
+    auto_detect_processor: bool = Field(True, description="Auto-detect processor by file extension")
+
+    model_config = {"extra": "allow"}
+
+
+class ValidationPipeline:
+    """Orchestrates validation across multiple validators.
+
+    The pipeline chains validators together, passing data and results through
+    each validator in sequence. It handles processor selection, result
+    aggregation, and provides hooks for auditing.
+
+    Example:
+        >>> from daytashield import ValidationPipeline, SchemaValidator, FreshnessValidator
+        >>> pipeline = ValidationPipeline([
+        ...     SchemaValidator(schema={"type": "object"}),
+        ...     FreshnessValidator(max_age="7d"),
+        ... ])
+        >>> result = pipeline.validate({"id": 1, "timestamp": "2024-01-01"})
+        >>> print(result.status)
+        ValidationStatus.PASSED
+
+    Attributes:
+        validators: List of validators to run
+        processors: Dict mapping extensions to processors
+        config: Pipeline configuration
+    """
+
+    def __init__(
+        self,
+        validators: list[BaseValidator] | None = None,
+        processors: dict[str, BaseProcessor] | None = None,
+        config: PipelineConfig | dict[str, Any] | None = None,
+    ):
+        """Initialize the validation pipeline.
+
+        Args:
+            validators: List of validators to apply (in order)
+            processors: Dict mapping file extensions to processors
+            config: Pipeline configuration
+        """
+        self.validators: list[BaseValidator] = validators or []
+        self.processors: dict[str, BaseProcessor] = processors or {}
+        
+        if config is None:
+            self.config = PipelineConfig()
+        elif isinstance(config, dict):
+            self.config = PipelineConfig(**config)
+        else:
+            self.config = config
+
+    def add_validator(self, validator: BaseValidator) -> ValidationPipeline:
+        """Add a validator to the pipeline.
+
+        Args:
+            validator: The validator to add
+
+        Returns:
+            Self for method chaining
+        """
+        self.validators.append(validator)
+        return self
+
+    def add_processor(self, extension: str, processor: BaseProcessor) -> ValidationPipeline:
+        """Register a processor for a file extension.
+
+        Args:
+            extension: File extension (e.g., ".pdf")
+            processor: The processor to use
+
+        Returns:
+            Self for method chaining
+        """
+        self.processors[extension.lower()] = processor
+        return self
+
+    def validate(
+        self,
+        data: Any,
+        source: str | Path | BinaryIO | bytes | None = None,
+        result: ValidationResult | None = None,
+    ) -> ValidationResult:
+        """Run all validators on the data.
+
+        This is the main entry point for validation. It:
+        1. Creates or uses an existing result
+        2. Processes the source if provided (file → structured data)
+        3. Runs each validator in sequence
+        4. Returns the aggregated result
+
+        Args:
+            data: The data to validate (dict, list, or ProcessedData)
+            source: Optional original source (file path, bytes, etc.)
+            result: Optional existing result to continue
+
+        Returns:
+            ValidationResult with all validation findings
+        """
+        start_time = time.perf_counter()
+
+        # Create or use existing result
+        if result is None:
+            result = create_result(status=ValidationStatus.PASSED, data=data)
+            if self.config.include_original_data:
+                result.original_data = data
+
+        # Process source if provided
+        if source is not None:
+            result = self._process_source(source, result)
+            if result.failed:
+                return result.complete()
+            # Use processed data for validation
+            if result.data is not None:
+                data = result.data
+
+        # Run validators
+        for validator in self.validators:
+            if not validator.should_run(data, result):
+                continue
+
+            try:
+                result = validator.validate(data, result)
+                result.validators_run.append(validator.name)
+
+                # Check for fail-fast
+                if self.config.fail_fast and result.failed:
+                    break
+
+            except Exception as e:
+                result.add_message(
+                    code="PIPELINE_ERROR",
+                    message=f"Validator {validator.name} raised an exception: {e}",
+                    severity=ValidationStatus.ERROR,
+                    validator="pipeline",
+                    details={"exception": str(e), "validator": validator.name},
+                )
+                result.status = ValidationStatus.ERROR
+                if self.config.fail_fast:
+                    break
+
+        # Calculate duration
+        end_time = time.perf_counter()
+        result.duration_ms = (end_time - start_time) * 1000
+        result.completed_at = datetime.now(timezone.utc)
+
+        return result
+
+    def validate_file(self, path: str | Path) -> ValidationResult:
+        """Validate a file, auto-detecting the processor.
+
+        Convenience method that reads a file, selects the appropriate
+        processor based on extension, and runs validation.
+
+        Args:
+            path: Path to the file to validate
+
+        Returns:
+            ValidationResult with all validation findings
+        """
+        path = Path(path) if isinstance(path, str) else path
+
+        if not path.exists():
+            result = create_result(status=ValidationStatus.ERROR)
+            result.add_message(
+                code="FILE_NOT_FOUND",
+                message=f"File not found: {path}",
+                severity=ValidationStatus.ERROR,
+                validator="pipeline",
+            )
+            return result.complete()
+
+        return self.validate(data=None, source=path)
+
+    def _process_source(
+        self, source: str | Path | BinaryIO | bytes, result: ValidationResult
+    ) -> ValidationResult:
+        """Process a source using the appropriate processor.
+
+        Args:
+            source: The data source
+            result: The current result
+
+        Returns:
+            Updated result with processed data
+        """
+        if not self.config.auto_detect_processor:
+            return result
+
+        # Determine file extension
+        extension: str | None = None
+        if isinstance(source, (str, Path)):
+            extension = Path(source).suffix.lower()
+        elif hasattr(source, "name") and source.name:
+            extension = Path(source.name).suffix.lower()
+
+        if extension and extension in self.processors:
+            processor = self.processors[extension]
+            try:
+                result = processor.process(source, result)
+                if result.provenance:
+                    result.provenance.processor_chain.append(processor.name)
+            except Exception as e:
+                result.add_message(
+                    code="PROCESSOR_ERROR",
+                    message=f"Processor {processor.name} failed: {e}",
+                    severity=ValidationStatus.ERROR,
+                    validator="pipeline",
+                    details={"exception": str(e), "processor": processor.name},
+                )
+                result.status = ValidationStatus.ERROR
+
+        return result
+
+    def __repr__(self) -> str:
+        validator_names = [v.name for v in self.validators]
+        return f"ValidationPipeline(validators={validator_names})"

daytashield/core/result.py

@@ -0,0 +1,185 @@
+"""Validation result types and data structures."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any
+from uuid import UUID, uuid4
+
+from pydantic import BaseModel, Field
+
+
+class ValidationStatus(str, Enum):
+    """Status of a validation operation."""
+
+    PASSED = "passed"
+    WARNING = "warning"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+    ERROR = "error"
+
+
+class ValidationMessage(BaseModel):
+    """A single validation message with context."""
+
+    code: str = Field(..., description="Machine-readable error/warning code")
+    message: str = Field(..., description="Human-readable description")
+    severity: ValidationStatus = Field(..., description="Severity level")
+    field: str | None = Field(None, description="Field path that triggered the message")
+    validator: str = Field(..., description="Name of the validator that produced this message")
+    details: dict[str, Any] = Field(default_factory=dict, description="Additional context")
+
+    def __str__(self) -> str:
+        field_str = f" [{self.field}]" if self.field else ""
+        return f"[{self.severity.value.upper()}] {self.validator}{field_str}: {self.message}"
+
+
+class Provenance(BaseModel):
+    """Tracks the origin and processing history of data."""
+
+    source_id: str = Field(..., description="Unique identifier for the data source")
+    source_type: str = Field(..., description="Type of source (file, api, stream)")
+    source_path: str | None = Field(None, description="Path or URL of the source")
+    checksum: str | None = Field(None, description="SHA-256 hash of the original data")
+    processed_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="When the data was processed",
+    )
+    processor_chain: list[str] = Field(
+        default_factory=list, description="List of processors applied"
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Additional source metadata")
+
+
+class ValidationResult(BaseModel):
+    """Complete result of a validation operation."""
+
+    id: UUID = Field(default_factory=uuid4, description="Unique result identifier")
+    status: ValidationStatus = Field(..., description="Overall validation status")
+    messages: list[ValidationMessage] = Field(
+        default_factory=list, description="All validation messages"
+    )
+    data: Any = Field(None, description="The validated/transformed data")
+    original_data: Any = Field(None, description="The original input data")
+    provenance: Provenance | None = Field(None, description="Data provenance information")
+    validators_run: list[str] = Field(
+        default_factory=list, description="Names of validators that were executed"
+    )
+    started_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="When validation started",
+    )
+    completed_at: datetime | None = Field(None, description="When validation completed")
+    duration_ms: float | None = Field(None, description="Total validation time in milliseconds")
+    metadata: dict[str, Any] = Field(default_factory=dict, description="Additional result metadata")
+
+    model_config = {"arbitrary_types_allowed": True}
+
+    @property
+    def passed(self) -> bool:
+        """Check if validation passed (no failures or errors)."""
+        return self.status in (ValidationStatus.PASSED, ValidationStatus.WARNING)
+
+    @property
+    def failed(self) -> bool:
+        """Check if validation failed."""
+        return self.status in (ValidationStatus.FAILED, ValidationStatus.ERROR)
+
+    @property
+    def errors(self) -> list[ValidationMessage]:
+        """Get all error-level messages."""
+        return [m for m in self.messages if m.severity == ValidationStatus.FAILED]
+
+    @property
+    def warnings(self) -> list[ValidationMessage]:
+        """Get all warning-level messages."""
+        return [m for m in self.messages if m.severity == ValidationStatus.WARNING]
+
+    def add_message(
+        self,
+        code: str,
+        message: str,
+        severity: ValidationStatus,
+        validator: str,
+        field: str | None = None,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        """Add a validation message to the result."""
+        self.messages.append(
+            ValidationMessage(
+                code=code,
+                message=message,
+                severity=severity,
+                field=field,
+                validator=validator,
+                details=details or {},
+            )
+        )
+
+    def merge(self, other: ValidationResult) -> ValidationResult:
+        """Merge another result into this one, combining messages and updating status."""
+        # Combine messages
+        self.messages.extend(other.messages)
+
+        # Update validators run
+        self.validators_run.extend(other.validators_run)
+
+        # Update status to the most severe
+        status_priority = {
+            ValidationStatus.ERROR: 4,
+            ValidationStatus.FAILED: 3,
+            ValidationStatus.WARNING: 2,
+            ValidationStatus.PASSED: 1,
+            ValidationStatus.SKIPPED: 0,
+        }
+
+        if status_priority[other.status] > status_priority[self.status]:
+            self.status = other.status
+
+        # Use the latest data
+        if other.data is not None:
+            self.data = other.data
+
+        # Merge metadata
+        self.metadata.update(other.metadata)
+
+        return self
+
+    def complete(self) -> ValidationResult:
+        """Mark the validation as complete and calculate duration."""
+        self.completed_at = datetime.now(timezone.utc)
+        if self.started_at:
+            delta = self.completed_at - self.started_at
+            self.duration_ms = delta.total_seconds() * 1000
+        return self
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to a dictionary suitable for JSON serialization."""
+        return self.model_dump(mode="json")
+
+    def __str__(self) -> str:
+        status_str = self.status.value.upper()
+        msg_count = len(self.messages)
+        duration_str = f" ({self.duration_ms:.1f}ms)" if self.duration_ms else ""
+        return f"ValidationResult[{status_str}] - {msg_count} message(s){duration_str}"
+
+    def __repr__(self) -> str:
+        return (
+            f"ValidationResult(id={self.id}, status={self.status.value}, "
+            f"messages={len(self.messages)}, validators={self.validators_run})"
+        )
+
+
+def create_result(
+    status: ValidationStatus = ValidationStatus.PASSED,
+    data: Any = None,
+    provenance: Provenance | None = None,
+) -> ValidationResult:
+    """Factory function to create a new validation result."""
+    return ValidationResult(
+        status=status,
+        data=data,
+        original_data=data,
+        provenance=provenance,
+    )