daytashield 0.1.1__py3-none-any.whl

daytashield/processors/json.py

@@ -0,0 +1,260 @@
+"""JSON file processor."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, BinaryIO
+
+from pydantic import Field
+
+from daytashield.core.result import ValidationResult, ValidationStatus
+from daytashield.processors.base import BaseProcessor, ProcessedData, ProcessorConfig
+
+
+class JSONProcessorConfig(ProcessorConfig):
+    """Configuration for JSON processing."""
+
+    encoding: str = Field("utf-8", description="File encoding")
+    allow_comments: bool = Field(False, description="Allow JSON5-style comments")
+    max_depth: int = Field(100, description="Maximum nesting depth")
+    flatten: bool = Field(False, description="Flatten nested structures")
+    flatten_separator: str = Field(".", description="Separator for flattened keys")
+
+
+class JSONProcessor(BaseProcessor):
+    """Processes JSON files to extract structured data.
+
+    Uses orjson for fast JSON parsing with:
+    - Streaming support for large files
+    - Schema extraction
+    - Nested structure analysis
+    - Optional flattening
+
+    Example:
+        >>> processor = JSONProcessor()
+        >>> result = processor.process("data.json")
+        >>> data = result.data.content  # Parsed JSON
+        >>> schema = result.data.metadata["schema"]  # Inferred structure
+    """
+
+    name = "json"
+    supported_extensions = [".json", ".jsonl", ".ndjson"]
+    supported_mime_types = ["application/json", "application/x-ndjson"]
+
+    def __init__(self, config: JSONProcessorConfig | dict[str, Any] | None = None):
+        """Initialize the JSON processor.
+
+        Args:
+            config: Processor configuration
+        """
+        if config is None:
+            super().__init__(JSONProcessorConfig())
+        elif isinstance(config, dict):
+            super().__init__(JSONProcessorConfig(**config))
+        else:
+            super().__init__(config)
+
+    def process(
+        self, source: str | Path | BinaryIO | bytes, result: ValidationResult | None = None
+    ) -> ValidationResult:
+        """Process a JSON file and extract content.
+
+        Args:
+            source: JSON file path, file object, or bytes
+            result: Optional existing ValidationResult
+
+        Returns:
+            ValidationResult with ProcessedData containing parsed JSON
+        """
+        # Create result if not provided
+        if result is None:
+            result, provenance = self._create_result(source)
+        else:
+            provenance = result.provenance
+
+        config = self.config
+        if not isinstance(config, JSONProcessorConfig):
+            config = JSONProcessorConfig()
+
+        try:
+            import orjson
+        except ImportError:
+            # Fall back to standard json
+            import json as orjson  # type: ignore[no-redef]
+
+        try:
+            # Read raw bytes
+            raw_bytes = self._read_source(source)
+
+            # Compute checksum if configured
+            if self.config.compute_checksum and provenance:
+                provenance.checksum = self._compute_checksum(raw_bytes)
+
+            # Determine if JSONL format
+            is_jsonl = False
+            if isinstance(source, (str, Path)):
+                ext = Path(source).suffix.lower()
+                is_jsonl = ext in (".jsonl", ".ndjson")
+
+            # Parse JSON
+            if is_jsonl:
+                content = self._parse_jsonl(raw_bytes, config.encoding)
+                content_type = "records"
+                record_count = len(content)
+            else:
+                content = orjson.loads(raw_bytes)
+                content_type = "array" if isinstance(content, list) else "object"
+                record_count = len(content) if isinstance(content, list) else None
+
+            # Analyze structure
+            structure_info = self._analyze_structure(content, config.max_depth)
+
+            # Optionally flatten
+            if config.flatten and isinstance(content, dict):
+                content = self._flatten(content, config.flatten_separator)
+
+            # Create processed data
+            processed = ProcessedData(
+                content=content,
+                content_type=content_type,
+                source_type="json",
+                metadata={
+                    "structure": structure_info,
+                    "is_jsonl": is_jsonl,
+                },
+                record_count=record_count,
+                raw_size_bytes=len(raw_bytes),
+            )
+
+            result.data = processed
+
+            # Add warnings for potential issues
+            if structure_info["max_depth"] > 10:
+                result.add_message(
+                    code="JSON_DEEP_NESTING",
+                    message=f"Deep nesting detected: {structure_info['max_depth']} levels",
+                    severity=ValidationStatus.WARNING,
+                    validator=self.name,
+                )
+                if result.status == ValidationStatus.PASSED:
+                    result.status = ValidationStatus.WARNING
+
+        except orjson.JSONDecodeError as e:  # type: ignore[union-attr]
+            result.add_message(
+                code="JSON_PARSE_ERROR",
+                message=f"Invalid JSON: {e}",
+                severity=ValidationStatus.ERROR,
+                validator=self.name,
+            )
+            result.status = ValidationStatus.ERROR
+
+        except Exception as e:
+            result.add_message(
+                code="JSON_PROCESSING_ERROR",
+                message=f"Failed to process JSON: {e}",
+                severity=ValidationStatus.ERROR,
+                validator=self.name,
+                details={"error": str(e)},
+            )
+            result.status = ValidationStatus.ERROR
+
+        return result
+
+    def _parse_jsonl(self, raw_bytes: bytes, encoding: str) -> list[Any]:
+        """Parse JSON Lines format.
+
+        Args:
+            raw_bytes: Raw file bytes
+            encoding: Text encoding
+
+        Returns:
+            List of parsed JSON objects
+        """
+        try:
+            import orjson
+
+            records = []
+            for line in raw_bytes.decode(encoding).splitlines():
+                line = line.strip()
+                if line:  # Skip empty lines
+                    records.append(orjson.loads(line))
+            return records
+        except ImportError:
+            import json
+
+            records = []
+            for line in raw_bytes.decode(encoding).splitlines():
+                line = line.strip()
+                if line:
+                    records.append(json.loads(line))
+            return records
+
+    def _analyze_structure(self, data: Any, max_depth: int) -> dict[str, Any]:
+        """Analyze JSON structure.
+
+        Args:
+            data: Parsed JSON data
+            max_depth: Maximum depth to analyze
+
+        Returns:
+            Structure analysis dict
+        """
+        info: dict[str, Any] = {
+            "type": type(data).__name__,
+            "max_depth": 0,
+            "total_keys": 0,
+            "array_lengths": [],
+        }
+
+        def analyze(obj: Any, depth: int) -> None:
+            if depth > max_depth:
+                return
+
+            info["max_depth"] = max(info["max_depth"], depth)
+
+            if isinstance(obj, dict):
+                info["total_keys"] += len(obj)
+                for value in obj.values():
+                    analyze(value, depth + 1)
+            elif isinstance(obj, list):
+                info["array_lengths"].append(len(obj))
+                for item in obj:
+                    analyze(item, depth + 1)
+
+        analyze(data, 0)
+
+        # Summarize array lengths
+        if info["array_lengths"]:
+            info["min_array_length"] = min(info["array_lengths"])
+            info["max_array_length"] = max(info["array_lengths"])
+            info["array_count"] = len(info["array_lengths"])
+        del info["array_lengths"]
+
+        return info
+
+    def _flatten(self, data: dict[str, Any], separator: str) -> dict[str, Any]:
+        """Flatten a nested dictionary.
+
+        Args:
+            data: Nested dictionary
+            separator: Key separator
+
+        Returns:
+            Flattened dictionary
+        """
+        result: dict[str, Any] = {}
+
+        def flatten_recursive(obj: Any, prefix: str) -> None:
+            if isinstance(obj, dict):
+                for key, value in obj.items():
+                    new_key = f"{prefix}{separator}{key}" if prefix else key
+                    flatten_recursive(value, new_key)
+            elif isinstance(obj, list):
+                for i, item in enumerate(obj):
+                    new_key = f"{prefix}[{i}]"
+                    flatten_recursive(item, new_key)
+            else:
+                result[prefix] = obj
+
+        flatten_recursive(data, "")
+        return result

daytashield/processors/pdf.py

@@ -0,0 +1,232 @@
+"""PDF document processor."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, BinaryIO
+
+from pydantic import Field
+
+from daytashield.core.result import ValidationResult, ValidationStatus
+from daytashield.processors.base import BaseProcessor, ProcessedData, ProcessorConfig
+
+
+class PDFProcessorConfig(ProcessorConfig):
+    """Configuration for PDF processing."""
+
+    extract_images: bool = Field(False, description="Extract embedded images")
+    extract_tables: bool = Field(True, description="Extract tables from PDF")
+    ocr_fallback: bool = Field(False, description="Use OCR if text extraction fails")
+    max_pages: int | None = Field(None, description="Maximum pages to process")
+    password: str | None = Field(None, description="PDF password if encrypted")
+
+
+class PDFProcessor(BaseProcessor):
+    """Processes PDF documents to extract text and metadata.
+
+    Uses pdfplumber for reliable text extraction with layout preservation.
+    Supports:
+    - Text extraction with layout
+    - Metadata extraction (author, title, dates)
+    - Table extraction
+    - Page-by-page processing
+    - Optional OCR fallback
+
+    Example:
+        >>> processor = PDFProcessor()
+        >>> result = processor.process("invoice.pdf")
+        >>> print(result.data.content)  # Extracted text
+        >>> print(result.data.metadata)  # PDF metadata
+    """
+
+    name = "pdf"
+    supported_extensions = [".pdf"]
+    supported_mime_types = ["application/pdf"]
+
+    def __init__(self, config: PDFProcessorConfig | dict[str, Any] | None = None):
+        """Initialize the PDF processor.
+
+        Args:
+            config: Processor configuration
+        """
+        if config is None:
+            super().__init__(PDFProcessorConfig())
+        elif isinstance(config, dict):
+            super().__init__(PDFProcessorConfig(**config))
+        else:
+            super().__init__(config)
+
+    def process(
+        self, source: str | Path | BinaryIO | bytes, result: ValidationResult | None = None
+    ) -> ValidationResult:
+        """Process a PDF file and extract content.
+
+        Args:
+            source: PDF file path, file object, or bytes
+            result: Optional existing ValidationResult
+
+        Returns:
+            ValidationResult with ProcessedData containing extracted text
+        """
+        # Create result if not provided
+        if result is None:
+            result, provenance = self._create_result(source)
+        else:
+            provenance = result.provenance
+
+        config = self.config
+        if not isinstance(config, PDFProcessorConfig):
+            config = PDFProcessorConfig()
+
+        try:
+            import pdfplumber
+        except ImportError:
+            result.add_message(
+                code="PDF_NO_PDFPLUMBER",
+                message="pdfplumber package not installed. Install with: pip install pdfplumber",
+                severity=ValidationStatus.ERROR,
+                validator=self.name,
+            )
+            result.status = ValidationStatus.ERROR
+            return result
+
+        try:
+            # Read the source
+            raw_bytes = self._read_source(source)
+
+            # Compute checksum if configured
+            if self.config.compute_checksum and provenance:
+                provenance.checksum = self._compute_checksum(raw_bytes)
+
+            # Open PDF
+            pdf_file: Any
+            if isinstance(source, (str, Path)):
+                pdf_file = pdfplumber.open(source, password=config.password)
+            else:
+                import io
+                pdf_file = pdfplumber.open(io.BytesIO(raw_bytes), password=config.password)
+
+            with pdf_file as pdf:
+                # Extract metadata
+                metadata = self._extract_metadata(pdf)
+
+                # Extract text from pages
+                pages_text: list[str] = []
+                tables: list[list[list[str]]] = []
+                page_count = len(pdf.pages)
+
+                max_pages = config.max_pages or page_count
+                for i, page in enumerate(pdf.pages[:max_pages]):
+                    # Extract text
+                    page_text = page.extract_text() or ""
+                    pages_text.append(page_text)
+
+                    # Extract tables if configured
+                    if config.extract_tables:
+                        page_tables = page.extract_tables() or []
+                        tables.extend(page_tables)
+
+                # Combine all text
+                full_text = "\n\n".join(pages_text)
+
+                # Check if text extraction worked
+                if not full_text.strip() and config.ocr_fallback:
+                    full_text = self._ocr_fallback(raw_bytes)
+                    metadata["ocr_used"] = True
+
+                # Create processed data
+                processed = ProcessedData(
+                    content=full_text,
+                    content_type="text",
+                    source_type="pdf",
+                    metadata=metadata,
+                    page_count=page_count,
+                    raw_size_bytes=len(raw_bytes),
+                )
+
+                # Add tables to metadata if extracted
+                if tables:
+                    processed.metadata["tables"] = tables
+                    processed.metadata["table_count"] = len(tables)
+
+                result.data = processed
+
+                # Add info message about extraction
+                result.metadata["pdf_pages"] = page_count
+                result.metadata["pdf_text_length"] = len(full_text)
+
+        except Exception as e:
+            result.add_message(
+                code="PDF_PROCESSING_ERROR",
+                message=f"Failed to process PDF: {e}",
+                severity=ValidationStatus.ERROR,
+                validator=self.name,
+                details={"error": str(e)},
+            )
+            result.status = ValidationStatus.ERROR
+
+        return result
+
+    def _extract_metadata(self, pdf: Any) -> dict[str, Any]:
+        """Extract metadata from PDF.
+
+        Args:
+            pdf: pdfplumber PDF object
+
+        Returns:
+            Dict of metadata
+        """
+        metadata: dict[str, Any] = {}
+
+        if hasattr(pdf, "metadata") and pdf.metadata:
+            pdf_meta = pdf.metadata
+            # Common PDF metadata fields
+            field_mapping = {
+                "Title": "title",
+                "Author": "author",
+                "Subject": "subject",
+                "Creator": "creator",
+                "Producer": "producer",
+                "CreationDate": "created_at",
+                "ModDate": "modified_at",
+                "Keywords": "keywords",
+            }
+
+            for pdf_key, our_key in field_mapping.items():
+                if pdf_key in pdf_meta and pdf_meta[pdf_key]:
+                    metadata[our_key] = pdf_meta[pdf_key]
+
+        metadata["page_count"] = len(pdf.pages)
+
+        return metadata
+
+    def _ocr_fallback(self, pdf_bytes: bytes) -> str:
+        """Attempt OCR on PDF pages.
+
+        Args:
+            pdf_bytes: Raw PDF bytes
+
+        Returns:
+            Extracted text via OCR
+        """
+        try:
+            import io
+
+            from pdf2image import convert_from_bytes
+            import pytesseract
+
+            # Convert PDF to images
+            images = convert_from_bytes(pdf_bytes)
+
+            # OCR each page
+            texts = []
+            for img in images:
+                text = pytesseract.image_to_string(img)
+                texts.append(text)
+
+            return "\n\n".join(texts)
+
+        except ImportError:
+            return "[OCR unavailable - install pytesseract and pdf2image]"
+        except Exception:
+            return "[OCR failed]"

daytashield/rules/__init__.py

@@ -0,0 +1,14 @@
+"""DaytaShield compliance rule packs."""
+
+from daytashield.rules.base import ComplianceRule, ComplianceViolation
+from daytashield.rules.gdpr import GDPRRules
+from daytashield.rules.hipaa import HIPAARules
+from daytashield.rules.pii import PIIDetector
+
+__all__ = [
+    "ComplianceRule",
+    "ComplianceViolation",
+    "HIPAARules",
+    "GDPRRules",
+    "PIIDetector",
+]

daytashield/rules/base.py

@@ -0,0 +1,67 @@
+"""Base compliance rule abstract class."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class ComplianceViolation(BaseModel):
+    """A compliance violation detected by a rule."""
+
+    code: str = Field(..., description="Machine-readable violation code")
+    message: str = Field(..., description="Human-readable description")
+    severity: str = Field("error", description="Severity: error, warning, info")
+    category: str = Field(..., description="Violation category (e.g., PHI, PII)")
+    field: str | None = Field(None, description="Field path where violation was found")
+    matched_value: str | None = Field(None, description="The value that matched (redacted)")
+    recommendation: str | None = Field(None, description="How to fix the violation")
+
+
+class ComplianceRule(ABC):
+    """Abstract base class for compliance rules.
+
+    Compliance rules detect specific types of violations in data,
+    such as exposed PII, missing consent, or regulatory violations.
+
+    Example:
+        >>> class MyRule(ComplianceRule):
+        ...     name = "my_rule"
+        ...     description = "Checks for custom violations"
+        ...
+        ...     def check(self, data, text_content):
+        ...         violations = []
+        ...         for field, text in text_content:
+        ...             if "secret" in text.lower():
+        ...                 violations.append(ComplianceViolation(
+        ...                     code="SECRET_EXPOSED",
+        ...                     message="Secret value detected",
+        ...                     category="security",
+        ...                     field=field,
+        ...                 ))
+        ...         return violations
+    """
+
+    name: str = "base_rule"
+    description: str = "Base compliance rule"
+    enabled: bool = True
+
+    @abstractmethod
+    def check(
+        self, data: Any, text_content: list[tuple[str, str]]
+    ) -> list[ComplianceViolation]:
+        """Check data for compliance violations.
+
+        Args:
+            data: The original data structure
+            text_content: List of (field_path, text_value) tuples
+
+        Returns:
+            List of ComplianceViolation objects
+        """
+        pass
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(name={self.name!r})"