msaas-doc-processing 0.1.0__tar.gz

msaas_doc_processing-0.1.0/.gitignore

@@ -0,0 +1,23 @@
+node_modules/
+dist/
+.next/
+.turbo/
+*.pyc
+__pycache__/
+.venv/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.env
+.env.*
+!.env.example
+!.env.*.example
+!.env.*.template
+.DS_Store
+coverage/
+
+# Runtime artifacts
+logs_llm/
+vectors.db
+vectors.db-shm
+vectors.db-wal

msaas_doc_processing-0.1.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: msaas-doc-processing
+Version: 0.1.0
+Summary: Document processing pipeline — upload, extract, version, convert
+License: MIT
+Requires-Python: >=3.12
+Requires-Dist: msaas-api-core
+Requires-Dist: msaas-errors
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pypdf>=4.0
+Provides-Extra: all
+Requires-Dist: pillow>=10.0; extra == 'all'
+Requires-Dist: python-docx>=1.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: pillow>=10.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: python-docx>=1.0; extra == 'dev'
+Provides-Extra: docx
+Requires-Dist: python-docx>=1.0; extra == 'docx'
+Provides-Extra: image
+Requires-Dist: pillow>=10.0; extra == 'image'

msaas_doc_processing-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "msaas-doc-processing"
+version = "0.1.0"
+description = "Document processing pipeline — upload, extract, version, convert"
+requires-python = ">=3.12"
+license = { text = "MIT" }
+dependencies = [
+    "msaas-errors",
+    "msaas-api-core",
+    "pydantic>=2.0",
+    "pypdf>=4.0",
+]
+
+[project.optional-dependencies]
+docx = ["python-docx>=1.0"]
+image = ["Pillow>=10.0"]
+all = ["python-docx>=1.0", "Pillow>=10.0"]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24.0",
+    "python-docx>=1.0",
+    "Pillow>=10.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/doc_processing"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+
+[tool.uv.sources]
+msaas-errors = { workspace = true }
+msaas-api-core = { workspace = true }

msaas_doc_processing-0.1.0/src/doc_processing/__init__.py

@@ -0,0 +1,21 @@
+"""Willian Doc Processing — document processing pipeline library."""
+
+from doc_processing.models import (
+    DocumentConfig,
+    DocumentFormat,
+    DocumentInfo,
+    DocumentVersion,
+    TextExtractionResult,
+)
+from doc_processing.processor import DocumentProcessor
+
+__all__ = [
+    # Processor
+    "DocumentProcessor",
+    # Models
+    "DocumentConfig",
+    "DocumentFormat",
+    "DocumentInfo",
+    "DocumentVersion",
+    "TextExtractionResult",
+]

msaas_doc_processing-0.1.0/src/doc_processing/extractors/__init__.py

@@ -0,0 +1 @@
+"""Document text extractors — pluggable backends for each format."""

msaas_doc_processing-0.1.0/src/doc_processing/extractors/base.py

@@ -0,0 +1,39 @@
+"""Abstract base extractor that all format-specific extractors must implement."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from doc_processing.models import DocumentFormat, TextExtractionResult
+
+
+class BaseExtractor(ABC):
+    """Base class for document text extraction implementations.
+
+    Each extractor handles one or more document formats and normalizes
+    the extraction output into a common TextExtractionResult model.
+    """
+
+    @abstractmethod
+    async def extract(self, content: bytes) -> TextExtractionResult:
+        """Extract text from raw document bytes.
+
+        Args:
+            content: Raw bytes of the document file.
+
+        Returns:
+            Extraction result with text, metadata, and timing information.
+        """
+
+    @abstractmethod
+    def supports(self, fmt: DocumentFormat) -> bool:
+        """Check whether this extractor supports the given format.
+
+        Args:
+            fmt: The document format to check.
+
+        Returns:
+            True if this extractor can handle the format.
+        """

msaas_doc_processing-0.1.0/src/doc_processing/extractors/docx.py

@@ -0,0 +1,88 @@
+"""DOCX text extractor using python-docx.
+
+This module is optional — it requires the ``python-docx`` package.
+Install with: ``pip install msaas-doc-processing[docx]``
+"""
+
+from __future__ import annotations
+
+import io
+import time
+
+from doc_processing.extractors.base import BaseExtractor
+from doc_processing.models import DocumentFormat, TextExtractionResult
+
+try:
+    import structlog
+
+    logger = structlog.get_logger(__name__)
+except ImportError:
+    import logging
+
+    logger = logging.getLogger(__name__)
+
+
+class DocxExtractor(BaseExtractor):
+    """Extract text from DOCX documents using python-docx.
+
+    Extracts text from paragraphs and tables, plus core metadata
+    (author, title, creation date).
+    """
+
+    async def extract(self, content: bytes) -> TextExtractionResult:
+        """Extract text from a DOCX file.
+
+        Args:
+            content: Raw DOCX bytes.
+
+        Returns:
+            Extraction result with paragraphs and table text combined.
+        """
+        from docx import Document
+
+        start = time.monotonic()
+
+        doc = Document(io.BytesIO(content))
+
+        # Extract paragraphs
+        paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
+
+        # Extract table content
+        table_texts: list[str] = []
+        for table in doc.tables:
+            for row in table.rows:
+                cells = [cell.text.strip() for cell in row.cells if cell.text.strip()]
+                if cells:
+                    table_texts.append(" | ".join(cells))
+
+        all_parts = paragraphs + table_texts
+        full_text = "\n\n".join(all_parts).strip()
+        word_count = len(full_text.split()) if full_text else 0
+
+        metadata: dict = {}
+        props = doc.core_properties
+        if props.title:
+            metadata["title"] = props.title
+        if props.author:
+            metadata["author"] = props.author
+        if props.created:
+            metadata["created"] = props.created.isoformat()
+        if props.modified:
+            metadata["modified"] = props.modified.isoformat()
+        if props.subject:
+            metadata["subject"] = props.subject
+
+        elapsed_ms = int((time.monotonic() - start) * 1000)
+
+        return TextExtractionResult(
+            text=full_text,
+            format=DocumentFormat.DOCX,
+            page_count=0,
+            word_count=word_count,
+            extraction_method="docx_parser",
+            metadata=metadata,
+            processing_time_ms=elapsed_ms,
+        )
+
+    def supports(self, fmt: DocumentFormat) -> bool:
+        return fmt == DocumentFormat.DOCX

msaas_doc_processing-0.1.0/src/doc_processing/extractors/pdf.py

@@ -0,0 +1,75 @@
+"""PDF text extractor using pypdf for digital text extraction."""
+
+from __future__ import annotations
+
+import io
+import time
+
+from doc_processing.extractors.base import BaseExtractor
+from doc_processing.models import DocumentFormat, TextExtractionResult
+
+try:
+    import structlog
+
+    logger = structlog.get_logger(__name__)
+except ImportError:
+    import logging
+
+    logger = logging.getLogger(__name__)
+
+
+class PDFExtractor(BaseExtractor):
+    """Extract text from PDF documents using pypdf.
+
+    Handles digitally-created PDFs with embedded text.  For scanned PDFs
+    (where extracted text is empty), the caller should fall back to an OCR
+    pipeline.
+    """
+
+    async def extract(self, content: bytes) -> TextExtractionResult:
+        """Extract text from a PDF file.
+
+        Args:
+            content: Raw PDF bytes.
+
+        Returns:
+            Extraction result.  ``text`` will be empty for scanned PDFs.
+        """
+        from pypdf import PdfReader
+
+        start = time.monotonic()
+
+        reader = PdfReader(io.BytesIO(content))
+        pages: list[str] = []
+        for page in reader.pages:
+            text = page.extract_text() or ""
+            pages.append(text)
+
+        full_text = "\n\n".join(pages).strip()
+        word_count = len(full_text.split()) if full_text else 0
+
+        metadata: dict = {}
+        if reader.metadata:
+            if reader.metadata.title:
+                metadata["title"] = reader.metadata.title
+            if reader.metadata.author:
+                metadata["author"] = reader.metadata.author
+            if reader.metadata.subject:
+                metadata["subject"] = reader.metadata.subject
+            if reader.metadata.creator:
+                metadata["creator"] = reader.metadata.creator
+
+        elapsed_ms = int((time.monotonic() - start) * 1000)
+
+        return TextExtractionResult(
+            text=full_text,
+            format=DocumentFormat.PDF,
+            page_count=len(reader.pages),
+            word_count=word_count,
+            extraction_method="digital" if full_text else "empty",
+            metadata=metadata,
+            processing_time_ms=elapsed_ms,
+        )
+
+    def supports(self, fmt: DocumentFormat) -> bool:
+        return fmt == DocumentFormat.PDF

msaas_doc_processing-0.1.0/src/doc_processing/extractors/text.py

@@ -0,0 +1,50 @@
+"""Plain text and markdown extractor."""
+
+from __future__ import annotations
+
+import time
+
+from doc_processing.extractors.base import BaseExtractor
+from doc_processing.models import DocumentFormat, TextExtractionResult
+
+
+class TextExtractor(BaseExtractor):
+    """Extract text from plain text and markdown documents.
+
+    Handles UTF-8 decoding, word counting, and basic metadata extraction.
+    """
+
+    async def extract(self, content: bytes) -> TextExtractionResult:
+        """Extract text from plain text or markdown bytes.
+
+        Args:
+            content: Raw bytes (assumed UTF-8).
+
+        Returns:
+            Extraction result with text and word count.
+        """
+        start = time.monotonic()
+
+        text = content.decode("utf-8", errors="replace").strip()
+        word_count = len(text.split()) if text else 0
+        line_count = text.count("\n") + 1 if text else 0
+
+        metadata: dict = {
+            "line_count": line_count,
+            "char_count": len(text),
+        }
+
+        elapsed_ms = int((time.monotonic() - start) * 1000)
+
+        return TextExtractionResult(
+            text=text,
+            format=DocumentFormat.TXT,
+            page_count=0,
+            word_count=word_count,
+            extraction_method="plain_text",
+            metadata=metadata,
+            processing_time_ms=elapsed_ms,
+        )
+
+    def supports(self, fmt: DocumentFormat) -> bool:
+        return fmt in {DocumentFormat.TXT, DocumentFormat.MARKDOWN}

msaas_doc_processing-0.1.0/src/doc_processing/models.py

@@ -0,0 +1,69 @@
+"""Domain models for document processing, metadata, and versioning."""
+
+from __future__ import annotations
+
+from datetime import datetime  # noqa: TC003
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+
+class DocumentFormat(StrEnum):
+    """Supported document formats."""
+
+    PDF = "pdf"
+    DOCX = "docx"
+    TXT = "txt"
+    IMAGE = "image"
+    MARKDOWN = "markdown"
+    HTML = "html"
+
+
+class DocumentConfig(BaseModel):
+    """Configuration for the document processing pipeline."""
+
+    max_file_size_mb: int = 50
+    allowed_formats: list[DocumentFormat] = Field(
+        default_factory=lambda: list(DocumentFormat),
+    )
+    extract_metadata: bool = True
+    store_versions: bool = True
+    max_versions: int = 10
+
+
+class DocumentInfo(BaseModel):
+    """Metadata about a processed document."""
+
+    id: str
+    filename: str
+    format: DocumentFormat
+    size_bytes: int
+    page_count: int = 0
+    word_count: int = 0
+    language: str = ""
+    title: str = ""
+    author: str = ""
+    created_at: datetime
+    metadata: dict = Field(default_factory=dict)
+
+
+class TextExtractionResult(BaseModel):
+    """Result of text extraction from a document."""
+
+    text: str
+    format: DocumentFormat
+    page_count: int = 0
+    word_count: int = 0
+    extraction_method: str = ""
+    metadata: dict = Field(default_factory=dict)
+    processing_time_ms: int = 0
+
+
+class DocumentVersion(BaseModel):
+    """A single version snapshot of a document."""
+
+    version: int
+    document_id: str
+    text_hash: str
+    changes_summary: str = ""
+    created_at: datetime

msaas_doc_processing-0.1.0/src/doc_processing/processor.py

@@ -0,0 +1,212 @@
+"""Core document processor — format detection, validation, and extraction."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from errors import ValidationError
+
+from doc_processing.extractors.pdf import PDFExtractor
+from doc_processing.extractors.text import TextExtractor
+
+if TYPE_CHECKING:
+    from doc_processing.extractors.base import BaseExtractor
+from doc_processing.models import (
+    DocumentConfig,
+    DocumentFormat,
+    DocumentInfo,
+    TextExtractionResult,
+)
+
+try:
+    import structlog
+
+    logger = structlog.get_logger(__name__)
+except ImportError:
+    import logging
+
+    logger = logging.getLogger(__name__)
+
+# Extension-to-format mapping
+_EXTENSION_MAP: dict[str, DocumentFormat] = {
+    ".pdf": DocumentFormat.PDF,
+    ".docx": DocumentFormat.DOCX,
+    ".txt": DocumentFormat.TXT,
+    ".text": DocumentFormat.TXT,
+    ".md": DocumentFormat.MARKDOWN,
+    ".markdown": DocumentFormat.MARKDOWN,
+    ".html": DocumentFormat.HTML,
+    ".htm": DocumentFormat.HTML,
+    ".jpg": DocumentFormat.IMAGE,
+    ".jpeg": DocumentFormat.IMAGE,
+    ".png": DocumentFormat.IMAGE,
+    ".tiff": DocumentFormat.IMAGE,
+    ".tif": DocumentFormat.IMAGE,
+    ".bmp": DocumentFormat.IMAGE,
+    ".webp": DocumentFormat.IMAGE,
+}
+
+# Magic bytes for format detection
+_MAGIC_BYTES: dict[bytes, DocumentFormat] = {
+    b"%PDF": DocumentFormat.PDF,
+    b"PK\x03\x04": DocumentFormat.DOCX,  # ZIP-based (DOCX, XLSX, etc.)
+}
+
+
+class DocumentProcessor:
+    """High-level document processing pipeline.
+
+    Detects format, validates constraints, routes to the appropriate
+    extractor, and returns structured results.
+
+    Usage::
+
+        processor = DocumentProcessor()
+        result = await processor.process(raw_bytes, "report.pdf")
+    """
+
+    def __init__(self, config: DocumentConfig | None = None) -> None:
+        self._config = config or DocumentConfig()
+        self._extractors: dict[DocumentFormat, BaseExtractor] = {}
+        self._init_extractors()
+
+    def _init_extractors(self) -> None:
+        """Register available extractors, gracefully skipping optional ones."""
+        self._extractors[DocumentFormat.PDF] = PDFExtractor()
+        self._extractors[DocumentFormat.TXT] = TextExtractor()
+        self._extractors[DocumentFormat.MARKDOWN] = TextExtractor()
+
+        try:
+            from doc_processing.extractors.docx import DocxExtractor
+
+            self._extractors[DocumentFormat.DOCX] = DocxExtractor()
+        except ImportError:
+            logger.info("python-docx not installed, DOCX extraction disabled")
+
+    @property
+    def config(self) -> DocumentConfig:
+        """Return the current configuration."""
+        return self._config
+
+    @property
+    def supported_formats(self) -> list[DocumentFormat]:
+        """Return list of formats with registered extractors."""
+        return list(self._extractors.keys())
+
+    def detect_format(self, filename: str, content: bytes | None = None) -> DocumentFormat:
+        """Detect document format from filename extension and optional magic bytes.
+
+        Args:
+            filename: Original filename with extension.
+            content: Optional raw bytes for magic-byte detection.
+
+        Returns:
+            Detected document format.
+
+        Raises:
+            ValidationError: If the format cannot be determined.
+        """
+        # Try magic bytes first when content is available
+        if content:
+            for magic, fmt in _MAGIC_BYTES.items():
+                if content[: len(magic)] == magic:
+                    return fmt
+
+        # Fall back to extension
+        suffix = Path(filename).suffix.lower()
+        fmt = _EXTENSION_MAP.get(suffix)
+        if fmt is not None:
+            return fmt
+
+        raise ValidationError(
+            f"Unsupported file format: {suffix or 'unknown'}",
+            details={"filename": filename, "extension": suffix},
+        )
+
+    def validate(self, filename: str, size_bytes: int) -> None:
+        """Validate file against configuration constraints.
+
+        Args:
+            filename: Original filename.
+            size_bytes: File size in bytes.
+
+        Raises:
+            ValidationError: If the file violates any constraint.
+        """
+        max_bytes = self._config.max_file_size_mb * 1024 * 1024
+        if size_bytes > max_bytes:
+            raise ValidationError(
+                f"File exceeds maximum size of {self._config.max_file_size_mb} MB",
+                details={
+                    "filename": filename,
+                    "size_bytes": size_bytes,
+                    "max_bytes": max_bytes,
+                },
+            )
+
+        fmt = self.detect_format(filename)
+        if fmt not in self._config.allowed_formats:
+            raise ValidationError(
+                f"Format '{fmt.value}' is not allowed",
+                details={
+                    "filename": filename,
+                    "format": fmt.value,
+                    "allowed_formats": [f.value for f in self._config.allowed_formats],
+                },
+            )
+
+    async def process(self, content: bytes, filename: str) -> TextExtractionResult:
+        """Process a document: detect format, validate, and extract text.
+
+        Args:
+            content: Raw document bytes.
+            filename: Original filename.
+
+        Returns:
+            Text extraction result with metadata.
+
+        Raises:
+            ValidationError: If validation fails or no extractor is available.
+        """
+        self.validate(filename, len(content))
+
+        fmt = self.detect_format(filename, content)
+        extractor = self._extractors.get(fmt)
+        if extractor is None:
+            raise ValidationError(
+                f"No extractor available for format '{fmt.value}'",
+                details={"format": fmt.value, "filename": filename},
+            )
+
+        logger.info("processing_document", filename=filename, format=fmt.value)
+        return await extractor.extract(content)
+
+    async def get_info(self, content: bytes, filename: str) -> DocumentInfo:
+        """Get document metadata without full text extraction.
+
+        Args:
+            content: Raw document bytes.
+            filename: Original filename.
+
+        Returns:
+            Document metadata.
+        """
+        fmt = self.detect_format(filename, content)
+        result = await self.process(content, filename)
+
+        return DocumentInfo(
+            id=uuid.uuid4().hex[:12],
+            filename=filename,
+            format=fmt,
+            size_bytes=len(content),
+            page_count=result.page_count,
+            word_count=result.word_count,
+            language="",
+            title=result.metadata.get("title", ""),
+            author=result.metadata.get("author", ""),
+            created_at=datetime.now(UTC),
+            metadata=result.metadata,
+        )

msaas_doc_processing-0.1.0/src/doc_processing/router.py

@@ -0,0 +1,68 @@
+"""FastAPI router exposing document processing endpoints.
+
+Usage::
+
+    from doc_processing.router import create_doc_router
+
+    app = FastAPI()
+    app.include_router(create_doc_router())
+"""
+
+from __future__ import annotations
+
+from doc_processing.models import DocumentInfo, TextExtractionResult
+from doc_processing.processor import DocumentProcessor
+
+
+def _get_fastapi():
+    try:
+        import fastapi
+
+        return fastapi
+    except ImportError:
+        raise ImportError(
+            "FastAPI is required for the API router. "
+            "Install with: pip install msaas-doc-processing[all]"
+        ) from None
+
+
+def create_doc_router(
+    processor: DocumentProcessor | None = None,
+    *,
+    prefix: str = "/documents",
+    tags: list[str] | None = None,
+):
+    """Create a FastAPI APIRouter with document processing endpoints.
+
+    Args:
+        processor: An optional DocumentProcessor instance.  Creates one
+            with default config if not provided.
+        prefix: URL prefix for all routes.
+        tags: OpenAPI tags for the router.
+
+    Returns:
+        A FastAPI APIRouter ready to be included in an app.
+    """
+    fastapi = _get_fastapi()
+    router = fastapi.APIRouter(prefix=prefix, tags=tags or ["documents"])
+    _processor = processor or DocumentProcessor()
+
+    @router.post("/process", response_model=TextExtractionResult)
+    async def process_document(
+        file: fastapi.UploadFile,
+    ) -> TextExtractionResult:
+        """Upload a document and extract its text content."""
+        content = await file.read()
+        filename = file.filename or "unknown"
+        return await _processor.process(content, filename)
+
+    @router.post("/info", response_model=DocumentInfo)
+    async def document_info(
+        file: fastapi.UploadFile,
+    ) -> DocumentInfo:
+        """Upload a document and retrieve its metadata."""
+        content = await file.read()
+        filename = file.filename or "unknown"
+        return await _processor.get_info(content, filename)
+
+    return router

msaas_doc_processing-0.1.0/tests/test_models.py

@@ -0,0 +1,142 @@
+"""Tests for document processing domain models."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from doc_processing.models import (
+    DocumentConfig,
+    DocumentFormat,
+    DocumentInfo,
+    DocumentVersion,
+    TextExtractionResult,
+)
+
+
+class TestDocumentFormat:
+    def test_values(self):
+        assert DocumentFormat.PDF == "pdf"
+        assert DocumentFormat.DOCX == "docx"
+        assert DocumentFormat.TXT == "txt"
+        assert DocumentFormat.IMAGE == "image"
+        assert DocumentFormat.MARKDOWN == "markdown"
+        assert DocumentFormat.HTML == "html"
+
+    def test_string_comparison(self):
+        assert DocumentFormat.PDF == "pdf"
+        assert DocumentFormat.DOCX == "docx"
+
+    def test_all_formats_listed(self):
+        assert len(DocumentFormat) == 6
+
+
+class TestDocumentConfig:
+    def test_defaults(self):
+        cfg = DocumentConfig()
+        assert cfg.max_file_size_mb == 50
+        assert cfg.extract_metadata is True
+        assert cfg.store_versions is True
+        assert cfg.max_versions == 10
+        assert len(cfg.allowed_formats) == len(DocumentFormat)
+
+    def test_custom_values(self):
+        cfg = DocumentConfig(
+            max_file_size_mb=10,
+            allowed_formats=[DocumentFormat.PDF, DocumentFormat.TXT],
+            extract_metadata=False,
+            store_versions=False,
+            max_versions=5,
+        )
+        assert cfg.max_file_size_mb == 10
+        assert len(cfg.allowed_formats) == 2
+        assert cfg.extract_metadata is False
+
+    def test_allowed_formats_default_contains_all(self):
+        cfg = DocumentConfig()
+        for fmt in DocumentFormat:
+            assert fmt in cfg.allowed_formats
+
+
+class TestDocumentInfo:
+    def test_create_info(self):
+        now = datetime.now(UTC)
+        info = DocumentInfo(
+            id="abc123",
+            filename="report.pdf",
+            format=DocumentFormat.PDF,
+            size_bytes=1024,
+            page_count=5,
+            word_count=500,
+            created_at=now,
+        )
+        assert info.id == "abc123"
+        assert info.filename == "report.pdf"
+        assert info.format == DocumentFormat.PDF
+        assert info.size_bytes == 1024
+        assert info.page_count == 5
+        assert info.word_count == 500
+
+    def test_defaults(self):
+        now = datetime.now(UTC)
+        info = DocumentInfo(
+            id="x",
+            filename="file.txt",
+            format=DocumentFormat.TXT,
+            size_bytes=100,
+            created_at=now,
+        )
+        assert info.page_count == 0
+        assert info.word_count == 0
+        assert info.language == ""
+        assert info.title == ""
+        assert info.author == ""
+        assert info.metadata == {}
+
+
+class TestTextExtractionResult:
+    def test_create_result(self):
+        result = TextExtractionResult(
+            text="Hello world",
+            format=DocumentFormat.TXT,
+            page_count=1,
+            word_count=2,
+            extraction_method="plain_text",
+            processing_time_ms=10,
+        )
+        assert result.text == "Hello world"
+        assert result.word_count == 2
+        assert result.extraction_method == "plain_text"
+
+    def test_defaults(self):
+        result = TextExtractionResult(text="", format=DocumentFormat.PDF)
+        assert result.page_count == 0
+        assert result.word_count == 0
+        assert result.extraction_method == ""
+        assert result.metadata == {}
+        assert result.processing_time_ms == 0
+
+
+class TestDocumentVersion:
+    def test_create_version(self):
+        now = datetime.now(UTC)
+        version = DocumentVersion(
+            version=1,
+            document_id="doc-123",
+            text_hash="abc123def456",
+            changes_summary="Initial version",
+            created_at=now,
+        )
+        assert version.version == 1
+        assert version.document_id == "doc-123"
+        assert version.text_hash == "abc123def456"
+        assert version.changes_summary == "Initial version"
+
+    def test_defaults(self):
+        now = datetime.now(UTC)
+        version = DocumentVersion(
+            version=2,
+            document_id="doc-456",
+            text_hash="xyz789",
+            created_at=now,
+        )
+        assert version.changes_summary == ""

msaas_doc_processing-0.1.0/tests/test_processor.py

@@ -0,0 +1,144 @@
+"""Tests for the DocumentProcessor — format detection, validation, and extraction."""
+
+from __future__ import annotations
+
+import pytest
+from doc_processing.models import DocumentConfig, DocumentFormat
+from doc_processing.processor import DocumentProcessor
+from errors import ValidationError
+
+
+class TestDetectFormat:
+    def test_pdf_by_extension(self):
+        processor = DocumentProcessor()
+        assert processor.detect_format("report.pdf") == DocumentFormat.PDF
+
+    def test_docx_by_extension(self):
+        processor = DocumentProcessor()
+        assert processor.detect_format("document.docx") == DocumentFormat.DOCX
+
+    def test_txt_by_extension(self):
+        processor = DocumentProcessor()
+        assert processor.detect_format("notes.txt") == DocumentFormat.TXT
+
+    def test_markdown_by_extension(self):
+        processor = DocumentProcessor()
+        assert processor.detect_format("README.md") == DocumentFormat.MARKDOWN
+
+    def test_html_by_extension(self):
+        processor = DocumentProcessor()
+        assert processor.detect_format("page.html") == DocumentFormat.HTML
+
+    def test_image_by_extension(self):
+        processor = DocumentProcessor()
+        assert processor.detect_format("photo.jpg") == DocumentFormat.IMAGE
+        assert processor.detect_format("scan.png") == DocumentFormat.IMAGE
+        assert processor.detect_format("doc.tiff") == DocumentFormat.IMAGE
+
+    def test_pdf_by_magic_bytes(self):
+        processor = DocumentProcessor()
+        content = b"%PDF-1.4 rest of the file..."
+        assert processor.detect_format("unknown", content) == DocumentFormat.PDF
+
+    def test_case_insensitive_extension(self):
+        processor = DocumentProcessor()
+        assert processor.detect_format("REPORT.PDF") == DocumentFormat.PDF
+        assert processor.detect_format("doc.TXT") == DocumentFormat.TXT
+
+    def test_unknown_format_raises(self):
+        processor = DocumentProcessor()
+        with pytest.raises(ValidationError, match="Unsupported file format"):
+            processor.detect_format("file.xyz")
+
+    def test_no_extension_raises(self):
+        processor = DocumentProcessor()
+        with pytest.raises(ValidationError, match="Unsupported file format"):
+            processor.detect_format("Makefile")
+
+
+class TestValidation:
+    def test_valid_file(self):
+        processor = DocumentProcessor()
+        processor.validate("report.pdf", 1024)  # Should not raise
+
+    def test_file_too_large(self):
+        config = DocumentConfig(max_file_size_mb=1)
+        processor = DocumentProcessor(config)
+        with pytest.raises(ValidationError, match="exceeds maximum size"):
+            processor.validate("report.pdf", 2 * 1024 * 1024)
+
+    def test_format_not_allowed(self):
+        config = DocumentConfig(allowed_formats=[DocumentFormat.PDF])
+        processor = DocumentProcessor(config)
+        with pytest.raises(ValidationError, match="not allowed"):
+            processor.validate("notes.txt", 100)
+
+    def test_allowed_format_passes(self):
+        config = DocumentConfig(allowed_formats=[DocumentFormat.PDF, DocumentFormat.TXT])
+        processor = DocumentProcessor(config)
+        processor.validate("notes.txt", 100)  # Should not raise
+
+
+class TestProcess:
+    async def test_process_text_file(self):
+        processor = DocumentProcessor()
+        content = b"Hello world, this is a test document."
+        result = await processor.process(content, "test.txt")
+
+        assert result.text == "Hello world, this is a test document."
+        assert result.format == DocumentFormat.TXT
+        assert result.word_count == 7
+        assert result.extraction_method == "plain_text"
+
+    async def test_process_markdown_file(self):
+        processor = DocumentProcessor()
+        content = b"# Title\n\nSome paragraph text here."
+        result = await processor.process(content, "doc.md")
+
+        assert "# Title" in result.text
+        assert result.word_count > 0
+
+    async def test_process_unknown_format_raises(self):
+        processor = DocumentProcessor()
+        with pytest.raises(ValidationError):
+            await processor.process(b"data", "file.xyz")
+
+    async def test_process_no_extractor_raises(self):
+        config = DocumentConfig(allowed_formats=list(DocumentFormat))
+        processor = DocumentProcessor(config)
+        # IMAGE format has no registered extractor
+        with pytest.raises(ValidationError, match="No extractor available"):
+            await processor.process(b"\x89PNG", "photo.png")
+
+    async def test_process_validates_size(self):
+        config = DocumentConfig(max_file_size_mb=0)
+        processor = DocumentProcessor(config)
+        with pytest.raises(ValidationError, match="exceeds maximum size"):
+            await processor.process(b"some content", "test.txt")
+
+
+class TestGetInfo:
+    async def test_get_info_text_file(self):
+        processor = DocumentProcessor()
+        content = b"A simple test file with some words."
+        info = await processor.get_info(content, "test.txt")
+
+        assert info.filename == "test.txt"
+        assert info.format == DocumentFormat.TXT
+        assert info.size_bytes == len(content)
+        assert info.word_count == 7
+        assert info.id  # Should have a generated ID
+
+
+class TestSupportedFormats:
+    def test_default_extractors(self):
+        processor = DocumentProcessor()
+        supported = processor.supported_formats
+        assert DocumentFormat.PDF in supported
+        assert DocumentFormat.TXT in supported
+        assert DocumentFormat.MARKDOWN in supported
+
+    def test_config_accessible(self):
+        config = DocumentConfig(max_file_size_mb=25)
+        processor = DocumentProcessor(config)
+        assert processor.config.max_file_size_mb == 25