msaas-doc-processing 0.1.0__py3-none-any.whl

doc_processing/__init__.py

@@ -0,0 +1,21 @@
+"""Willian Doc Processing — document processing pipeline library."""
+
+from doc_processing.models import (
+    DocumentConfig,
+    DocumentFormat,
+    DocumentInfo,
+    DocumentVersion,
+    TextExtractionResult,
+)
+from doc_processing.processor import DocumentProcessor
+
+__all__ = [
+    # Processor
+    "DocumentProcessor",
+    # Models
+    "DocumentConfig",
+    "DocumentFormat",
+    "DocumentInfo",
+    "DocumentVersion",
+    "TextExtractionResult",
+]

doc_processing/extractors/__init__.py

@@ -0,0 +1 @@
+"""Document text extractors — pluggable backends for each format."""

doc_processing/extractors/base.py

@@ -0,0 +1,39 @@
+"""Abstract base extractor that all format-specific extractors must implement."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from doc_processing.models import DocumentFormat, TextExtractionResult
+
+
+class BaseExtractor(ABC):
+    """Base class for document text extraction implementations.
+
+    Each extractor handles one or more document formats and normalizes
+    the extraction output into a common TextExtractionResult model.
+    """
+
+    @abstractmethod
+    async def extract(self, content: bytes) -> TextExtractionResult:
+        """Extract text from raw document bytes.
+
+        Args:
+            content: Raw bytes of the document file.
+
+        Returns:
+            Extraction result with text, metadata, and timing information.
+        """
+
+    @abstractmethod
+    def supports(self, fmt: DocumentFormat) -> bool:
+        """Check whether this extractor supports the given format.
+
+        Args:
+            fmt: The document format to check.
+
+        Returns:
+            True if this extractor can handle the format.
+        """

doc_processing/extractors/docx.py

@@ -0,0 +1,88 @@
+"""DOCX text extractor using python-docx.
+
+This module is optional — it requires the ``python-docx`` package.
+Install with: ``pip install msaas-doc-processing[docx]``
+"""
+
+from __future__ import annotations
+
+import io
+import time
+
+from doc_processing.extractors.base import BaseExtractor
+from doc_processing.models import DocumentFormat, TextExtractionResult
+
+try:
+    import structlog
+
+    logger = structlog.get_logger(__name__)
+except ImportError:
+    import logging
+
+    logger = logging.getLogger(__name__)
+
+
+class DocxExtractor(BaseExtractor):
+    """Extract text from DOCX documents using python-docx.
+
+    Extracts text from paragraphs and tables, plus core metadata
+    (author, title, creation date).
+    """
+
+    async def extract(self, content: bytes) -> TextExtractionResult:
+        """Extract text from a DOCX file.
+
+        Args:
+            content: Raw DOCX bytes.
+
+        Returns:
+            Extraction result with paragraphs and table text combined.
+        """
+        from docx import Document
+
+        start = time.monotonic()
+
+        doc = Document(io.BytesIO(content))
+
+        # Extract paragraphs
+        paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
+
+        # Extract table content
+        table_texts: list[str] = []
+        for table in doc.tables:
+            for row in table.rows:
+                cells = [cell.text.strip() for cell in row.cells if cell.text.strip()]
+                if cells:
+                    table_texts.append(" | ".join(cells))
+
+        all_parts = paragraphs + table_texts
+        full_text = "\n\n".join(all_parts).strip()
+        word_count = len(full_text.split()) if full_text else 0
+
+        metadata: dict = {}
+        props = doc.core_properties
+        if props.title:
+            metadata["title"] = props.title
+        if props.author:
+            metadata["author"] = props.author
+        if props.created:
+            metadata["created"] = props.created.isoformat()
+        if props.modified:
+            metadata["modified"] = props.modified.isoformat()
+        if props.subject:
+            metadata["subject"] = props.subject
+
+        elapsed_ms = int((time.monotonic() - start) * 1000)
+
+        return TextExtractionResult(
+            text=full_text,
+            format=DocumentFormat.DOCX,
+            page_count=0,
+            word_count=word_count,
+            extraction_method="docx_parser",
+            metadata=metadata,
+            processing_time_ms=elapsed_ms,
+        )
+
+    def supports(self, fmt: DocumentFormat) -> bool:
+        return fmt == DocumentFormat.DOCX

doc_processing/extractors/pdf.py

@@ -0,0 +1,75 @@
+"""PDF text extractor using pypdf for digital text extraction."""
+
+from __future__ import annotations
+
+import io
+import time
+
+from doc_processing.extractors.base import BaseExtractor
+from doc_processing.models import DocumentFormat, TextExtractionResult
+
+try:
+    import structlog
+
+    logger = structlog.get_logger(__name__)
+except ImportError:
+    import logging
+
+    logger = logging.getLogger(__name__)
+
+
+class PDFExtractor(BaseExtractor):
+    """Extract text from PDF documents using pypdf.
+
+    Handles digitally-created PDFs with embedded text.  For scanned PDFs
+    (where extracted text is empty), the caller should fall back to an OCR
+    pipeline.
+    """
+
+    async def extract(self, content: bytes) -> TextExtractionResult:
+        """Extract text from a PDF file.
+
+        Args:
+            content: Raw PDF bytes.
+
+        Returns:
+            Extraction result.  ``text`` will be empty for scanned PDFs.
+        """
+        from pypdf import PdfReader
+
+        start = time.monotonic()
+
+        reader = PdfReader(io.BytesIO(content))
+        pages: list[str] = []
+        for page in reader.pages:
+            text = page.extract_text() or ""
+            pages.append(text)
+
+        full_text = "\n\n".join(pages).strip()
+        word_count = len(full_text.split()) if full_text else 0
+
+        metadata: dict = {}
+        if reader.metadata:
+            if reader.metadata.title:
+                metadata["title"] = reader.metadata.title
+            if reader.metadata.author:
+                metadata["author"] = reader.metadata.author
+            if reader.metadata.subject:
+                metadata["subject"] = reader.metadata.subject
+            if reader.metadata.creator:
+                metadata["creator"] = reader.metadata.creator
+
+        elapsed_ms = int((time.monotonic() - start) * 1000)
+
+        return TextExtractionResult(
+            text=full_text,
+            format=DocumentFormat.PDF,
+            page_count=len(reader.pages),
+            word_count=word_count,
+            extraction_method="digital" if full_text else "empty",
+            metadata=metadata,
+            processing_time_ms=elapsed_ms,
+        )
+
+    def supports(self, fmt: DocumentFormat) -> bool:
+        return fmt == DocumentFormat.PDF

doc_processing/extractors/text.py

@@ -0,0 +1,50 @@
+"""Plain text and markdown extractor."""
+
+from __future__ import annotations
+
+import time
+
+from doc_processing.extractors.base import BaseExtractor
+from doc_processing.models import DocumentFormat, TextExtractionResult
+
+
+class TextExtractor(BaseExtractor):
+    """Extract text from plain text and markdown documents.
+
+    Handles UTF-8 decoding, word counting, and basic metadata extraction.
+    """
+
+    async def extract(self, content: bytes) -> TextExtractionResult:
+        """Extract text from plain text or markdown bytes.
+
+        Args:
+            content: Raw bytes (assumed UTF-8).
+
+        Returns:
+            Extraction result with text and word count.
+        """
+        start = time.monotonic()
+
+        text = content.decode("utf-8", errors="replace").strip()
+        word_count = len(text.split()) if text else 0
+        line_count = text.count("\n") + 1 if text else 0
+
+        metadata: dict = {
+            "line_count": line_count,
+            "char_count": len(text),
+        }
+
+        elapsed_ms = int((time.monotonic() - start) * 1000)
+
+        return TextExtractionResult(
+            text=text,
+            format=DocumentFormat.TXT,
+            page_count=0,
+            word_count=word_count,
+            extraction_method="plain_text",
+            metadata=metadata,
+            processing_time_ms=elapsed_ms,
+        )
+
+    def supports(self, fmt: DocumentFormat) -> bool:
+        return fmt in {DocumentFormat.TXT, DocumentFormat.MARKDOWN}

doc_processing/models.py

@@ -0,0 +1,69 @@
+"""Domain models for document processing, metadata, and versioning."""
+
+from __future__ import annotations
+
+from datetime import datetime  # noqa: TC003
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+
+class DocumentFormat(StrEnum):
+    """Supported document formats."""
+
+    PDF = "pdf"
+    DOCX = "docx"
+    TXT = "txt"
+    IMAGE = "image"
+    MARKDOWN = "markdown"
+    HTML = "html"
+
+
+class DocumentConfig(BaseModel):
+    """Configuration for the document processing pipeline."""
+
+    max_file_size_mb: int = 50
+    allowed_formats: list[DocumentFormat] = Field(
+        default_factory=lambda: list(DocumentFormat),
+    )
+    extract_metadata: bool = True
+    store_versions: bool = True
+    max_versions: int = 10
+
+
+class DocumentInfo(BaseModel):
+    """Metadata about a processed document."""
+
+    id: str
+    filename: str
+    format: DocumentFormat
+    size_bytes: int
+    page_count: int = 0
+    word_count: int = 0
+    language: str = ""
+    title: str = ""
+    author: str = ""
+    created_at: datetime
+    metadata: dict = Field(default_factory=dict)
+
+
+class TextExtractionResult(BaseModel):
+    """Result of text extraction from a document."""
+
+    text: str
+    format: DocumentFormat
+    page_count: int = 0
+    word_count: int = 0
+    extraction_method: str = ""
+    metadata: dict = Field(default_factory=dict)
+    processing_time_ms: int = 0
+
+
+class DocumentVersion(BaseModel):
+    """A single version snapshot of a document."""
+
+    version: int
+    document_id: str
+    text_hash: str
+    changes_summary: str = ""
+    created_at: datetime

doc_processing/processor.py

@@ -0,0 +1,212 @@
+"""Core document processor — format detection, validation, and extraction."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from errors import ValidationError
+
+from doc_processing.extractors.pdf import PDFExtractor
+from doc_processing.extractors.text import TextExtractor
+
+if TYPE_CHECKING:
+    from doc_processing.extractors.base import BaseExtractor
+from doc_processing.models import (
+    DocumentConfig,
+    DocumentFormat,
+    DocumentInfo,
+    TextExtractionResult,
+)
+
+try:
+    import structlog
+
+    logger = structlog.get_logger(__name__)
+except ImportError:
+    import logging
+
+    logger = logging.getLogger(__name__)
+
+# Extension-to-format mapping
+_EXTENSION_MAP: dict[str, DocumentFormat] = {
+    ".pdf": DocumentFormat.PDF,
+    ".docx": DocumentFormat.DOCX,
+    ".txt": DocumentFormat.TXT,
+    ".text": DocumentFormat.TXT,
+    ".md": DocumentFormat.MARKDOWN,
+    ".markdown": DocumentFormat.MARKDOWN,
+    ".html": DocumentFormat.HTML,
+    ".htm": DocumentFormat.HTML,
+    ".jpg": DocumentFormat.IMAGE,
+    ".jpeg": DocumentFormat.IMAGE,
+    ".png": DocumentFormat.IMAGE,
+    ".tiff": DocumentFormat.IMAGE,
+    ".tif": DocumentFormat.IMAGE,
+    ".bmp": DocumentFormat.IMAGE,
+    ".webp": DocumentFormat.IMAGE,
+}
+
+# Magic bytes for format detection
+_MAGIC_BYTES: dict[bytes, DocumentFormat] = {
+    b"%PDF": DocumentFormat.PDF,
+    b"PK\x03\x04": DocumentFormat.DOCX,  # ZIP-based (DOCX, XLSX, etc.)
+}
+
+
+class DocumentProcessor:
+    """High-level document processing pipeline.
+
+    Detects format, validates constraints, routes to the appropriate
+    extractor, and returns structured results.
+
+    Usage::
+
+        processor = DocumentProcessor()
+        result = await processor.process(raw_bytes, "report.pdf")
+    """
+
+    def __init__(self, config: DocumentConfig | None = None) -> None:
+        self._config = config or DocumentConfig()
+        self._extractors: dict[DocumentFormat, BaseExtractor] = {}
+        self._init_extractors()
+
+    def _init_extractors(self) -> None:
+        """Register available extractors, gracefully skipping optional ones."""
+        self._extractors[DocumentFormat.PDF] = PDFExtractor()
+        self._extractors[DocumentFormat.TXT] = TextExtractor()
+        self._extractors[DocumentFormat.MARKDOWN] = TextExtractor()
+
+        try:
+            from doc_processing.extractors.docx import DocxExtractor
+
+            self._extractors[DocumentFormat.DOCX] = DocxExtractor()
+        except ImportError:
+            logger.info("python-docx not installed, DOCX extraction disabled")
+
+    @property
+    def config(self) -> DocumentConfig:
+        """Return the current configuration."""
+        return self._config
+
+    @property
+    def supported_formats(self) -> list[DocumentFormat]:
+        """Return list of formats with registered extractors."""
+        return list(self._extractors.keys())
+
+    def detect_format(self, filename: str, content: bytes | None = None) -> DocumentFormat:
+        """Detect document format from filename extension and optional magic bytes.
+
+        Args:
+            filename: Original filename with extension.
+            content: Optional raw bytes for magic-byte detection.
+
+        Returns:
+            Detected document format.
+
+        Raises:
+            ValidationError: If the format cannot be determined.
+        """
+        # Try magic bytes first when content is available
+        if content:
+            for magic, fmt in _MAGIC_BYTES.items():
+                if content[: len(magic)] == magic:
+                    return fmt
+
+        # Fall back to extension
+        suffix = Path(filename).suffix.lower()
+        fmt = _EXTENSION_MAP.get(suffix)
+        if fmt is not None:
+            return fmt
+
+        raise ValidationError(
+            f"Unsupported file format: {suffix or 'unknown'}",
+            details={"filename": filename, "extension": suffix},
+        )
+
+    def validate(self, filename: str, size_bytes: int) -> None:
+        """Validate file against configuration constraints.
+
+        Args:
+            filename: Original filename.
+            size_bytes: File size in bytes.
+
+        Raises:
+            ValidationError: If the file violates any constraint.
+        """
+        max_bytes = self._config.max_file_size_mb * 1024 * 1024
+        if size_bytes > max_bytes:
+            raise ValidationError(
+                f"File exceeds maximum size of {self._config.max_file_size_mb} MB",
+                details={
+                    "filename": filename,
+                    "size_bytes": size_bytes,
+                    "max_bytes": max_bytes,
+                },
+            )
+
+        fmt = self.detect_format(filename)
+        if fmt not in self._config.allowed_formats:
+            raise ValidationError(
+                f"Format '{fmt.value}' is not allowed",
+                details={
+                    "filename": filename,
+                    "format": fmt.value,
+                    "allowed_formats": [f.value for f in self._config.allowed_formats],
+                },
+            )
+
+    async def process(self, content: bytes, filename: str) -> TextExtractionResult:
+        """Process a document: detect format, validate, and extract text.
+
+        Args:
+            content: Raw document bytes.
+            filename: Original filename.
+
+        Returns:
+            Text extraction result with metadata.
+
+        Raises:
+            ValidationError: If validation fails or no extractor is available.
+        """
+        self.validate(filename, len(content))
+
+        fmt = self.detect_format(filename, content)
+        extractor = self._extractors.get(fmt)
+        if extractor is None:
+            raise ValidationError(
+                f"No extractor available for format '{fmt.value}'",
+                details={"format": fmt.value, "filename": filename},
+            )
+
+        logger.info("processing_document", filename=filename, format=fmt.value)
+        return await extractor.extract(content)
+
+    async def get_info(self, content: bytes, filename: str) -> DocumentInfo:
+        """Get document metadata without full text extraction.
+
+        Args:
+            content: Raw document bytes.
+            filename: Original filename.
+
+        Returns:
+            Document metadata.
+        """
+        fmt = self.detect_format(filename, content)
+        result = await self.process(content, filename)
+
+        return DocumentInfo(
+            id=uuid.uuid4().hex[:12],
+            filename=filename,
+            format=fmt,
+            size_bytes=len(content),
+            page_count=result.page_count,
+            word_count=result.word_count,
+            language="",
+            title=result.metadata.get("title", ""),
+            author=result.metadata.get("author", ""),
+            created_at=datetime.now(UTC),
+            metadata=result.metadata,
+        )

doc_processing/router.py

@@ -0,0 +1,68 @@
+"""FastAPI router exposing document processing endpoints.
+
+Usage::
+
+    from doc_processing.router import create_doc_router
+
+    app = FastAPI()
+    app.include_router(create_doc_router())
+"""
+
+from __future__ import annotations
+
+from doc_processing.models import DocumentInfo, TextExtractionResult
+from doc_processing.processor import DocumentProcessor
+
+
+def _get_fastapi():
+    try:
+        import fastapi
+
+        return fastapi
+    except ImportError:
+        raise ImportError(
+            "FastAPI is required for the API router. "
+            "Install with: pip install msaas-doc-processing[all]"
+        ) from None
+
+
+def create_doc_router(
+    processor: DocumentProcessor | None = None,
+    *,
+    prefix: str = "/documents",
+    tags: list[str] | None = None,
+):
+    """Create a FastAPI APIRouter with document processing endpoints.
+
+    Args:
+        processor: An optional DocumentProcessor instance.  Creates one
+            with default config if not provided.
+        prefix: URL prefix for all routes.
+        tags: OpenAPI tags for the router.
+
+    Returns:
+        A FastAPI APIRouter ready to be included in an app.
+    """
+    fastapi = _get_fastapi()
+    router = fastapi.APIRouter(prefix=prefix, tags=tags or ["documents"])
+    _processor = processor or DocumentProcessor()
+
+    @router.post("/process", response_model=TextExtractionResult)
+    async def process_document(
+        file: fastapi.UploadFile,
+    ) -> TextExtractionResult:
+        """Upload a document and extract its text content."""
+        content = await file.read()
+        filename = file.filename or "unknown"
+        return await _processor.process(content, filename)
+
+    @router.post("/info", response_model=DocumentInfo)
+    async def document_info(
+        file: fastapi.UploadFile,
+    ) -> DocumentInfo:
+        """Upload a document and retrieve its metadata."""
+        content = await file.read()
+        filename = file.filename or "unknown"
+        return await _processor.get_info(content, filename)
+
+    return router

msaas_doc_processing-0.1.0.dist-info/METADATA

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: msaas-doc-processing
+Version: 0.1.0
+Summary: Document processing pipeline — upload, extract, version, convert
+License: MIT
+Requires-Python: >=3.12
+Requires-Dist: msaas-api-core
+Requires-Dist: msaas-errors
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pypdf>=4.0
+Provides-Extra: all
+Requires-Dist: pillow>=10.0; extra == 'all'
+Requires-Dist: python-docx>=1.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: pillow>=10.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: python-docx>=1.0; extra == 'dev'
+Provides-Extra: docx
+Requires-Dist: python-docx>=1.0; extra == 'docx'
+Provides-Extra: image
+Requires-Dist: pillow>=10.0; extra == 'image'

msaas_doc_processing-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+doc_processing/__init__.py,sha256=-3d2jTyxfFvg-s64MKaEuOr_DO6IjflefSaGxcfMiWQ,454
+doc_processing/models.py,sha256=E2XwU2jMlNOCcYxa4TMSSSaeQz5XsUeK3_V1SLOVctM,1577
+doc_processing/processor.py,sha256=tvj-vD2iWjpxOuhnlRlYT1H0z7LT8pXi6WqzEblRJb0,6895
+doc_processing/router.py,sha256=C5j27jGfykB1RRUjkudnwrOO9YgzER_TZ2e7F02K794,2026
+doc_processing/extractors/__init__.py,sha256=nVhpeUCcpwHijbtOYaoqugdGxn6CHr9Hbodwn2Pk0cA,71
+doc_processing/extractors/base.py,sha256=lIESQwF-UUEY50E6zdl5R86diZqOaYYAQwLKbdtg-98,1125
+doc_processing/extractors/docx.py,sha256=TDRRylQ0S7HChV7TqKDc16pV2p4XgZ64vXat_J34xsI,2576
+doc_processing/extractors/pdf.py,sha256=rXP25nUrCcnzMWFcGQWMzDCBRJ-rnitDxfW0hyr3YWk,2248
+doc_processing/extractors/text.py,sha256=1TkxNp6LZu1ga0epXActGT7UAMQvNqpCPYfnBw_kohA,1473
+msaas_doc_processing-0.1.0.dist-info/METADATA,sha256=dYSFMsDbyxcKXG4ZQxtQR0dXh_TO4cycxpfWiBYn35g,749
+msaas_doc_processing-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+msaas_doc_processing-0.1.0.dist-info/RECORD,,

msaas_doc_processing-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any