ai-pipeline-core 0.3.4__py3-none-any.whl → 0.4.0__py3-none-any.whl

ai_pipeline_core/documents/mime_type.py

@@ -67,19 +67,8 @@ def detect_mime_type(content: bytes, name: str) -> str:
         Only the first 1024 bytes are analyzed for content detection.
         Extension-based detection is O(1) lookup.
 
-    Note:
-        Extension-based detection is preferred for text formats as
-        content analysis can sometimes misidentify structured text.
-
-    Example:
-        >>> detect_mime_type(b'{"key": "value"}', "data.json")
-        'application/json'
-        >>> detect_mime_type(b'Hello World', "text.txt")
-        'text/plain'
-        >>> detect_mime_type(b'', "empty.txt")
-        'text/plain'
-        >>> detect_mime_type(b'\\x89PNG', "image.xyz")
-        'image/png'  # Magic detects PNG despite wrong extension
+    Extension-based detection is preferred for text formats as
+    content analysis can sometimes misidentify structured text.
     """
     # Check for empty content
     if len(content) == 0:
@@ -99,40 +88,13 @@ def detect_mime_type(content: bytes, name: str) -> str:
             return mime
     except (AttributeError, OSError, magic.MagicException) as e:
         logger.warning(f"MIME detection failed for {name}: {e}")
-    except Exception as e:
-        logger.error(f"Unexpected error in MIME detection for {name}: {e}")
+    except Exception:
+        logger.exception(f"Unexpected error in MIME detection for {name}")
 
     # Final fallback based on extension or default
     return EXTENSION_MIME_MAP.get(ext, "application/octet-stream")
 
 
-def mime_type_from_extension(name: str) -> str:
-    """Get MIME type based solely on file extension.
-
-    Simple extension-based MIME type detection without content analysis.
-    This is a legacy function maintained for backward compatibility.
-
-    Args:
-        name: Filename with extension.
-
-    Returns:
-        MIME type based on extension, or 'application/octet-stream'
-        if extension is unknown.
-
-    Note:
-        Prefer detect_mime_type() for more accurate detection.
-        This function only checks the file extension.
-
-    Example:
-        >>> mime_type_from_extension("document.pdf")
-        'application/pdf'
-        >>> mime_type_from_extension("unknown.xyz")
-        'application/octet-stream'
-    """
-    ext = name.lower().split(".")[-1] if "." in name else ""
-    return EXTENSION_MIME_MAP.get(ext, "application/octet-stream")
-
-
 def is_text_mime_type(mime_type: str) -> bool:
     """Check if MIME type represents text-based content.
 
@@ -153,13 +115,6 @@ def is_text_mime_type(mime_type: str) -> bool:
         - application/yaml
         - application/x-yaml
 
-    Example:
-        >>> is_text_mime_type('text/plain')
-        True
-        >>> is_text_mime_type('application/json')
-        True
-        >>> is_text_mime_type('image/png')
-        False
     """
     text_types = [
         "text/",
@@ -181,15 +136,8 @@ def is_json_mime_type(mime_type: str) -> bool:
     Returns:
         True if MIME type is 'application/json', False otherwise.
 
-    Note:
-        Only matches exact 'application/json', not variants like
-        'application/ld+json' or 'application/vnd.api+json'.
-
-    Example:
-        >>> is_json_mime_type('application/json')
-        True
-        >>> is_json_mime_type('text/json')  # Not standard JSON MIME
-        False
+    Only matches exact 'application/json', not variants like
+    'application/ld+json' or 'application/vnd.api+json'.
     """
     return mime_type == "application/json"
 
@@ -209,13 +157,8 @@ def is_yaml_mime_type(mime_type: str) -> bool:
         - application/yaml (standard)
         - application/x-yaml (legacy)
 
-    Example:
-        >>> is_yaml_mime_type('application/yaml')
-        True
-        >>> is_yaml_mime_type('application/x-yaml')
-        True
     """
-    return mime_type == "application/yaml" or mime_type == "application/x-yaml"
+    return mime_type in {"application/yaml", "application/x-yaml"}
 
 
 def is_pdf_mime_type(mime_type: str) -> bool:
@@ -227,15 +170,8 @@ def is_pdf_mime_type(mime_type: str) -> bool:
     Returns:
         True if MIME type is 'application/pdf', False otherwise.
 
-    Note:
-        PDF documents require special handling in the LLM module
-        and are supported by certain vision-capable models.
-
-    Example:
-        >>> is_pdf_mime_type('application/pdf')
-        True
-        >>> is_pdf_mime_type('text/plain')
-        False
+    PDF documents require special handling in the LLM module
+    and are supported by certain vision-capable models.
     """
     return mime_type == "application/pdf"
 
@@ -257,15 +193,8 @@ def is_image_mime_type(mime_type: str) -> bool:
         - image/webp
         - image/svg+xml
 
-    Note:
-        Image documents are automatically encoded for vision-capable
-        LLM models in the AIMessages.document_to_prompt() method.
-
-    Example:
-        >>> is_image_mime_type('image/png')
-        True
-        >>> is_image_mime_type('application/pdf')
-        False
+    Image documents are automatically encoded for vision-capable
+    LLM models in the AIMessages.document_to_prompt() method.
     """
     return mime_type.startswith("image/")
 
@@ -285,8 +214,6 @@ def is_llm_supported_image(mime_type: str) -> bool:
     Unsupported image formats (gif, bmp, tiff, svg, etc.) need conversion
     to PNG before sending to the LLM.
 
-    @public
-
     Args:
         mime_type: MIME type string to check.
 

ai_pipeline_core/documents/utils.py

@@ -5,15 +5,14 @@ canonical key generation, and hash validation used throughout the document syste
 """
 
 import re
-from typing import Any, Iterable, Type
+from collections.abc import Iterable
+from typing import Any
 from urllib.parse import urlparse
 
 
 def sanitize_url(url: str) -> str:
     """Sanitize URL or query string for use in filenames.
 
-    @public
-
     Removes or replaces characters that are invalid in filenames.
 
     Args:
@@ -63,15 +62,13 @@ def camel_to_snake(name: str) -> str:
 
 
 def canonical_name_key(
-    obj_or_name: Type[Any] | str,
+    obj_or_name: type[Any] | str,
     *,
     max_parent_suffixes: int = 3,
     extra_suffixes: Iterable[str] = (),
 ) -> str:
     """Produce a canonical snake_case key from a class or name.
 
-    @public
-
     Process:
       1) Starting with the class name (or given string),
       2) Stripping any trailing parent class names (up to `max_parent_suffixes` from the MRO),
@@ -120,8 +117,6 @@ def canonical_name_key(
 def is_document_sha256(value: str) -> bool:
     """Check if a string is a valid base32-encoded SHA256 hash with proper entropy.
 
-    @public
-
     This function validates that a string is not just formatted like a SHA256 hash,
     but actually has the entropy characteristics of a real hash. It checks:
     1. Correct length (52 characters without padding)
@@ -174,7 +169,4 @@ def is_document_sha256(value: str) -> bool:
     # Require at least 8 unique characters (out of 32 possible in base32)
     # This prevents patterns like "AAAAAAA..." from being identified as real hashes
     unique_chars = len(set(value))
-    if unique_chars < 8:
-        return False
-
-    return True
+    return unique_chars >= 8

ai_pipeline_core/exceptions.py

@@ -1,97 +1,45 @@
 """Exception hierarchy for AI Pipeline Core.
 
-@public
-
 This module defines the exception hierarchy used throughout the AI Pipeline Core library.
 All exceptions inherit from PipelineCoreError, providing a consistent error handling interface.
 """
 
 
 class PipelineCoreError(Exception):
-    """Base exception for all AI Pipeline Core errors.
-
-    @public
-    """
-
-    pass
+    """Base exception for all AI Pipeline Core errors."""
 
 
 class DocumentError(PipelineCoreError):
-    """Base exception for document-related errors.
-
-    @public
-    """
-
-    pass
+    """Base exception for document-related errors."""
 
 
 class DocumentValidationError(DocumentError):
-    """Raised when document validation fails.
-
-    @public
-    """
-
-    pass
+    """Raised when document validation fails."""
 
 
 class DocumentSizeError(DocumentValidationError):
-    """Raised when document content exceeds MAX_CONTENT_SIZE limit.
-
-    @public
-    """
-
-    pass
+    """Raised when document content exceeds MAX_CONTENT_SIZE limit."""
 
 
 class DocumentNameError(DocumentValidationError):
-    """Raised when document name contains invalid characters or patterns.
-
-    @public
-    """
-
-    pass
+    """Raised when document name contains invalid characters or patterns."""
 
 
 class LLMError(PipelineCoreError):
-    """Raised when LLM generation fails after all retries.
-
-    @public
-    """
-
-    pass
+    """Raised when LLM generation fails after all retries."""
 
 
 class PromptError(PipelineCoreError):
-    """Base exception for prompt template errors.
-
-    @public
-    """
-
-    pass
+    """Base exception for prompt template errors."""
 
 
 class PromptRenderError(PromptError):
-    """Raised when Jinja2 template rendering fails.
-
-    @public
-    """
-
-    pass
+    """Raised when Jinja2 template rendering fails."""
 
 
 class PromptNotFoundError(PromptError):
-    """Raised when prompt template file is not found in search paths.
-
-    @public
-    """
-
-    pass
+    """Raised when prompt template file is not found in search paths."""
 
 
 class MimeTypeError(DocumentError):
-    """Raised when MIME type detection or validation fails.
-
-    @public
-    """
-
-    pass
+    """Raised when MIME type detection or validation fails."""

ai_pipeline_core/images/__init__.py

@@ -1,67 +1,53 @@
 """Image processing utilities for LLM vision models.
 
-@public
-
 Splits large images, compresses to JPEG, and respects model-specific constraints.
 Designed for website screenshots, document pages, and other visual content
 sent to vision-capable LLMs.
-
-Quick Start:
-    >>> from ai_pipeline_core.images import process_image, ImagePreset
-    >>>
-    >>> result = process_image(screenshot_bytes)
-    >>> for part in result:
-    ...     send_to_llm(part.data, context=part.label)
-    >>>
-    >>> result = process_image(screenshot_bytes, preset=ImagePreset.GEMINI)
 """
 
 from enum import StrEnum
 
 from pydantic import BaseModel, Field
 
-from ai_pipeline_core.documents import Document, TemporaryDocument
+from ai_pipeline_core.documents import Document
 
 from ._processing import execute_split, load_and_normalize, plan_split
 
 __all__ = [
+    "ImageDocument",
+    "ImagePart",
     "ImagePreset",
     "ImageProcessingConfig",
-    "ImagePart",
-    "ProcessedImage",
     "ImageProcessingError",
+    "ProcessedImage",
     "process_image",
     "process_image_to_documents",
 ]
 
 
+class ImageDocument(Document):  # noqa: RUF067
+    """Concrete document for processed image parts."""
+
+
 # ---------------------------------------------------------------------------
 # Configuration
 # ---------------------------------------------------------------------------
 
 
-class ImagePreset(StrEnum):
-    """Presets for LLM vision model constraints.
-
-    @public
-    """
+class ImagePreset(StrEnum):  # noqa: RUF067
+    """Presets for LLM vision model constraints."""
 
     GEMINI = "gemini"
     CLAUDE = "claude"
     GPT4V = "gpt4v"
 
 
-class ImageProcessingConfig(BaseModel):
+class ImageProcessingConfig(BaseModel):  # noqa: RUF067
     """Configuration for image processing.
 
-    @public
-
     Use ``for_preset`` for standard configurations or construct directly for
     custom constraints.
 
-    Example:
-        >>> config = ImageProcessingConfig.for_preset(ImagePreset.GEMINI)
-        >>> config = ImageProcessingConfig(max_dimension=2000, jpeg_quality=80)
     """
 
     model_config = {"frozen": True}
@@ -98,14 +84,11 @@ class ImageProcessingConfig(BaseModel):
 
     @classmethod
     def for_preset(cls, preset: ImagePreset) -> "ImageProcessingConfig":
-        """Create configuration from a model preset.
-
-        @public
-        """
+        """Create configuration from a model preset."""
         return _PRESETS[preset]
 
 
-_PRESETS: dict[ImagePreset, ImageProcessingConfig] = {
+_PRESETS: dict[ImagePreset, ImageProcessingConfig] = {  # noqa: RUF067
     ImagePreset.GEMINI: ImageProcessingConfig(
         max_dimension=3000,
         max_pixels=9_000_000,
@@ -129,11 +112,8 @@ _PRESETS: dict[ImagePreset, ImageProcessingConfig] = {
 # ---------------------------------------------------------------------------
 
 
-class ImagePart(BaseModel):
-    """A single processed image part.
-
-    @public
-    """
+class ImagePart(BaseModel):  # noqa: RUF067
+    """A single processed image part."""
 
     model_config = {"frozen": True}
 
@@ -147,20 +127,15 @@ class ImagePart(BaseModel):
 
     @property
     def label(self) -> str:
-        """Human-readable label for LLM context, 1-indexed.
-
-        @public
-        """
+        """Human-readable label for LLM context, 1-indexed."""
         if self.total == 1:
             return "Full image"
         return f"Part {self.index + 1}/{self.total}"
 
 
-class ProcessedImage(BaseModel):
+class ProcessedImage(BaseModel):  # noqa: RUF067
     """Result of image processing.
 
-    @public
-
     Iterable: ``for part in result`` iterates over parts.
     """
 
@@ -176,10 +151,7 @@ class ProcessedImage(BaseModel):
 
     @property
     def compression_ratio(self) -> float:
-        """Output size / input size (lower means more compression).
-
-        @public
-        """
+        """Output size / input size (lower means more compression)."""
         if self.original_bytes <= 0:
             return 1.0
         return self.output_bytes / self.original_bytes
@@ -199,11 +171,8 @@ class ProcessedImage(BaseModel):
 # ---------------------------------------------------------------------------
 
 
-class ImageProcessingError(Exception):
-    """Image processing failed.
-
-    @public
-    """
+class ImageProcessingError(Exception):  # noqa: RUF067
+    """Image processing failed."""
 
 
 # ---------------------------------------------------------------------------
@@ -211,15 +180,13 @@ class ImageProcessingError(Exception):
 # ---------------------------------------------------------------------------
 
 
-def process_image(
+def process_image(  # noqa: RUF067
     image: bytes | Document,
     preset: ImagePreset = ImagePreset.GEMINI,
     config: ImageProcessingConfig | None = None,
 ) -> ProcessedImage:
     """Process an image for LLM vision models.
 
-    @public
-
     Splits tall images vertically with overlap, trims width if needed, and
     compresses to JPEG.  The default preset is **GEMINI** (3 000 px, 9 M pixels).
 
@@ -234,10 +201,6 @@ def process_image(
     Raises:
         ImageProcessingError: If the image cannot be decoded or processed.
 
-    Example:
-        >>> result = process_image(screenshot_bytes)
-        >>> for part in result:
-        ...     print(part.label, len(part.data))
     """
     effective = config if config is not None else ImageProcessingConfig.for_preset(preset)
 
@@ -248,7 +211,7 @@ def process_image(
     elif isinstance(image, bytes):  # type: ignore[reportUnnecessaryIsInstance]
         raw = image
     else:
-        raise ImageProcessingError(f"Unsupported image input type: {type(image)}")
+        raise ImageProcessingError(f"Unsupported image input type: {type(image)}")  # pyright: ignore[reportUnreachable]
 
     if not raw:
         raise ImageProcessingError("Empty image data")
@@ -306,42 +269,26 @@ def process_image(
     )
 
 
-def process_image_to_documents(
+def process_image_to_documents(  # noqa: RUF067
     image: bytes | Document,
     preset: ImagePreset = ImagePreset.GEMINI,
     config: ImageProcessingConfig | None = None,
     name_prefix: str = "image",
-    sources: list[str] | None = None,
-) -> list[TemporaryDocument]:
-    """Process an image and return parts as ``TemporaryDocument`` list.
-
-    @public
+    sources: tuple[str, ...] | None = None,
+) -> list[ImageDocument]:
+    """Process an image and return parts as ImageDocument list.
 
     Convenience wrapper around ``process_image`` for direct integration
     with ``AIMessages``.
-
-    Args:
-        image: Raw image bytes or a Document.
-        preset: Model preset (ignored when *config* is provided).
-        config: Custom configuration.
-        name_prefix: Prefix for generated document names.
-        sources: Optional provenance references attached to each document.
-
-    Returns:
-        List of ``TemporaryDocument`` instances with JPEG image data.
-
-    Example:
-        >>> docs = process_image_to_documents(screenshot_bytes)
-        >>> messages = AIMessages(docs)
     """
     result = process_image(image, preset=preset, config=config)
 
-    # Resolve sources
-    doc_sources: list[str] = list(sources or [])
+    source_list: list[str] = list(sources or ())
     if isinstance(image, Document):
-        doc_sources.append(image.sha256)
+        source_list.append(image.sha256)
+    doc_sources = tuple(source_list) if source_list else None
 
-    documents: list[TemporaryDocument] = []
+    documents: list[ImageDocument] = []
     for part in result.parts:
         if len(result.parts) == 1:
             name = f"{name_prefix}.jpg"
@@ -351,11 +298,11 @@ def process_image_to_documents(
             desc = part.label
 
         documents.append(
-            TemporaryDocument.create(
+            ImageDocument.create(
                 name=name,
                 content=part.data,
                 description=desc,
-                sources=doc_sources or None,
+                sources=doc_sources,
             )
         )
 

ai_pipeline_core/images/_processing.py

@@ -21,7 +21,7 @@ class SplitPlan:
     warnings: list[str]
 
 
-def plan_split(
+def plan_split(  # noqa: PLR0917
     width: int,
     height: int,
     max_dimension: int,
@@ -71,10 +71,7 @@ def plan_split(
 
     # Auto-reduce if exceeds max_parts
     if num_parts > max_parts:
-        warnings.append(
-            f"Image requires {num_parts} parts but max is {max_parts}. "
-            f"Reducing to {max_parts} parts with larger step."
-        )
+        warnings.append(f"Image requires {num_parts} parts but max is {max_parts}. Reducing to {max_parts} parts with larger step.")
         num_parts = max_parts
         if num_parts > 1:
             step = (height - tile_h) // (num_parts - 1)
@@ -97,10 +94,7 @@ def load_and_normalize(data: bytes) -> Image.Image:
     img.load()
 
     if img.width * img.height > PIL_MAX_PIXELS:
-        raise ValueError(
-            f"Image too large: {img.width}x{img.height} = {img.width * img.height:,} pixels "
-            f"(limit: {PIL_MAX_PIXELS:,})"
-        )
+        raise ValueError(f"Image too large: {img.width}x{img.height} = {img.width * img.height:,} pixels (limit: {PIL_MAX_PIXELS:,})")
 
     # Fix EXIF orientation (important for mobile photos)
     img = ImageOps.exif_transpose(img)
@@ -110,7 +104,7 @@ def load_and_normalize(data: bytes) -> Image.Image:
 def encode_jpeg(img: Image.Image, quality: int) -> bytes:
     """Encode PIL Image as JPEG bytes."""
     # Convert to RGB if needed (JPEG doesn't support alpha)
-    if img.mode not in ("RGB", "L"):
+    if img.mode not in {"RGB", "L"}:
         img = img.convert("RGB")
 
     buf = BytesIO()
@@ -135,7 +129,7 @@ def execute_split(
         width = plan.trim_width
 
     # Convert to RGB once for JPEG
-    if img.mode not in ("RGB", "L"):
+    if img.mode not in {"RGB", "L"}:
         img = img.convert("RGB")
 
     parts: list[tuple[bytes, int, int, int, int]] = []

ai_pipeline_core/llm/__init__.py

@@ -1,7 +1,8 @@
 """Large Language Model integration via LiteLLM proxy.
 
 This package provides OpenAI API-compatible LLM interactions with built-in retry logic,
-LMNR tracing, and structured output generation using Pydantic models.
+LMNR tracing, and structured output generation using Pydantic models. Supports per-call
+observability via purpose and expected_cost parameters for span naming and cost tracking.
 """
 
 from .ai_messages import AIMessages, AIMessageType
@@ -10,15 +11,16 @@ from .client import (
     generate_structured,
 )
 from .model_options import ModelOptions
-from .model_response import ModelResponse, StructuredModelResponse
+from .model_response import Citation, ModelResponse, StructuredModelResponse
 from .model_types import ModelName
 
 __all__ = [
-    "AIMessages",
     "AIMessageType",
+    "AIMessages",
+    "Citation",
     "ModelName",
-    "ModelResponse",
     "ModelOptions",
+    "ModelResponse",
     "StructuredModelResponse",
     "generate",
     "generate_structured",