ai-pipeline-core 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

ai_pipeline_core/__init__.py

@@ -118,7 +118,7 @@ from .prompt_manager import PromptManager
 from .settings import Settings
 from .tracing import TraceInfo, TraceLevel, set_trace_cost, trace
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"
 
 __all__ = [
     # Config/Settings

ai_pipeline_core/documents/document.py

@@ -51,6 +51,7 @@ from .mime_type import (
 )
 
 TModel = TypeVar("TModel", bound=BaseModel)
+TDocument = TypeVar("TDocument", bound="Document")
 
 
 class Document(BaseModel, ABC):
@@ -97,6 +98,8 @@ class Document(BaseModel, ABC):
     - Support for text, JSON, YAML, PDF, and image formats
     - Conversion utilities between different formats
     - Source provenance tracking via sources field
+    - Document type conversion via model_convert() method
+    - Standard Pydantic model_copy() for same-type copying
 
     Class Variables:
         MAX_CONTENT_SIZE: Maximum allowed content size in bytes (default 25MB)
@@ -223,6 +226,14 @@ class Document(BaseModel, ABC):
         ...     sources=[source_doc.sha256]  # Reference source document
         ... )
         >>> processed.has_source(source_doc)  # True
+        >>>
+        >>> # Document copying and type conversion:
+        >>> # Standard Pydantic model_copy (doesn't validate updates)
+        >>> copied = doc.model_copy(update={"name": "new_name.json"})
+        >>> # Type conversion with validation via model_convert
+        >>> task_doc = MyTaskDoc.create(name="temp.json", content={"data": "value"})
+        >>> flow_doc = task_doc.model_convert(MyFlowDoc)  # Convert to FlowDocument
+        >>> flow_doc.is_flow  # True
     """
 
     MAX_CONTENT_SIZE: ClassVar[int] = 25 * 1024 * 1024
@@ -1498,6 +1509,8 @@ class Document(BaseModel, ABC):
                 - sha256: Full SHA256 hash in base32 encoding without padding (str)
                 - mime_type: Detected MIME type (str)
                 - sources: List of source strings (list[dict])
+                - canonical_name: Canonical snake_case name for debug tracing (str)
+                - class_name: Name of the actual document class for debug tracing (str)
                 - content: Encoded content (str)
                 - content_encoding: Either "utf-8" or "base64" (str)
 
@@ -1521,10 +1534,12 @@ class Document(BaseModel, ABC):
             "sha256": self.sha256,
             "mime_type": self.mime_type,
             "sources": self.sources,
+            "canonical_name": canonical_name_key(self.__class__),
+            "class_name": self.__class__.__name__,
         }
 
         # Try to encode content as UTF-8, fall back to base64
-        if self.is_text or self.mime_type.startswith("text/"):
+        if self.is_text:
             try:
                 result["content"] = self.content.decode("utf-8")
                 result["content_encoding"] = "utf-8"
@@ -1600,3 +1615,96 @@ class Document(BaseModel, ABC):
             description=data.get("description"),
             sources=data.get("sources", []),
         )
+
+    @final
+    def model_convert(
+        self,
+        new_type: type[TDocument],
+        *,
+        update: dict[str, Any] | None = None,
+        deep: bool = False,
+    ) -> TDocument:
+        """Convert document to a different Document type with optional updates.
+
+        @public
+
+        Creates a new document of a different type, preserving all attributes
+        while allowing updates. This is useful for converting between document
+        types (e.g., TaskDocument to FlowDocument) while maintaining data integrity.
+
+        Args:
+            new_type: Target Document class for conversion. Must be a concrete
+                     subclass of Document (not abstract classes like Document,
+                     FlowDocument, or TaskDocument).
+            update: Dictionary of attributes to update. Supports any attributes
+                   that the Document constructor accepts (name, content,
+                   description, sources).
+            deep: Whether to perform a deep copy of mutable attributes.
+
+        Returns:
+            New Document instance of the specified type.
+
+        Raises:
+            TypeError: If new_type is not a subclass of Document, is an abstract
+                      class, or if update contains invalid attributes.
+            DocumentNameError: If the name violates the target type's FILES enum.
+            DocumentSizeError: If content exceeds MAX_CONTENT_SIZE.
+
+        Example:
+            >>> # Convert TaskDocument to FlowDocument
+            >>> task_doc = MyTaskDoc.create(name="temp.json", content={"data": "value"})
+            >>> flow_doc = task_doc.model_convert(MyFlowDoc)
+            >>> assert flow_doc.is_flow
+            >>> assert flow_doc.content == task_doc.content
+            >>>
+            >>> # Convert with updates
+            >>> updated = task_doc.model_convert(
+            ...     MyFlowDoc,
+            ...     update={"name": "permanent.json", "description": "Converted"}
+            ... )
+            >>>
+            >>> # Track document lineage
+            >>> derived = doc.model_convert(
+            ...     ProcessedDoc,
+            ...     update={"sources": [doc.sha256]}
+            ... )
+        """
+        # Validate new_type
+        try:
+            # Use a runtime check to ensure it's a class
+            if not isinstance(new_type, type):  # type: ignore[reportIncompatibleArgumentType]
+                raise TypeError(f"new_type must be a class, got {new_type}")
+            if not issubclass(new_type, Document):  # type: ignore[reportIncompatibleArgumentType]
+                raise TypeError(f"new_type must be a subclass of Document, got {new_type}")
+        except (TypeError, AttributeError):
+            # Not a class at all
+            raise TypeError(f"new_type must be a subclass of Document, got {new_type}")
+
+        # Check for abstract classes by name (avoid circular imports)
+        class_name = new_type.__name__
+        if class_name == "Document":
+            raise TypeError("Cannot instantiate abstract Document class directly")
+        if class_name == "FlowDocument":
+            raise TypeError("Cannot instantiate abstract FlowDocument class directly")
+        if class_name == "TaskDocument":
+            raise TypeError("Cannot instantiate abstract TaskDocument class directly")
+
+        # Get current document data with proper typing
+        data: dict[str, Any] = {
+            "name": self.name,
+            "content": self.content,
+            "description": self.description,
+            "sources": self.sources.copy() if deep else self.sources,
+        }
+
+        # Apply updates if provided
+        if update:
+            data.update(update)
+
+        # Create new document of target type
+        return new_type(
+            name=data["name"],
+            content=data["content"],
+            description=data.get("description"),
+            sources=data.get("sources", []),
+        )

ai_pipeline_core/documents/document_list.py

@@ -3,7 +3,8 @@
 @public
 """
 
-from typing import Any, Iterable, SupportsIndex, Union, overload
+from copy import deepcopy
+from typing import Any, Callable, Iterable, SupportsIndex, Union, overload
 
 from typing_extensions import Self
 
@@ -37,6 +38,7 @@ class DocumentList(list[Document]):
         documents: list[Document] | None = None,
         validate_same_type: bool = False,
         validate_duplicates: bool = False,
+        frozen: bool = False,
     ) -> None:
         """Initialize DocumentList.
 
@@ -46,12 +48,15 @@ class DocumentList(list[Document]):
             documents: Initial list of documents.
             validate_same_type: Enforce same document type.
             validate_duplicates: Prevent duplicate filenames.
+            frozen: If True, list is immutable from creation.
         """
         super().__init__()
         self._validate_same_type = validate_same_type
         self._validate_duplicates = validate_duplicates
+        self._frozen = False  # Initialize as unfrozen to allow initial population
         if documents:
             self.extend(documents)
+        self._frozen = frozen  # Set frozen state after initial population
 
     def _validate_no_duplicates(self) -> None:
         """Check for duplicate document names.
@@ -109,18 +114,51 @@ class DocumentList(list[Document]):
         self._validate_no_description_files()
         self._validate_types()
 
+    def freeze(self) -> None:
+        """Permanently freeze the list, preventing modifications.
+
+        Once frozen, the list cannot be unfrozen.
+        """
+        self._frozen = True
+
+    def copy(self) -> "DocumentList":
+        """Create an unfrozen deep copy of the list.
+
+        Returns:
+            New unfrozen DocumentList with deep-copied documents.
+        """
+        copied_docs = deepcopy(list(self))
+        return DocumentList(
+            documents=copied_docs,
+            validate_same_type=self._validate_same_type,
+            validate_duplicates=self._validate_duplicates,
+            frozen=False,  # Copies are always unfrozen
+        )
+
+    def _check_frozen(self) -> None:
+        """Check if list is frozen and raise if it is.
+
+        Raises:
+            RuntimeError: If the list is frozen.
+        """
+        if self._frozen:
+            raise RuntimeError("Cannot modify frozen DocumentList")
+
     def append(self, document: Document) -> None:
         """Add a document to the end of the list."""
+        self._check_frozen()
         super().append(document)
         self._validate()
 
     def extend(self, documents: Iterable[Document]) -> None:
         """Add multiple documents to the list."""
+        self._check_frozen()
         super().extend(documents)
         self._validate()
 
     def insert(self, index: SupportsIndex, document: Document) -> None:
         """Insert a document at the specified position."""
+        self._check_frozen()
         super().insert(index, document)
         self._validate()
 
@@ -132,6 +170,7 @@ class DocumentList(list[Document]):
 
     def __setitem__(self, index: Union[SupportsIndex, slice], value: Any) -> None:
         """Set item or slice with validation."""
+        self._check_frozen()
         super().__setitem__(index, value)
         self._validate()
 
@@ -141,10 +180,48 @@ class DocumentList(list[Document]):
         Returns:
             Self: This DocumentList after modification.
         """
+        self._check_frozen()
         result = super().__iadd__(other)
         self._validate()
         return result
 
+    def __delitem__(self, index: Union[SupportsIndex, slice]) -> None:
+        """Delete item or slice from list."""
+        self._check_frozen()
+        super().__delitem__(index)
+
+    def pop(self, index: SupportsIndex = -1) -> Document:
+        """Remove and return item at index.
+
+        Returns:
+            Document removed from the list.
+        """
+        self._check_frozen()
+        return super().pop(index)
+
+    def remove(self, document: Document) -> None:
+        """Remove first occurrence of document."""
+        self._check_frozen()
+        super().remove(document)
+
+    def clear(self) -> None:
+        """Remove all items from list."""
+        self._check_frozen()
+        super().clear()
+
+    def reverse(self) -> None:
+        """Reverse list in place."""
+        self._check_frozen()
+        super().reverse()
+
+    def sort(self, *, key: Callable[[Document], Any] | None = None, reverse: bool = False) -> None:
+        """Sort list in place."""
+        self._check_frozen()
+        if key is None:
+            super().sort(reverse=reverse)  # type: ignore[call-arg]
+        else:
+            super().sort(key=key, reverse=reverse)
+
     @overload
     def filter_by(self, arg: str) -> "DocumentList": ...
 

ai_pipeline_core/llm/__init__.py

@@ -8,8 +8,6 @@ from .ai_messages import AIMessages, AIMessageType
 from .client import (
     generate,
     generate_structured,
-    generate_with_retry_for_testing,
-    process_messages_for_testing,
 )
 from .model_options import ModelOptions
 from .model_response import ModelResponse, StructuredModelResponse
@@ -24,7 +22,4 @@ __all__ = [
     "StructuredModelResponse",
     "generate",
     "generate_structured",
-    # Internal functions exposed for testing only
-    "process_messages_for_testing",
-    "generate_with_retry_for_testing",
 ]

ai_pipeline_core/llm/ai_messages.py

@@ -9,6 +9,8 @@ including text, documents, and model responses.
 import base64
 import hashlib
 import json
+from copy import deepcopy
+from typing import Any, Callable, Iterable, SupportsIndex, Union
 
 from openai.types.chat import (
     ChatCompletionContentPartParam,
@@ -64,8 +66,8 @@ class AIMessages(list[AIMessageType]):
     CAUTION: AIMessages is a list subclass. Always use list construction (e.g.,
     `AIMessages(["text"])`) or empty constructor with append (e.g.,
     `AIMessages(); messages.append("text")`). Never pass raw strings directly to the
-    constructor (`AIMessages("text")`) as this will iterate over the string characters
-    instead of treating it as a single message.
+    constructor (`AIMessages("text")`) as this will raise a TypeError to prevent
+    accidental character iteration.
 
     Example:
         >>> from ai_pipeline_core import llm
@@ -75,6 +77,127 @@ class AIMessages(list[AIMessageType]):
         >>> messages.append(response)  # Add the actual response
     """
 
+    def __init__(self, iterable: Iterable[AIMessageType] | None = None, *, frozen: bool = False):
+        """Initialize AIMessages with optional iterable.
+
+        Args:
+            iterable: Optional iterable of messages (list, tuple, etc.).
+                     Must not be a string.
+            frozen: If True, list is immutable from creation.
+
+        Raises:
+            TypeError: If a string is passed directly to the constructor.
+        """
+        if isinstance(iterable, str):
+            raise TypeError(
+                "AIMessages cannot be constructed from a string directly. "
+                "Use AIMessages(['text']) for a single message or "
+                "AIMessages() and then append('text')."
+            )
+        self._frozen = False  # Initialize as unfrozen to allow initial population
+        if iterable is None:
+            super().__init__()
+        else:
+            super().__init__(iterable)
+        self._frozen = frozen  # Set frozen state after initial population
+
+    def freeze(self) -> None:
+        """Permanently freeze the list, preventing modifications.
+
+        Once frozen, the list cannot be unfrozen.
+        """
+        self._frozen = True
+
+    def copy(self) -> "AIMessages":
+        """Create an unfrozen deep copy of the list.
+
+        Returns:
+            New unfrozen AIMessages with deep-copied messages.
+        """
+        copied_messages = deepcopy(list(self))
+        return AIMessages(copied_messages, frozen=False)
+
+    def _check_frozen(self) -> None:
+        """Check if list is frozen and raise if it is.
+
+        Raises:
+            RuntimeError: If the list is frozen.
+        """
+        if self._frozen:
+            raise RuntimeError("Cannot modify frozen AIMessages")
+
+    def append(self, message: AIMessageType) -> None:
+        """Add a message to the end of the list."""
+        self._check_frozen()
+        super().append(message)
+
+    def extend(self, messages: Iterable[AIMessageType]) -> None:
+        """Add multiple messages to the list."""
+        self._check_frozen()
+        super().extend(messages)
+
+    def insert(self, index: SupportsIndex, message: AIMessageType) -> None:
+        """Insert a message at the specified position."""
+        self._check_frozen()
+        super().insert(index, message)
+
+    def __setitem__(
+        self,
+        index: Union[SupportsIndex, slice],
+        value: Union[AIMessageType, Iterable[AIMessageType]],
+    ) -> None:
+        """Set item or slice."""
+        self._check_frozen()
+        super().__setitem__(index, value)  # type: ignore[arg-type]
+
+    def __iadd__(self, other: Iterable[AIMessageType]) -> "AIMessages":
+        """In-place addition (+=).
+
+        Returns:
+            This AIMessages instance after modification.
+        """
+        self._check_frozen()
+        return super().__iadd__(other)
+
+    def __delitem__(self, index: Union[SupportsIndex, slice]) -> None:
+        """Delete item or slice from list."""
+        self._check_frozen()
+        super().__delitem__(index)
+
+    def pop(self, index: SupportsIndex = -1) -> AIMessageType:
+        """Remove and return item at index.
+
+        Returns:
+            AIMessageType removed from the list.
+        """
+        self._check_frozen()
+        return super().pop(index)
+
+    def remove(self, message: AIMessageType) -> None:
+        """Remove first occurrence of message."""
+        self._check_frozen()
+        super().remove(message)
+
+    def clear(self) -> None:
+        """Remove all items from list."""
+        self._check_frozen()
+        super().clear()
+
+    def reverse(self) -> None:
+        """Reverse list in place."""
+        self._check_frozen()
+        super().reverse()
+
+    def sort(
+        self, *, key: Callable[[AIMessageType], Any] | None = None, reverse: bool = False
+    ) -> None:
+        """Sort list in place."""
+        self._check_frozen()
+        if key is None:
+            super().sort(reverse=reverse)  # type: ignore[call-arg]
+        else:
+            super().sort(key=key, reverse=reverse)
+
     def get_last_message(self) -> AIMessageType:
         """Get the last message in the conversation.
 

ai_pipeline_core/llm/client.py

@@ -37,7 +37,7 @@ def _process_messages(
     context: AIMessages,
     messages: AIMessages,
     system_prompt: str | None = None,
-    cache_ttl: str | None = "120s",
+    cache_ttl: str | None = "5m",
 ) -> list[ChatCompletionMessageParam]:
     """Process and format messages for LLM API consumption.
 
@@ -245,7 +245,7 @@ async def generate(
         model: Model to use (e.g., "gpt-5", "gemini-2.5-pro", "grok-4").
                Accepts predefined models or any string for custom models.
         context: Static context to cache (documents, examples, instructions).
-                Defaults to None (empty context). Cached for 120 seconds.
+                Defaults to None (empty context). Cached for 5 minutes by default.
         messages: Dynamic messages/queries. AIMessages or str ONLY.
                  Do not pass Document or DocumentList directly.
                  If string, converted to AIMessages internally.
@@ -338,13 +338,13 @@ async def generate(
         - Context caching saves ~50-90% tokens on repeated calls
         - First call: full token cost
         - Subsequent calls (within cache TTL): only messages tokens
-        - Default cache TTL is 120s (production-optimized)
+        - Default cache TTL is 5m (production-optimized)
         - Default retry logic: 3 attempts with 10s delay (production-optimized)
 
     Caching:
         When enabled in your LiteLLM proxy and supported by the upstream provider,
         context messages may be cached to reduce token usage on repeated calls.
-        Default TTL is 120s (optimized for production workloads). Configure caching
+        Default TTL is 5m (optimized for production workloads). Configure caching
         behavior centrally via your LiteLLM proxy settings, not per API call.
         Savings depend on provider and payload; treat this as an optimization, not a guarantee.
 
@@ -524,6 +524,8 @@ async def generate_structured(
     if isinstance(messages, str):
         messages = AIMessages([messages])
 
+    assert isinstance(messages, AIMessages)
+
     # Call the internal generate function with structured output enabled
     try:
         response = await _generate_with_retry(model, context, messages, options)
@@ -555,9 +557,3 @@ async def generate_structured(
 
     # Create a StructuredModelResponse with the parsed value
     return StructuredModelResponse[T](chat_completion=response, parsed_value=parsed_value)
-
-
-# Public aliases for testing internal functions
-# These are exported to allow testing of implementation details
-process_messages_for_testing = _process_messages
-generate_with_retry_for_testing = _generate_with_retry

ai_pipeline_core/llm/model_options.py

@@ -45,7 +45,7 @@ class ModelOptions(BaseModel):
 
         timeout: Maximum seconds to wait for response (default: 300).
 
-        cache_ttl: Cache TTL for context messages (default: "120s").
+        cache_ttl: Cache TTL for context messages (default: "5m").
                    String format like "60s", "5m", or None to disable caching.
                    Applied to the last context message for efficient token reuse.
 
@@ -109,7 +109,7 @@ class ModelOptions(BaseModel):
         - search_context_size only works with search models
         - reasoning_effort only works with models that support explicit reasoning
         - response_format is set internally by generate_structured()
-        - cache_ttl accepts formats like "120s", "5m", "1h" or None to disable caching
+        - cache_ttl accepts formats like "120s", "5m" (default), "1h" or None to disable caching
     """
 
     temperature: float | None = None
@@ -118,11 +118,13 @@ class ModelOptions(BaseModel):
     reasoning_effort: Literal["low", "medium", "high"] | None = None
     retries: int = 3
     retry_delay_seconds: int = 10
-    timeout: int = 300
-    cache_ttl: str | None = "120s"
+    timeout: int = 600
+    cache_ttl: str | None = "5m"
     service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
     max_completion_tokens: int | None = None
     response_format: type[BaseModel] | None = None
+    verbosity: Literal["low", "medium", "high"] | None = None
+    usage_tracking: bool = True
 
     def to_openai_completion_kwargs(self) -> dict[str, Any]:
         """Convert options to OpenAI API completion parameters.
@@ -159,7 +161,7 @@ class ModelOptions(BaseModel):
         Note:
             - system_prompt is handled separately in _process_messages()
             - retries and retry_delay_seconds are used by retry logic
-            - extra_body is always included for potential extensions
+            - extra_body always includes usage tracking for cost monitoring
         """
         kwargs: dict[str, Any] = {
             "timeout": self.timeout,
@@ -184,4 +186,10 @@ class ModelOptions(BaseModel):
         if self.service_tier:
             kwargs["service_tier"] = self.service_tier
 
+        if self.verbosity:
+            kwargs["verbosity"] = self.verbosity
+
+        if self.usage_tracking:
+            kwargs["extra_body"]["usage"] = {"include": True}
+
         return kwargs

ai_pipeline_core/llm/model_response.py

@@ -110,7 +110,8 @@ class ModelResponse(ChatCompletion):
             >>> if "error" in response.content.lower():
             ...     # Handle error case
         """
-        return self.choices[0].message.content or ""
+        content = self.choices[0].message.content or ""
+        return content.split("</think>")[-1].strip()
 
     def set_model_options(self, options: dict[str, Any]) -> None:
         """Store the model configuration used for generation.

ai_pipeline_core/llm/model_types.py

@@ -21,12 +21,12 @@ ModelName: TypeAlias = (
         # Small models
         "gemini-2.5-flash",
         "gpt-5-mini",
-        "grok-3-mini",
+        "grok-4-fast",
         # Search models
         "gemini-2.5-flash-search",
         "sonar-pro-search",
         "gpt-4o-search",
-        "grok-3-mini-search",
+        "grok-4-fast-search",
     ]
     | str
 )
@@ -47,7 +47,7 @@ Model categories:
         High-capability models for complex tasks requiring deep reasoning,
         nuanced understanding, or creative generation.
 
-    Small models (gemini-2.5-flash, gpt-5-mini, grok-3-mini):
+    Small models (gemini-2.5-flash, gpt-5-mini, grok-4-fast):
         Efficient models optimized for speed and cost, suitable for
         simpler tasks or high-volume processing.
 

ai_pipeline_core/pipeline.py

@@ -222,6 +222,7 @@ def pipeline_task(
     trace_input_formatter: Callable[..., str] | None = None,
     trace_output_formatter: Callable[..., str] | None = None,
     trace_cost: float | None = None,
+    trace_trim_documents: bool = True,
     # prefect passthrough
     name: str | None = None,
     description: str | None = None,
@@ -262,6 +263,7 @@ def pipeline_task(
     trace_input_formatter: Callable[..., str] | None = None,
     trace_output_formatter: Callable[..., str] | None = None,
     trace_cost: float | None = None,
+    trace_trim_documents: bool = True,
     # prefect passthrough
     name: str | None = None,
     description: str | None = None,
@@ -318,6 +320,8 @@ def pipeline_task(
         trace_cost: Optional cost value to track in metadata. When provided and > 0,
              sets gen_ai.usage.output_cost, gen_ai.usage.cost, and cost metadata.
              Also forces trace level to "always" if not already set.
+        trace_trim_documents: Trim document content in traces to first 100 chars (default True).
+                             Reduces trace size with large documents.
 
         Prefect task parameters:
         name: Task name (defaults to function name).
@@ -424,6 +428,7 @@ def pipeline_task(
             ignore_inputs=trace_ignore_inputs,
             input_formatter=trace_input_formatter,
             output_formatter=trace_output_formatter,
+            trim_documents=trace_trim_documents,
         )(_wrapper)
 
         return cast(
@@ -474,6 +479,7 @@ def pipeline_flow(
     trace_input_formatter: Callable[..., str] | None = None,
     trace_output_formatter: Callable[..., str] | None = None,
     trace_cost: float | None = None,
+    trace_trim_documents: bool = True,
     # prefect passthrough
     name: str | None = None,
     version: str | None = None,
@@ -536,6 +542,8 @@ def pipeline_flow(
         trace_cost: Optional cost value to track in metadata. When provided and > 0,
              sets gen_ai.usage.output_cost, gen_ai.usage.cost, and cost metadata.
              Also forces trace level to "always" if not already set.
+        trace_trim_documents: Trim document content in traces to first 100 chars (default True).
+                             Reduces trace size with large documents.
 
         Prefect flow parameters:
         name: Flow name (defaults to function name).
@@ -670,6 +678,7 @@ def pipeline_flow(
             ignore_inputs=trace_ignore_inputs,
             input_formatter=trace_input_formatter,
             output_formatter=trace_output_formatter,
+            trim_documents=trace_trim_documents,
         )(_wrapper)
 
         # --- Publish a schema where `documents` accepts str (path) OR DocumentList ---

ai_pipeline_core/prompt_manager.py

@@ -18,6 +18,8 @@ Key features:
     - Jinja2 template rendering with context
     - Smart path resolution (.jinja2/.jinja extension handling)
     - Clear error messages for missing templates
+    - Built-in global variables:
+        - current_date: Current date in format "03 January 2025" (string)
 
 Example:
     >>> from ai_pipeline_core import PromptManager
@@ -45,6 +47,7 @@ Note:
     The extension can be omitted when calling get().
 """
 
+from datetime import datetime
 from pathlib import Path
 from typing import Any
 
@@ -103,6 +106,8 @@ class PromptManager:
         {% if instructions %}
         Instructions: {{ instructions }}
         {% endif %}
+
+        Date: {{ current_date }}  # Current date in format "03 January 2025"
         ```
 
     Note:
@@ -214,6 +219,9 @@ class PromptManager:
             autoescape=False,  # Important for prompt engineering
         )
 
+        # Add current_date as a global string (format: "03 January 2025")
+        self.env.globals["current_date"] = datetime.now().strftime("%d %B %Y")  # type: ignore[assignment]
+
     def get(self, prompt_path: str, **kwargs: Any) -> str:
         """Load and render a Jinja2 template with the given context.
 

ai_pipeline_core/tracing.py

@@ -9,6 +9,7 @@ This module centralizes:
 from __future__ import annotations
 
 import inspect
+import json
 import os
 from functools import wraps
 from typing import Any, Callable, Literal, ParamSpec, TypeVar, cast, overload
@@ -16,6 +17,10 @@ from typing import Any, Callable, Literal, ParamSpec, TypeVar, cast, overload
 from lmnr import Attributes, Instruments, Laminar, observe
 from pydantic import BaseModel
 
+# Import for document trimming - needed for isinstance checks
+# These are lazy imports only used when trim_documents is enabled
+from ai_pipeline_core.documents import Document, DocumentList
+from ai_pipeline_core.llm import AIMessages, ModelResponse
 from ai_pipeline_core.settings import settings
 
 # ---------------------------------------------------------------------------
@@ -34,6 +39,145 @@ Values:
 """
 
 
+# ---------------------------------------------------------------------------
+# Serialization helpers
+# ---------------------------------------------------------------------------
+def _serialize_for_tracing(obj: Any) -> Any:
+    """Convert objects to JSON-serializable format for tracing.
+
+    Handles Pydantic models, Documents, and other special types.
+    This is extracted for better testability.
+
+    Args:
+        obj: Object to serialize
+
+    Returns:
+        JSON-serializable representation of the object
+    """
+    # Our Document types - handle first to ensure serialize_model is used
+    if isinstance(obj, Document):
+        return obj.serialize_model()
+    # DocumentList
+    if isinstance(obj, DocumentList):
+        return [doc.serialize_model() for doc in obj]
+    # AIMessages
+    if isinstance(obj, AIMessages):
+        result = []
+        for msg in obj:
+            if isinstance(msg, Document):
+                result.append(msg.serialize_model())
+            else:
+                result.append(msg)
+        return result
+    # ModelResponse (special Pydantic model) - use standard model_dump
+    if isinstance(obj, ModelResponse):
+        return obj.model_dump()
+    # Pydantic models - use custom serializer that respects Document.serialize_model()
+    if isinstance(obj, BaseModel):
+        # For Pydantic models, we need to handle Document fields specially
+        data = {}
+        for field_name, field_value in obj.__dict__.items():
+            if isinstance(field_value, Document):
+                # Use serialize_model for Documents to get base_type
+                data[field_name] = field_value.serialize_model()
+            elif isinstance(field_value, BaseModel):
+                # Recursively handle nested Pydantic models
+                data[field_name] = _serialize_for_tracing(field_value)
+            else:
+                # Let Pydantic handle other fields normally
+                data[field_name] = field_value
+        return data
+    # Fallback to string representation
+    try:
+        return str(obj)
+    except Exception:
+        return f"<{type(obj).__name__}>"
+
+
+# ---------------------------------------------------------------------------
+# Document trimming utilities
+# ---------------------------------------------------------------------------
+def _trim_document_content(doc_dict: dict[str, Any]) -> dict[str, Any]:
+    """Trim document content based on document type and content type.
+
+    For non-FlowDocuments:
+    - Text content: Keep first 100 and last 100 chars (unless < 250 total)
+    - Binary content: Remove content entirely
+
+    For FlowDocuments:
+    - Text content: Keep full content
+    - Binary content: Remove content entirely
+
+    Args:
+        doc_dict: Document dictionary with base_type, content, and content_encoding
+
+    Returns:
+        Modified document dictionary with trimmed content
+    """
+    # Check if this looks like a document (has required fields)
+    if not isinstance(doc_dict, dict):  # type: ignore[reportUnknownArgumentType]
+        return doc_dict
+
+    if "base_type" not in doc_dict or "content" not in doc_dict:
+        return doc_dict
+
+    base_type = doc_dict.get("base_type")
+    content = doc_dict.get("content", "")
+    content_encoding = doc_dict.get("content_encoding", "utf-8")
+
+    # For binary content (base64 encoded), remove content
+    if content_encoding == "base64":
+        doc_dict = doc_dict.copy()
+        doc_dict["content"] = "[binary content removed]"
+        return doc_dict
+
+    # For FlowDocuments with text content, keep full content
+    if base_type == "flow":
+        return doc_dict
+
+    # For other documents (task, temporary), trim text content
+    if isinstance(content, str) and len(content) > 250:
+        doc_dict = doc_dict.copy()
+        # Keep first 100 and last 100 characters
+        trimmed_chars = len(content) - 200  # Number of characters removed
+        doc_dict["content"] = (
+            content[:100] + f" ... [trimmed {trimmed_chars} chars] ... " + content[-100:]
+        )
+
+    return doc_dict
+
+
+def _trim_documents_in_data(data: Any) -> Any:
+    """Recursively trim document content in nested data structures.
+
+    Processes dictionaries, lists, and nested structures to find and trim
+    documents based on their type and content.
+
+    Args:
+        data: Input data that may contain documents
+
+    Returns:
+        Data with document content trimmed according to rules
+    """
+    if isinstance(data, dict):
+        # Check if this is a document
+        if "base_type" in data and "content" in data:
+            # This is a document, trim it
+            return _trim_document_content(data)
+        else:
+            # Recursively process dictionary values
+            return {k: _trim_documents_in_data(v) for k, v in data.items()}
+    elif isinstance(data, list):
+        # Process each item in list
+        return [_trim_documents_in_data(item) for item in data]
+    elif isinstance(data, tuple):
+        # Process tuples
+        return tuple(_trim_documents_in_data(item) for item in data)
+    else:
+        # Return other types unchanged
+        return data
+
+
 # ---------------------------------------------------------------------------
 # ``TraceInfo`` – metadata container
 # ---------------------------------------------------------------------------
@@ -175,6 +319,7 @@ def trace(
     output_formatter: Callable[..., str] | None = None,
     ignore_exceptions: bool = False,
     preserve_global_context: bool = True,
+    trim_documents: bool = True,
 ) -> Callable[[Callable[P, R]], Callable[P, R]]: ...
 
 
@@ -201,6 +346,7 @@ def trace(
     output_formatter: Callable[..., str] | None = None,
     ignore_exceptions: bool = False,
     preserve_global_context: bool = True,
+    trim_documents: bool = True,
 ) -> Callable[[Callable[P, R]], Callable[P, R]] | Callable[P, R]:
     """Add Laminar observability tracing to any function.
 
@@ -257,6 +403,12 @@ def trace(
         preserve_global_context: Maintain Laminar's global context across
                                 calls (default True). Set False for isolated traces.
 
+        trim_documents: Automatically trim document content in traces (default True).
+                       When enabled, non-FlowDocument text content is trimmed to
+                       first/last 100 chars, and all binary content is removed.
+                       FlowDocuments keep full text content but binary is removed.
+                       Helps reduce trace size for large documents.
+
     Returns:
         Decorated function with same signature but added tracing.
 
@@ -363,6 +515,72 @@ def trace(
         _output_formatter = output_formatter
         _ignore_exceptions = ignore_exceptions
         _preserve_global_context = preserve_global_context
+        _trim_documents = trim_documents
+
+        # Create document trimming formatters if needed
+        def _create_trimming_input_formatter(*args, **kwargs) -> str:
+            # First, let any custom formatter process the data
+            if _input_formatter:
+                result = _input_formatter(*args, **kwargs)
+                # If formatter returns string, try to parse and trim
+                if isinstance(result, str):  # type: ignore[reportUnknownArgumentType]
+                    try:
+                        data = json.loads(result)
+                        trimmed = _trim_documents_in_data(data)
+                        return json.dumps(trimmed)
+                    except (json.JSONDecodeError, TypeError):
+                        return result
+                else:
+                    # If formatter returns dict/list, trim it
+                    trimmed = _trim_documents_in_data(result)
+                    return json.dumps(trimmed) if not isinstance(trimmed, str) else trimmed
+            else:
+                # No custom formatter - mimic Laminar's get_input_from_func_args
+                # Build a dict with parameter names as keys (like Laminar does)
+                params = list(sig.parameters.keys())
+                data = {}
+
+                # Map args to parameter names
+                for i, arg in enumerate(args):
+                    if i < len(params):
+                        data[params[i]] = arg
+
+                # Add kwargs
+                data.update(kwargs)
+
+                # Serialize with our helper function
+                serialized = json.dumps(data, default=_serialize_for_tracing)
+                parsed = json.loads(serialized)
+
+                # Trim documents in the serialized data
+                trimmed = _trim_documents_in_data(parsed)
+                return json.dumps(trimmed)
+
+        def _create_trimming_output_formatter(result: Any) -> str:
+            # First, let any custom formatter process the data
+            if _output_formatter:
+                formatted = _output_formatter(result)
+                # If formatter returns string, try to parse and trim
+                if isinstance(formatted, str):  # type: ignore[reportUnknownArgumentType]
+                    try:
+                        data = json.loads(formatted)
+                        trimmed = _trim_documents_in_data(data)
+                        return json.dumps(trimmed)
+                    except (json.JSONDecodeError, TypeError):
+                        return formatted
+                else:
+                    # If formatter returns dict/list, trim it
+                    trimmed = _trim_documents_in_data(formatted)
+                    return json.dumps(trimmed) if not isinstance(trimmed, str) else trimmed
+            else:
+                # No custom formatter, serialize result with smart defaults
+                # Serialize with our extracted helper function
+                serialized = json.dumps(result, default=_serialize_for_tracing)
+                parsed = json.loads(serialized)
+
+                # Trim documents in the serialized data
+                trimmed = _trim_documents_in_data(parsed)
+                return json.dumps(trimmed)
 
         # --- Helper function for runtime logic ---
         def _prepare_and_get_observe_params(runtime_kwargs: dict[str, Any]) -> dict[str, Any]:
@@ -401,10 +619,19 @@ def trace(
                 observe_params["ignore_output"] = _ignore_output
             if _ignore_inputs is not None:
                 observe_params["ignore_inputs"] = _ignore_inputs
-            if _input_formatter is not None:
-                observe_params["input_formatter"] = _input_formatter
-            if _output_formatter is not None:
-                observe_params["output_formatter"] = _output_formatter
+
+            # Use trimming formatters if trim_documents is enabled
+            if _trim_documents:
+                # Use the trimming formatters (which may wrap custom formatters)
+                observe_params["input_formatter"] = _create_trimming_input_formatter
+                observe_params["output_formatter"] = _create_trimming_output_formatter
+            else:
+                # Use custom formatters directly if provided
+                if _input_formatter is not None:
+                    observe_params["input_formatter"] = _input_formatter
+                if _output_formatter is not None:
+                    observe_params["output_formatter"] = _output_formatter
+
             if _ignore_exceptions:
                 observe_params["ignore_exceptions"] = _ignore_exceptions
             if _preserve_global_context:

{ai_pipeline_core-0.2.0.dist-info → ai_pipeline_core-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-pipeline-core
-Version: 0.2.0
+Version: 0.2.2
 Summary: Core utilities for AI-powered processing pipelines using prefect
 Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
 Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core
@@ -20,10 +20,10 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.12
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: jinja2>=3.1.6
-Requires-Dist: lmnr>=0.7.6
-Requires-Dist: openai>=1.99.9
+Requires-Dist: lmnr>=0.7.13
+Requires-Dist: openai>=1.108.1
 Requires-Dist: prefect-gcp[cloud-storage]>=0.6.10
-Requires-Dist: prefect>=3.4.13
+Requires-Dist: prefect>=3.4.19
 Requires-Dist: pydantic-settings>=2.10.1
 Requires-Dist: pydantic>=2.11.7
 Requires-Dist: python-magic>=0.4.27
@@ -224,9 +224,17 @@ if doc.is_text:
 # Parse structured data
 data = doc.as_json()  # or as_yaml(), as_pydantic_model()
 
+# Convert between document types (new in v0.2.1)
+task_doc = flow_doc.model_convert(TaskDocument)  # Convert FlowDocument to TaskDocument
+new_doc = doc.model_convert(OtherDocType, content={"new": "data"})  # With content update
+
 # Enhanced filtering (new in v0.1.14)
 filtered = documents.filter_by([Doc1, Doc2, Doc3])  # Multiple types
 named = documents.filter_by(["file1.txt", "file2.txt"])  # Multiple names
+
+# Immutable collections (new in v0.2.1)
+frozen_docs = DocumentList(docs, frozen=True)  # Immutable document list
+frozen_msgs = AIMessages(messages, frozen=True)  # Immutable message list
 ```
 
 ### LLM Integration
@@ -312,13 +320,18 @@ async def process_chunk(data: str) -> str:
     set_trace_cost(0.05)  # Track costs (new in v0.1.14)
     return result
 
-@pipeline_flow(config=MyFlowConfig)  # Full observability and orchestration
+@pipeline_flow(
+    config=MyFlowConfig,
+    trace_trim_documents=True  # Trim large documents in traces (new in v0.2.1)
+)
 async def main_flow(
     project_name: str,
     documents: DocumentList,
     flow_options: FlowOptions
 ) -> DocumentList:
     # Your pipeline logic
+    # Large documents are automatically trimmed to 100 chars in traces
+    # for better observability without overwhelming the tracing UI
     return DocumentList(results)
 ```
 

{ai_pipeline_core-0.2.0.dist-info → ai_pipeline_core-0.2.2.dist-info}/RECORD

@@ -1,14 +1,14 @@
-ai_pipeline_core/__init__.py,sha256=BBZn5MBlfCWAq1nFwNxsKnvBLfmPB43TnSEH7edde64,5720
+ai_pipeline_core/__init__.py,sha256=LH0lGm02zWS9l7b3uzvvzOfSh7eDPok7RjVTP2_-Mv0,5720
 ai_pipeline_core/exceptions.py,sha256=vx-XLTw2fJSPs-vwtXVYtqoQUcOc0JeI7UmHqRqQYWU,1569
-ai_pipeline_core/pipeline.py,sha256=z3zTHAvDkXAsTJEzkpw1gXonNH8hioNAN2wUybGa1j0,28372
+ai_pipeline_core/pipeline.py,sha256=_00Qctqd7QibyXaetZv6KfyWoW9KZIRdndkYItNHWWI,28921
 ai_pipeline_core/prefect.py,sha256=91ZgLJHsDsRUW77CpNmkKxYs3RCJuucPM3pjKmNBeDg,2199
-ai_pipeline_core/prompt_manager.py,sha256=p7D0vv_nMmny0rmvxrVyYmXPRjmPJo9qI-pRZe4__Bk,11690
+ai_pipeline_core/prompt_manager.py,sha256=FAtb1yK7bGuAeuIJ523LOX9bd7TrcHG-TqZ7Lz4RJC0,12087
 ai_pipeline_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ai_pipeline_core/settings.py,sha256=-a9jVGg77xifj2SagCR9shXfzXUd-2MlrlquEu4htG8,5035
-ai_pipeline_core/tracing.py,sha256=gy5E4OVr3KX-wZ4zWkk3RSER5Ulw8Q_qyp9YoEMCRj4,21963
+ai_pipeline_core/tracing.py,sha256=9RaJaAX5Vp2C8t73TaY-a9gpVy6a_VtSY0JPohIoQsc,31460
 ai_pipeline_core/documents/__init__.py,sha256=WHStvGZiSyybOcMTYxSV24U6MA3Am_0_Az5p-DuMFrk,738
-ai_pipeline_core/documents/document.py,sha256=dCbKcgemW8nWHEUJdK5MqSe3XTIo-voi6td4hqWqvSw,62270
-ai_pipeline_core/documents/document_list.py,sha256=iiW8p5p8PhoUgMJnCnPE5GF5xpf6lWJKIzDyYEf6BkM,13738
+ai_pipeline_core/documents/document.py,sha256=L3S_bfOiViMZLYRcmbV4-s3qO8HoGmqJ5g3bXNVs_3Q,67082
+ai_pipeline_core/documents/document_list.py,sha256=Y_NCjfM_CjkIwHRD2iyGgYBuIykN8lT2IIH_uWOiGis,16254
 ai_pipeline_core/documents/flow_document.py,sha256=g9wlRJRJgy4RsrrZ_P5Qu6kj0FuUFfhfUsRFgtq4NIM,3918
 ai_pipeline_core/documents/mime_type.py,sha256=DkW88K95el5nAmhC00XLS0G3WpDXgs5IRsBWbKiqG3Y,7995
 ai_pipeline_core/documents/task_document.py,sha256=40tFavBLX3FhK9-CRsuOH-3gUZ0zvEkqv9XcMFr8ySk,4077
@@ -17,12 +17,12 @@ ai_pipeline_core/documents/utils.py,sha256=ZyJNjFN7ihWno0K7dJZed7twYmmPLA0z40UzF
 ai_pipeline_core/flow/__init__.py,sha256=2BfWYMOPYW5teGzwo-qzpn_bom1lxxry0bPsjVgcsCk,188
 ai_pipeline_core/flow/config.py,sha256=3PCDph2n8dj-txqAvd9Wflbi_6lmfXFR9rUhM-szGSQ,18887
 ai_pipeline_core/flow/options.py,sha256=2rKR2GifhXcyw8avI_oiEDMLC2jm5Qzpw8z56pbxUMo,2285
-ai_pipeline_core/llm/__init__.py,sha256=tSj3Mll8SebivP4J5khdXhM9fnbujnbRh0i5yQRoDJQ,857
-ai_pipeline_core/llm/ai_messages.py,sha256=eSmMwTqGvtBeMoWuukzciQRDIIAfs-cnEXjlaADIYkw,9027
-ai_pipeline_core/llm/client.py,sha256=pxedLnxb9dEu5I9XHTFgXEYWxMv7HOVHhESxIw1hANA,22946
-ai_pipeline_core/llm/model_options.py,sha256=YT_lHazZPa0IbHOuLbWXerRODEDb62sKFM97olSxcAU,7693
-ai_pipeline_core/llm/model_response.py,sha256=xKJPsqFHtOGfqpKlsGzyBHPbqjEjNfP-Ix3lGVdiTjQ,15289
-ai_pipeline_core/llm/model_types.py,sha256=HrQCe_R86yWv5z_83yB-zoMFp6M5Ee9nSeimZmckqtA,2791
+ai_pipeline_core/llm/__init__.py,sha256=3B_vtEzxrzidP1qOUNQ4RxlUmxZ2MBKQcUhQiTybM9g,661
+ai_pipeline_core/llm/ai_messages.py,sha256=ML4rSCCEEu9_83Mnfn7r4yx0pUkarvnBsrxRZbO4ulw,13126
+ai_pipeline_core/llm/client.py,sha256=oByE8whI1lvyqYUh6q3tKgXJhDiWiJWGztlfoZswrFE,22776
+ai_pipeline_core/llm/model_options.py,sha256=7J9qt7P1qCnSP_NrBzPwx_P-HwkXDYFxKcYzriIJ3U4,7972
+ai_pipeline_core/llm/model_response.py,sha256=iNSKobR3gzZ-CSC8hz8-grgL7jdd2IcnCSX0exdlg7o,15345
+ai_pipeline_core/llm/model_types.py,sha256=2J4Qsb1x21I4eo_VPeaMMOW8shOGPqzJuoGjTLcBFPM,2791
 ai_pipeline_core/logging/__init__.py,sha256=Nz6-ghAoENsgNmLD2ma9TW9M0U2_QfxuQ5DDW6Vt6M0,651
 ai_pipeline_core/logging/logging.yml,sha256=YTW48keO_K5bkkb-KXGM7ZuaYKiquLsjsURei8Ql0V4,1353
 ai_pipeline_core/logging/logging_config.py,sha256=pV2x6GgMPXrzPH27sicCSXfw56beio4C2JKCJ3NsXrg,6207
@@ -32,7 +32,7 @@ ai_pipeline_core/simple_runner/cli.py,sha256=yVyuxLY2RZvdNwmwT5LCe-km2nQJzWTPI0v
 ai_pipeline_core/simple_runner/simple_runner.py,sha256=f6cIodYkul-Apu1d63T6kR5DZpiaCWpphUcEPp5XjFo,9102
 ai_pipeline_core/storage/__init__.py,sha256=tcIkjJ3zPBLCyetwiJDewBvS2sbRJrDlBh3gEsQm08E,184
 ai_pipeline_core/storage/storage.py,sha256=ClMr419Y-eU2RuOjZYd51dC0stWQk28Vb56PvQaoUwc,20007
-ai_pipeline_core-0.2.0.dist-info/METADATA,sha256=3U4rWNVFQ_Agwpv6NH2e4k0falKwNpDHn-6GncwbcSs,14556
-ai_pipeline_core-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ai_pipeline_core-0.2.0.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
-ai_pipeline_core-0.2.0.dist-info/RECORD,,
+ai_pipeline_core-0.2.2.dist-info/METADATA,sha256=EbqjpaeIwuScRMLTKdfYdut57O8GMUZ-HWYcioQ9r1A,15159
+ai_pipeline_core-0.2.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ai_pipeline_core-0.2.2.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
+ai_pipeline_core-0.2.2.dist-info/RECORD,,