ai-pipeline-core 0.1.7__tar.gz → 0.1.10__tar.gz

{ai_pipeline_core-0.1.7 → ai_pipeline_core-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-pipeline-core
-Version: 0.1.7
+Version: 0.1.10
 Summary: Core utilities for AI-powered processing pipelines using prefect
 Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
 Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core
@@ -151,7 +151,7 @@ async def process_document(doc: Document):
     return response.parsed
 ```
 
-### Enhanced Pipeline Decorators (New in v0.1.7)
+### Enhanced Pipeline Decorators
 ```python
 from ai_pipeline_core import pipeline_flow, pipeline_task
 from ai_pipeline_core.flow import FlowOptions
@@ -182,7 +182,7 @@ async def my_pipeline(
     return DocumentList(results)
 ```
 
-### Simple Runner Utility (New in v0.1.7)
+### Simple Runner Utility
 ```python
 from ai_pipeline_core.simple_runner import run_cli, run_pipeline
 from ai_pipeline_core.flow import FlowOptions
@@ -206,7 +206,7 @@ async def main():
     )
 ```
 
-### Clean Prefect Decorators (New in v0.1.7)
+### Clean Prefect Decorators
 ```python
 # Import clean Prefect decorators without tracing
 from ai_pipeline_core.prefect import flow, task
@@ -214,12 +214,12 @@ from ai_pipeline_core.prefect import flow, task
 # Or use pipeline decorators with tracing
 from ai_pipeline_core import pipeline_flow, pipeline_task
 
-@task  # Clean Prefect task
+@task  # Clean Prefect task (supports both sync and async)
 def compute(x: int) -> int:
     return x * 2
 
-@pipeline_task(trace_level="always")  # With tracing
-def compute_traced(x: int) -> int:
+@pipeline_task(trace_level="always")  # With tracing (async only)
+async def compute_traced(x: int) -> int:
     return x * 2
 ```
 
@@ -246,12 +246,12 @@ docs = DocumentList([doc1, doc2])
 Managed AI interactions with built-in retry logic, cost tracking, and structured outputs.
 
 **Supported Models** (via LiteLLM proxy):
-- OpenAI: GPT-4, GPT-5 series
-- Anthropic: Claude 3 series
-- Google: Gemini 2.5 series
-- xAI: Grok models
-- Perplexity: Sonar models (with search capabilities)
-- And many more through LiteLLM compatibility
+- OpenAI: gpt-5
+- Anthropic: claude-4
+- Google: gemini-2.5
+- xAI: grok-3, grok-4
+- Perplexity: sonar-pro-search
+- And many more through LiteLLM compatibility. Every model from openrouter should work.
 
 ```python
 from ai_pipeline_core.llm import generate_structured, AIMessages, ModelOptions
@@ -328,13 +328,13 @@ ai_pipeline_core/
 │   └── model_options.py # Configuration models
 ├── flow/              # Prefect flow utilities
 │   ├── config.py      # Type-safe flow configuration
-│   └── options.py     # FlowOptions base class (v0.1.7)
-├── simple_runner/     # Pipeline execution utilities (v0.1.7)
+│   └── options.py     # FlowOptions base class
+├── simple_runner/     # Pipeline execution utilities
 │   ├── cli.py         # CLI interface
 │   └── simple_runner.py # Core runner logic
 ├── logging/           # Structured logging
-├── pipeline.py        # Enhanced decorators (v0.1.7)
-├── prefect.py         # Clean Prefect exports (v0.1.7)
+├── pipeline.py        # Enhanced decorators
+├── prefect.py         # Clean Prefect exports
 ├── tracing.py         # Observability decorators
 └── settings.py        # Centralized configuration
 ```
@@ -345,6 +345,7 @@ ai_pipeline_core/
 ```bash
 make test           # Run all tests
 make test-cov      # Run with coverage report
+make test-showcase # Test the showcase.py CLI example
 pytest tests/test_documents.py::TestDocument::test_creation  # Single test
 ```
 
@@ -481,6 +482,22 @@ For learning purposes, see [CLAUDE.md](CLAUDE.md) for our comprehensive coding s
 
 - [CLAUDE.md](CLAUDE.md) - Detailed coding standards and architecture guide
 
+## Examples
+
+### In This Repository
+- [showcase.py](examples/showcase.py) - Complete example demonstrating all core features including the CLI runner
+  ```bash
+  # Run the showcase example with CLI
+  python examples/showcase.py ./output --temperature 0.7 --batch-size 5
+
+  # Show help
+  python examples/showcase.py --help
+  ```
+- [showcase.jinja2](examples/showcase.jinja2) - Example Jinja2 prompt template
+
+### Real-World Application
+- [AI Documentation Writer](https://github.com/bbarwik/ai-documentation-writer) - Production-ready example showing how to build sophisticated AI pipelines for automated documentation generation. See [examples/ai-documentation-writer.md](examples/ai-documentation-writer.md) for a detailed overview.
+
 ### dependencies_docs/ Directory
 > [!NOTE]
 > The `dependencies_docs/` directory contains guides for AI assistants (like Claude Code) on how to interact with the project's external dependencies and tooling, NOT user documentation for ai-pipeline-core itself. These files are excluded from repository listings to avoid confusion.
@@ -511,29 +528,9 @@ Built with:
 - [LiteLLM](https://litellm.ai/) - LLM proxy
 - [Pydantic](https://pydantic-docs.helpmanual.io/) - Data validation
 
-## What's New in v0.1.7
-
-### Major Additions
-- **Enhanced Pipeline Decorators**: New `pipeline_flow` and `pipeline_task` decorators combining Prefect functionality with automatic LMNR tracing
-- **FlowOptions Base Class**: Extensible configuration system for flows with type-safe inheritance
-- **Simple Runner Module**: CLI and programmatic utilities for easy pipeline execution
-- **Clean Prefect Exports**: Separate imports for Prefect decorators with and without tracing
-- **Expanded Exports**: All major components now accessible from top-level package import
-
-### API Improvements
-- Better type inference for document flows with custom options
-- Support for custom FlowOptions inheritance in pipeline flows
-- Improved error messages for invalid flow signatures
-- Enhanced document utility functions (`canonical_name_key`, `sanitize_url`)
-
-### Developer Experience
-- Simplified imports - most components available from `ai_pipeline_core` directly
-- Better separation of concerns between clean Prefect and traced pipeline decorators
-- More intuitive flow configuration with `FlowOptions` inheritance
-
 ## Stability Notice
 
-**Current Version**: 0.1.7
+**Current Version**: 0.1.10
 **Status**: Internal Preview
 **API Stability**: Unstable - Breaking changes expected
 **Recommended Use**: Learning and reference only

{ai_pipeline_core-0.1.7 → ai_pipeline_core-0.1.10}/README.md

@@ -109,7 +109,7 @@ async def process_document(doc: Document):
     return response.parsed
 ```
 
-### Enhanced Pipeline Decorators (New in v0.1.7)
+### Enhanced Pipeline Decorators
 ```python
 from ai_pipeline_core import pipeline_flow, pipeline_task
 from ai_pipeline_core.flow import FlowOptions
@@ -140,7 +140,7 @@ async def my_pipeline(
     return DocumentList(results)
 ```
 
-### Simple Runner Utility (New in v0.1.7)
+### Simple Runner Utility
 ```python
 from ai_pipeline_core.simple_runner import run_cli, run_pipeline
 from ai_pipeline_core.flow import FlowOptions
@@ -164,7 +164,7 @@ async def main():
     )
 ```
 
-### Clean Prefect Decorators (New in v0.1.7)
+### Clean Prefect Decorators
 ```python
 # Import clean Prefect decorators without tracing
 from ai_pipeline_core.prefect import flow, task
@@ -172,12 +172,12 @@ from ai_pipeline_core.prefect import flow, task
 # Or use pipeline decorators with tracing
 from ai_pipeline_core import pipeline_flow, pipeline_task
 
-@task  # Clean Prefect task
+@task  # Clean Prefect task (supports both sync and async)
 def compute(x: int) -> int:
     return x * 2
 
-@pipeline_task(trace_level="always")  # With tracing
-def compute_traced(x: int) -> int:
+@pipeline_task(trace_level="always")  # With tracing (async only)
+async def compute_traced(x: int) -> int:
     return x * 2
 ```
 
@@ -204,12 +204,12 @@ docs = DocumentList([doc1, doc2])
 Managed AI interactions with built-in retry logic, cost tracking, and structured outputs.
 
 **Supported Models** (via LiteLLM proxy):
-- OpenAI: GPT-4, GPT-5 series
-- Anthropic: Claude 3 series
-- Google: Gemini 2.5 series
-- xAI: Grok models
-- Perplexity: Sonar models (with search capabilities)
-- And many more through LiteLLM compatibility
+- OpenAI: gpt-5
+- Anthropic: claude-4
+- Google: gemini-2.5
+- xAI: grok-3, grok-4
+- Perplexity: sonar-pro-search
+- And many more through LiteLLM compatibility. Every model from openrouter should work.
 
 ```python
 from ai_pipeline_core.llm import generate_structured, AIMessages, ModelOptions
@@ -286,13 +286,13 @@ ai_pipeline_core/
 │   └── model_options.py # Configuration models
 ├── flow/              # Prefect flow utilities
 │   ├── config.py      # Type-safe flow configuration
-│   └── options.py     # FlowOptions base class (v0.1.7)
-├── simple_runner/     # Pipeline execution utilities (v0.1.7)
+│   └── options.py     # FlowOptions base class
+├── simple_runner/     # Pipeline execution utilities
 │   ├── cli.py         # CLI interface
 │   └── simple_runner.py # Core runner logic
 ├── logging/           # Structured logging
-├── pipeline.py        # Enhanced decorators (v0.1.7)
-├── prefect.py         # Clean Prefect exports (v0.1.7)
+├── pipeline.py        # Enhanced decorators
+├── prefect.py         # Clean Prefect exports
 ├── tracing.py         # Observability decorators
 └── settings.py        # Centralized configuration
 ```
@@ -303,6 +303,7 @@ ai_pipeline_core/
 ```bash
 make test           # Run all tests
 make test-cov      # Run with coverage report
+make test-showcase # Test the showcase.py CLI example
 pytest tests/test_documents.py::TestDocument::test_creation  # Single test
 ```
 
@@ -439,6 +440,22 @@ For learning purposes, see [CLAUDE.md](CLAUDE.md) for our comprehensive coding s
 
 - [CLAUDE.md](CLAUDE.md) - Detailed coding standards and architecture guide
 
+## Examples
+
+### In This Repository
+- [showcase.py](examples/showcase.py) - Complete example demonstrating all core features including the CLI runner
+  ```bash
+  # Run the showcase example with CLI
+  python examples/showcase.py ./output --temperature 0.7 --batch-size 5
+
+  # Show help
+  python examples/showcase.py --help
+  ```
+- [showcase.jinja2](examples/showcase.jinja2) - Example Jinja2 prompt template
+
+### Real-World Application
+- [AI Documentation Writer](https://github.com/bbarwik/ai-documentation-writer) - Production-ready example showing how to build sophisticated AI pipelines for automated documentation generation. See [examples/ai-documentation-writer.md](examples/ai-documentation-writer.md) for a detailed overview.
+
 ### dependencies_docs/ Directory
 > [!NOTE]
 > The `dependencies_docs/` directory contains guides for AI assistants (like Claude Code) on how to interact with the project's external dependencies and tooling, NOT user documentation for ai-pipeline-core itself. These files are excluded from repository listings to avoid confusion.
@@ -469,29 +486,9 @@ Built with:
 - [LiteLLM](https://litellm.ai/) - LLM proxy
 - [Pydantic](https://pydantic-docs.helpmanual.io/) - Data validation
 
-## What's New in v0.1.7
-
-### Major Additions
-- **Enhanced Pipeline Decorators**: New `pipeline_flow` and `pipeline_task` decorators combining Prefect functionality with automatic LMNR tracing
-- **FlowOptions Base Class**: Extensible configuration system for flows with type-safe inheritance
-- **Simple Runner Module**: CLI and programmatic utilities for easy pipeline execution
-- **Clean Prefect Exports**: Separate imports for Prefect decorators with and without tracing
-- **Expanded Exports**: All major components now accessible from top-level package import
-
-### API Improvements
-- Better type inference for document flows with custom options
-- Support for custom FlowOptions inheritance in pipeline flows
-- Improved error messages for invalid flow signatures
-- Enhanced document utility functions (`canonical_name_key`, `sanitize_url`)
-
-### Developer Experience
-- Simplified imports - most components available from `ai_pipeline_core` directly
-- Better separation of concerns between clean Prefect and traced pipeline decorators
-- More intuitive flow configuration with `FlowOptions` inheritance
-
 ## Stability Notice
 
-**Current Version**: 0.1.7
+**Current Version**: 0.1.10
 **Status**: Internal Preview
 **API Stability**: Unstable - Breaking changes expected
 **Recommended Use**: Learning and reference only

{ai_pipeline_core-0.1.7 → ai_pipeline_core-0.1.10}/ai_pipeline_core/__init__.py

@@ -6,6 +6,7 @@ from .documents import (
     DocumentList,
     FlowDocument,
     TaskDocument,
+    TemporaryDocument,
     canonical_name_key,
     sanitize_url,
 )
@@ -27,12 +28,12 @@ from .logging import (
 )
 from .logging import get_pipeline_logger as get_logger
 from .pipeline import pipeline_flow, pipeline_task
-from .prefect import flow, task
+from .prefect import disable_run_logger, prefect_test_harness
 from .prompt_manager import PromptManager
 from .settings import settings
 from .tracing import TraceInfo, TraceLevel, trace
 
-__version__ = "0.1.7"
+__version__ = "0.1.10"
 
 __all__ = [
     # Config/Settings
@@ -49,17 +50,18 @@ __all__ = [
     "DocumentList",
     "FlowDocument",
     "TaskDocument",
+    "TemporaryDocument",
     "canonical_name_key",
     "sanitize_url",
     # Flow/Task
     "FlowConfig",
     "FlowOptions",
-    # Prefect decorators (clean, no tracing)
-    "task",
-    "flow",
     # Pipeline decorators (with tracing)
     "pipeline_task",
     "pipeline_flow",
+    # Prefect decorators (clean, no tracing)
+    "prefect_test_harness",
+    "disable_run_logger",
     # LLM
     "llm",
     "ModelName",

{ai_pipeline_core-0.1.7 → ai_pipeline_core-0.1.10}/ai_pipeline_core/documents/__init__.py

@@ -2,6 +2,7 @@ from .document import Document
 from .document_list import DocumentList
 from .flow_document import FlowDocument
 from .task_document import TaskDocument
+from .temporary_document import TemporaryDocument
 from .utils import canonical_name_key, sanitize_url
 
 __all__ = [
@@ -9,6 +10,7 @@ __all__ = [
     "DocumentList",
     "FlowDocument",
     "TaskDocument",
+    "TemporaryDocument",
     "canonical_name_key",
     "sanitize_url",
 ]

{ai_pipeline_core-0.1.7 → ai_pipeline_core-0.1.10}/ai_pipeline_core/documents/document.py

@@ -6,7 +6,19 @@ from abc import ABC, abstractmethod
 from base64 import b32encode
 from enum import StrEnum
 from functools import cached_property
-from typing import Any, ClassVar, Literal, Self, TypeVar
+from io import BytesIO
+from typing import (
+    Any,
+    ClassVar,
+    Literal,
+    Self,
+    TypeVar,
+    cast,
+    final,
+    get_args,
+    get_origin,
+    overload,
+)
 
 from pydantic import BaseModel, ConfigDict, field_serializer, field_validator
 from ruamel.yaml import YAML
@@ -23,64 +35,107 @@ from .mime_type import (
 )
 
 TModel = TypeVar("TModel", bound=BaseModel)
+ContentInput = bytes | str | BaseModel | list[str] | Any
 
 
 class Document(BaseModel, ABC):
-    """Abstract base class for all documents"""
+    """Abstract base class for all documents.
+
+    Warning: Document subclasses should NOT start with 'Test' prefix as this
+    causes conflicts with pytest test discovery. Classes with 'Test' prefix
+    will be rejected at definition time.
+    """
 
     MAX_CONTENT_SIZE: ClassVar[int] = 25 * 1024 * 1024  # 25MB default
     DESCRIPTION_EXTENSION: ClassVar[str] = ".description.md"
     MARKDOWN_LIST_SEPARATOR: ClassVar[str] = "\n\n---\n\n"
 
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        """Validate subclass names to prevent pytest conflicts."""
+        super().__init_subclass__(**kwargs)
+        if cls.__name__.startswith("Test"):
+            raise TypeError(
+                f"Document subclass '{cls.__name__}' cannot start with 'Test' prefix. "
+                "This causes conflicts with pytest test discovery. "
+                "Please use a different name (e.g., 'SampleDocument', 'ExampleDocument')."
+            )
+        if hasattr(cls, "FILES"):
+            files = getattr(cls, "FILES")
+            if not issubclass(files, StrEnum):
+                raise TypeError(
+                    f"Document subclass '{cls.__name__}'.FILES must be an Enum of string values"
+                )
+        # Check that the Document's model_fields only contain the allowed fields
+        # It prevents AI models from adding additional fields to documents
+        allowed = {"name", "description", "content"}
+        current = set(getattr(cls, "model_fields", {}).keys())
+        extras = current - allowed
+        if extras:
+            raise TypeError(
+                f"Document subclass '{cls.__name__}' cannot declare additional fields: "
+                f"{', '.join(sorted(extras))}. Only {', '.join(sorted(allowed))} are allowed."
+            )
+
     def __init__(self, **data: Any) -> None:
         """Prevent direct instantiation of abstract Document class."""
         if type(self) is Document:
             raise TypeError("Cannot instantiate abstract Document class directly")
         super().__init__(**data)
 
-    # Optional enum of allowed file names. Subclasses may set this.
-    # This is used to validate the document name.
-    FILES: ClassVar[type[StrEnum] | None] = None
-
     name: str
     description: str | None = None
     content: bytes
 
     # Pydantic configuration
     model_config = ConfigDict(
-        frozen=True,  # Make documents immutable
+        frozen=True,
         arbitrary_types_allowed=True,
+        extra="forbid",
     )
 
     @abstractmethod
-    def get_base_type(self) -> Literal["flow", "task"]:
+    def get_base_type(self) -> Literal["flow", "task", "temporary"]:
         """Get the type of the document - must be implemented by subclasses"""
         raise NotImplementedError("Subclasses must implement this method")
 
+    @final
     @property
-    def base_type(self) -> Literal["flow", "task"]:
+    def base_type(self) -> Literal["flow", "task", "temporary"]:
         """Alias for document_type for backward compatibility"""
         return self.get_base_type()
 
+    @final
     @property
     def is_flow(self) -> bool:
         """Check if document is a flow document"""
         return self.get_base_type() == "flow"
 
+    @final
     @property
     def is_task(self) -> bool:
         """Check if document is a task document"""
         return self.get_base_type() == "task"
 
+    @final
+    @property
+    def is_temporary(self) -> bool:
+        """Check if document is a temporary document"""
+        return self.get_base_type() == "temporary"
+
+    @final
     @classmethod
     def get_expected_files(cls) -> list[str] | None:
         """
         Return the list of allowed file names for this document class, or None if unrestricted.
         """
-        if cls.FILES is None:
+        if not hasattr(cls, "FILES"):
+            return None
+        files = getattr(cls, "FILES")
+        if not files:
             return None
+        assert issubclass(files, StrEnum)
         try:
-            values = [member.value for member in cls.FILES]
+            values = [member.value for member in files]
         except TypeError:
             raise DocumentNameError(f"{cls.__name__}.FILES must be an Enum of string values")
         if len(values) == 0:
@@ -100,14 +155,10 @@ class Document(BaseModel, ABC):
         Override this method in subclasses for custom conventions (regex, prefixes, etc.).
         Raise DocumentNameError when invalid.
         """
-        if cls.FILES is None:
+        allowed = cls.get_expected_files()
+        if not allowed:
             return
 
-        try:
-            allowed = {str(member.value) for member in cls.FILES}  # type: ignore[arg-type]
-        except TypeError:
-            raise DocumentNameError(f"{cls.__name__}.FILES must be an Enum of string values")
-
         if len(allowed) > 0 and name not in allowed:
             allowed_str = ", ".join(sorted(allowed))
             raise DocumentNameError(f"Invalid filename '{name}'. Allowed names: {allowed_str}")
@@ -151,16 +202,19 @@ class Document(BaseModel, ABC):
             # Fall back to base64 for binary content
             return base64.b64encode(v).decode("ascii")
 
+    @final
     @property
     def id(self) -> str:
         """Return the first 6 characters of the SHA256 hash of the content, encoded in base32"""
         return self.sha256[:6]
 
+    @final
     @cached_property
     def sha256(self) -> str:
         """Full SHA256 hash of content, encoded in base32"""
         return b32encode(hashlib.sha256(self.content).digest()).decode("ascii").upper()
 
+    @final
     @property
     def size(self) -> int:
         """Size of content in bytes"""
@@ -210,23 +264,61 @@ class Document(BaseModel, ABC):
         """Parse document as JSON"""
         return json.loads(self.as_text())
 
-    def as_pydantic_model(self, model_type: type[TModel]) -> TModel:
+    @overload
+    def as_pydantic_model(self, model_type: type[TModel]) -> TModel: ...
+
+    @overload
+    def as_pydantic_model(self, model_type: type[list[TModel]]) -> list[TModel]: ...
+
+    def as_pydantic_model(
+        self, model_type: type[TModel] | type[list[TModel]]
+    ) -> TModel | list[TModel]:
         """Parse document as a pydantic model and return the validated instance"""
         data = self.as_yaml() if is_yaml_mime_type(self.mime_type) else self.as_json()
-        return model_type.model_validate(data)
+
+        if get_origin(model_type) is list:
+            if not isinstance(data, list):
+                raise ValueError(f"Expected list data for {model_type}, got {type(data)}")
+            item_type = get_args(model_type)[0]
+            return [item_type.model_validate(item) for item in data]
+
+        # At this point model_type must be type[TModel], not type[list[TModel]]
+        single_model = cast(type[TModel], model_type)
+        return single_model.model_validate(data)
 
     def as_markdown_list(self) -> list[str]:
         """Parse document as a markdown list"""
         return self.as_text().split(self.MARKDOWN_LIST_SEPARATOR)
 
+    @overload
+    @classmethod
+    def create(cls, name: str, content: ContentInput, /) -> Self: ...
+    @overload
+    @classmethod
+    def create(cls, name: str, *, content: ContentInput) -> Self: ...
+    @overload
+    @classmethod
+    def create(cls, name: str, description: str | None, content: ContentInput, /) -> Self: ...
+    @overload
+    @classmethod
+    def create(cls, name: str, description: str | None, *, content: ContentInput) -> Self: ...
+
     @classmethod
     def create(
         cls,
         name: str,
-        description: str | None,
-        content: bytes | str | BaseModel | list[str] | Any,
+        description: ContentInput = None,
+        content: ContentInput = None,
     ) -> Self:
         """Create a document from a name, description, and content"""
+        if content is None:
+            if description is None:
+                raise ValueError(f"Unsupported content type: {type(content)} for {name}")
+            content = description
+            description = None
+        else:
+            assert description is None or isinstance(description, str)
+
         is_yaml_extension = name.endswith(".yaml") or name.endswith(".yml")
         is_json_extension = name.endswith(".json")
         is_markdown_extension = name.endswith(".md")
@@ -237,6 +329,14 @@ class Document(BaseModel, ABC):
             content = content.encode("utf-8")
         elif is_str_list and is_markdown_extension:
             return cls.create_as_markdown_list(name, description, content)  # type: ignore[arg-type]
+        elif isinstance(content, list) and all(isinstance(item, BaseModel) for item in content):
+            # Handle list[BaseModel] for JSON/YAML files
+            if is_yaml_extension:
+                return cls.create_as_yaml(name, description, content)
+            elif is_json_extension:
+                return cls.create_as_json(name, description, content)
+            else:
+                raise ValueError(f"list[BaseModel] requires .json or .yaml extension, got {name}")
         elif is_yaml_extension:
             return cls.create_as_yaml(name, description, content)
         elif is_json_extension:
@@ -246,6 +346,7 @@ class Document(BaseModel, ABC):
 
         return cls(name=name, description=description, content=content)
 
+    @final
     @classmethod
     def create_as_markdown_list(cls, name: str, description: str | None, items: list[str]) -> Self:
         """Create a document from a name, description, and list of strings"""
@@ -258,15 +359,19 @@ class Document(BaseModel, ABC):
         content = Document.MARKDOWN_LIST_SEPARATOR.join(cleaned_items)
         return cls.create(name, description, content)
 
+    @final
     @classmethod
     def create_as_json(cls, name: str, description: str | None, data: Any) -> Self:
         """Create a document from a name, description, and JSON data"""
         assert name.endswith(".json"), f"Document name must end with .json: {name}"
         if isinstance(data, BaseModel):
             data = data.model_dump(mode="json")
+        elif isinstance(data, list) and all(isinstance(item, BaseModel) for item in data):
+            data = [item.model_dump(mode="json") for item in data]
         content = json.dumps(data, indent=2).encode("utf-8")
         return cls.create(name, description, content)
 
+    @final
     @classmethod
     def create_as_yaml(cls, name: str, description: str | None, data: Any) -> Self:
         """Create a document from a name, description, and YAML data"""
@@ -274,16 +379,18 @@ class Document(BaseModel, ABC):
             f"Document name must end with .yaml or .yml: {name}"
         )
         if isinstance(data, BaseModel):
-            data = data.model_dump()
+            data = data.model_dump(mode="json")
+        elif isinstance(data, list) and all(isinstance(item, BaseModel) for item in data):
+            data = [item.model_dump(mode="json") for item in data]
         yaml = YAML()
         yaml.indent(mapping=2, sequence=4, offset=2)
-        from io import BytesIO
 
         stream = BytesIO()
         yaml.dump(data, stream)
         content = stream.getvalue()
         return cls.create(name, description, content)
 
+    @final
     def serialize_model(self) -> dict[str, Any]:
         """Serialize document to a dictionary with proper encoding."""
         result = {
@@ -312,6 +419,7 @@ class Document(BaseModel, ABC):
 
         return result
 
+    @final
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> Self:
         """Deserialize document from dictionary."""

ai_pipeline_core-0.1.10/ai_pipeline_core/documents/temporary_document.py

@@ -0,0 +1,16 @@
+"""Task-specific document base class."""
+
+from typing import Literal, final
+
+from .document import Document
+
+
+@final
+class TemporaryDocument(Document):
+    """
+    Temporary document is a document that is not persisted in any case.
+    """
+
+    def get_base_type(self) -> Literal["temporary"]:
+        """Get the document type."""
+        return "temporary"