ai-pipeline-core 0.2.9__py3-none-any.whl → 0.3.0__py3-none-any.whl

{ai_pipeline_core-0.2.9.dist-info → ai_pipeline_core-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-pipeline-core
-Version: 0.2.9
+Version: 0.3.0
 Summary: Core utilities for AI-powered processing pipelines using prefect
 Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
 Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core
@@ -63,7 +63,7 @@ AI Pipeline Core is a production-ready framework that combines document processi
 - **Structured Output**: Type-safe generation with Pydantic model validation
 - **Workflow Orchestration**: Prefect-based flows and tasks with automatic retries
 - **Observability**: Built-in distributed tracing via Laminar (LMNR) with cost tracking for debugging and monitoring
-- **Local Development**: Simple runner for testing pipelines without infrastructure
+- **Deployment**: Unified pipeline execution for local, CLI, and production environments
 
 ## Installation
 
@@ -177,7 +177,7 @@ doc = MyDocument.create(
 # Parse back to original type
 data = doc.parse(dict)  # Returns {"key": "value"}
 
-# Document provenance tracking (new in v0.1.14)
+# Document provenance tracking
 doc_with_sources = MyDocument.create(
     name="derived.json",
     content={"result": "processed"},
@@ -224,15 +224,15 @@ if doc.is_text:
 # Parse structured data
 data = doc.as_json()  # or as_yaml(), as_pydantic_model()
 
-# Convert between document types (new in v0.2.1)
+# Convert between document types
 task_doc = flow_doc.model_convert(TaskDocument)  # Convert FlowDocument to TaskDocument
 new_doc = doc.model_convert(OtherDocType, content={"new": "data"})  # With content update
 
-# Enhanced filtering (new in v0.1.14)
+# Enhanced filtering
 filtered = documents.filter_by([Doc1, Doc2, Doc3])  # Multiple types
 named = documents.filter_by(["file1.txt", "file2.txt"])  # Multiple names
 
-# Immutable collections (new in v0.2.1)
+# Immutable collections
 frozen_docs = DocumentList(docs, frozen=True)  # Immutable document list
 frozen_msgs = AIMessages(messages, frozen=True)  # Immutable message list
 ```
@@ -268,7 +268,7 @@ r2 = await llm.generate(
     messages="Key points?"   # Different query
 )
 
-# Custom cache TTL (new in v0.1.14)
+# Custom cache TTL
 response = await llm.generate(
     model="gpt-5",
     context=static_context,
@@ -317,12 +317,12 @@ from ai_pipeline_core import pipeline_flow, pipeline_task, set_trace_cost
 @pipeline_task  # Automatic retry, tracing, and monitoring
 async def process_chunk(data: str) -> str:
     result = await transform(data)
-    set_trace_cost(0.05)  # Track costs (new in v0.1.14)
+    set_trace_cost(0.05)  # Track costs
     return result
 
 @pipeline_flow(
     config=MyFlowConfig,
-    trace_trim_documents=True  # Trim large documents in traces (new in v0.2.1)
+    trace_trim_documents=True  # Trim large documents in traces
 )
 async def main_flow(
     project_name: str,
@@ -458,18 +458,21 @@ For AI assistants:
 ```
 ai-pipeline-core/
 ├── ai_pipeline_core/
-│   ├── documents/      # Document abstraction system
-│   ├── flow/           # Flow configuration and options
-│   ├── llm/            # LLM client and response handling
-│   ├── logging/        # Logging infrastructure
-│   ├── tracing.py      # Distributed tracing
-│   ├── pipeline.py     # Pipeline decorators
+│   ├── deployment/      # Pipeline deployment and execution
+│   ├── documents/       # Document abstraction system
+│   ├── flow/            # Flow configuration and options
+│   ├── llm/             # LLM client and response handling
+│   ├── logging/         # Logging infrastructure
+│   ├── prompt_builder/  # Document-aware prompt construction
+│   ├── pipeline.py      # Pipeline decorators
+│   ├── progress.py      # Intra-flow progress tracking
 │   ├── prompt_manager.py # Jinja2 template management
-│   └── settings.py     # Configuration management
-├── tests/              # Comprehensive test suite
-├── examples/           # Usage examples
-├── API.md             # Complete API reference
-└── pyproject.toml     # Project configuration
+│   ├── settings.py      # Configuration management
+│   └── tracing.py       # Distributed tracing
+├── tests/               # Comprehensive test suite
+├── examples/            # Usage examples
+├── API.md               # Complete API reference
+└── pyproject.toml       # Project configuration
 ```
 
 ## Contributing

ai_pipeline_core-0.3.0.dist-info/RECORD

@@ -0,0 +1,49 @@
+ai_pipeline_core/__init__.py,sha256=q8sas8GxIyZf4h0RPqzv06ppo8hy0gl8-GjDEVh71XQ,6087
+ai_pipeline_core/exceptions.py,sha256=vx-XLTw2fJSPs-vwtXVYtqoQUcOc0JeI7UmHqRqQYWU,1569
+ai_pipeline_core/pipeline.py,sha256=t9qH-V6umpKY5MhGuXFgUGfdzGyxzVlS0n9RoKLfnug,28704
+ai_pipeline_core/prefect.py,sha256=91ZgLJHsDsRUW77CpNmkKxYs3RCJuucPM3pjKmNBeDg,2199
+ai_pipeline_core/progress.py,sha256=Ppxk4OOm84Y0x3t-Y3CmHsL4PovQLNUxXMu24zRCD-Q,3621
+ai_pipeline_core/prompt_manager.py,sha256=FAtb1yK7bGuAeuIJ523LOX9bd7TrcHG-TqZ7Lz4RJC0,12087
+ai_pipeline_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ai_pipeline_core/settings.py,sha256=IMrFaX0i-WIlaOA5O53ipNSta6KQVSFHc1aJXmS3nSo,5078
+ai_pipeline_core/tracing.py,sha256=YksAxjSJ7PgmrEQ5ZxfpEACZfD9G6KuV7b0LoGM-ogo,31538
+ai_pipeline_core/deployment/__init__.py,sha256=FN2HVoM80x2GJuNs7o4DnccB8HWWibgM1pJesB942CM,1259
+ai_pipeline_core/deployment/base.py,sha256=JYf8XLFR73c0H24dr6atK7yUcoE0vLxbYZ8EkQpEwN4,24791
+ai_pipeline_core/deployment/contract.py,sha256=0DKt5eqNE-grcITwMNq9CuBdo5WxdopEjDeQFzFZxhU,2225
+ai_pipeline_core/deployment/helpers.py,sha256=3nRuCyABkUEDZiL0q9u19XHpjA4527B6rsxQNOGTohw,3460
+ai_pipeline_core/documents/__init__.py,sha256=WHStvGZiSyybOcMTYxSV24U6MA3Am_0_Az5p-DuMFrk,738
+ai_pipeline_core/documents/document.py,sha256=hdTh36KGEcrDollTnQmTI66DJIqYfe4X42Y0q7Cm4fY,68153
+ai_pipeline_core/documents/document_list.py,sha256=Y_NCjfM_CjkIwHRD2iyGgYBuIykN8lT2IIH_uWOiGis,16254
+ai_pipeline_core/documents/flow_document.py,sha256=QK6RxNQu449IRAosOHSk3G_5yIq5I7yLBOSQPCd3m64,4141
+ai_pipeline_core/documents/mime_type.py,sha256=JFEOq4HwlIW2snobyNfWwySdT7urZSWkobiRMVs2fSE,7959
+ai_pipeline_core/documents/task_document.py,sha256=uASmAaxNkYtuqQrBM57vutFT9DXNTbqv0wbwwF55E3I,4300
+ai_pipeline_core/documents/temporary_document.py,sha256=jaz2ZHC5CmSbVbkXdI7pOB5DGEuhH16C0Yutv-lS_UI,2708
+ai_pipeline_core/documents/utils.py,sha256=ZyJNjFN7ihWno0K7dJZed7twYmmPLA0z40UzFw1A3A8,5465
+ai_pipeline_core/flow/__init__.py,sha256=2BfWYMOPYW5teGzwo-qzpn_bom1lxxry0bPsjVgcsCk,188
+ai_pipeline_core/flow/config.py,sha256=a9FALpgrFsdz-D7HU3diVeUzbaBvLwI8hsPviuj001s,19389
+ai_pipeline_core/flow/options.py,sha256=mhToZ9u18WCMBEYJL1MYKzh8fH9lSsAUqQtU8tNnD18,2304
+ai_pipeline_core/llm/__init__.py,sha256=3B_vtEzxrzidP1qOUNQ4RxlUmxZ2MBKQcUhQiTybM9g,661
+ai_pipeline_core/llm/ai_messages.py,sha256=Onin3UPdbJQNl3WfY3-_jE5KRmF-ciXsa5K6UPOiy5s,14410
+ai_pipeline_core/llm/client.py,sha256=4nCoJOdTtye1novQiUW3AFPjZBF_TfsD7J09sl9kbd4,24973
+ai_pipeline_core/llm/model_options.py,sha256=uRNIHfVeh2sgt1mZBiOUx6hPQ6GKjB8b7TytZJ6afKg,11768
+ai_pipeline_core/llm/model_response.py,sha256=-fKJcblDP_Z6NV9CGp4bm_hitb0Z0jyy0ZndCQUpRkQ,13493
+ai_pipeline_core/llm/model_types.py,sha256=MukKpS7vWeWAfHhKDxRlQFm5jeBloT_o6amO4qUzjWo,2761
+ai_pipeline_core/logging/__init__.py,sha256=Nz6-ghAoENsgNmLD2ma9TW9M0U2_QfxuQ5DDW6Vt6M0,651
+ai_pipeline_core/logging/logging.yml,sha256=YTW48keO_K5bkkb-KXGM7ZuaYKiquLsjsURei8Ql0V4,1353
+ai_pipeline_core/logging/logging_config.py,sha256=pV2x6GgMPXrzPH27sicCSXfw56beio4C2JKCJ3NsXrg,6207
+ai_pipeline_core/logging/logging_mixin.py,sha256=OTye2pbUbG5oYZkI06TNkGCEa4y0ldePz5IAfdmNUPU,8090
+ai_pipeline_core/prompt_builder/__init__.py,sha256=-v0SKZlir07xRzxXwv75VP66aINRUiKH0VUgB-PCDmI,195
+ai_pipeline_core/prompt_builder/documents_prompt.jinja2,sha256=LPql5AaFhFWtDfhnBWvi-bWbz5vdgsWqKGzcqxWfLIM,1075
+ai_pipeline_core/prompt_builder/global_cache.py,sha256=9_9zoF6-sr3KBMxF5QLD3vxqXg9B2tT8o9ViplzUCNg,2811
+ai_pipeline_core/prompt_builder/new_core_documents_prompt.jinja2,sha256=M8uPpwf-uLpsWWJT9DY_DnjrLToGPVnrD-gVhQrQdaQ,229
+ai_pipeline_core/prompt_builder/prompt_builder.py,sha256=OAu3b8stzmFoAvPD7BDwnk8TkAxG8JDe3kAN7EhGTK0,9365
+ai_pipeline_core/prompt_builder/system_prompt.jinja2,sha256=-1jLcfvAG07Zfl-dnYrjfVcAG4PWeeoeWpaKJGY3rKQ,3945
+ai_pipeline_core/storage/__init__.py,sha256=tcIkjJ3zPBLCyetwiJDewBvS2sbRJrDlBh3gEsQm08E,184
+ai_pipeline_core/storage/storage.py,sha256=ClMr419Y-eU2RuOjZYd51dC0stWQk28Vb56PvQaoUwc,20007
+ai_pipeline_core/utils/__init__.py,sha256=TJSmEm1Quf-gKwXrxM96u2IGzVolUyeNNfLMPoLstXI,254
+ai_pipeline_core/utils/deploy.py,sha256=rAtRuwkmGkc-fqvDMXpt08OzLrD7KTDMAmLDC9wYg7Y,13147
+ai_pipeline_core/utils/remote_deployment.py,sha256=U7MNJ1SU1mg3RrJyLqpuN_4pwqm8LSsFZbypJvjGPoo,4630
+ai_pipeline_core-0.3.0.dist-info/METADATA,sha256=qDOFXeCZIsQj85TBq59eadO_yNQQbHraP9ku3CE-xR0,15264
+ai_pipeline_core-0.3.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+ai_pipeline_core-0.3.0.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
+ai_pipeline_core-0.3.0.dist-info/RECORD,,

{ai_pipeline_core-0.2.9.dist-info → ai_pipeline_core-0.3.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

ai_pipeline_core/simple_runner/__init__.py

@@ -1,14 +0,0 @@
-"""Simple pipeline execution for local development.
-
-Utilities for running AI pipelines locally without full Prefect orchestration.
-"""
-
-from .cli import run_cli
-from .simple_runner import FlowSequence, run_pipeline, run_pipelines
-
-__all__ = [
-    "run_cli",
-    "run_pipeline",
-    "run_pipelines",
-    "FlowSequence",
-]

ai_pipeline_core/simple_runner/cli.py

@@ -1,254 +0,0 @@
-"""Command-line interface for simple pipeline execution."""
-
-import asyncio
-import os
-import sys
-from contextlib import ExitStack
-from pathlib import Path
-from typing import Callable, Type, TypeVar, cast
-
-from lmnr import Laminar
-from pydantic import ValidationError
-from pydantic_settings import CliPositionalArg, SettingsConfigDict
-
-from ai_pipeline_core.documents import DocumentList
-from ai_pipeline_core.flow.options import FlowOptions
-from ai_pipeline_core.logging import get_pipeline_logger, setup_logging
-from ai_pipeline_core.prefect import disable_run_logger, prefect_test_harness
-from ai_pipeline_core.settings import settings
-
-from .simple_runner import FlowSequence, run_pipelines
-
-logger = get_pipeline_logger(__name__)
-
-TOptions = TypeVar("TOptions", bound=FlowOptions)
-"""Type variable for FlowOptions subclasses used in CLI."""
-
-InitializerFunc = Callable[[FlowOptions], tuple[str, DocumentList]] | None
-"""Function type for custom pipeline initialization.
-
-Initializers can create initial documents or setup project state
-before flow execution begins.
-
-Args:
-    FlowOptions: Parsed CLI options
-
-Returns:
-    Tuple of (project_name, initial_documents) or None
-"""
-
-
-def _initialize_environment() -> None:
-    """Initialize logging and observability systems.
-
-    Sets up the pipeline logging configuration and attempts to
-    initialize LMNR (Laminar) for distributed tracing. Failures
-    in LMNR initialization are logged but don't stop execution.
-
-    Side effects:
-        - Configures Python logging system
-        - Initializes Laminar SDK if API key is available
-        - Logs initialization status
-
-    Note:
-        Called automatically by run_cli before parsing arguments.
-    """
-    setup_logging()
-    try:
-        Laminar.initialize()
-        logger.info("LMNR tracing initialized.")
-    except Exception as e:
-        logger.warning(f"Failed to initialize LMNR tracing: {e}")
-
-
-def _running_under_pytest() -> bool:
-    """Check if code is running under pytest.
-
-    Detects pytest execution context to determine whether test
-    fixtures will provide necessary contexts (like Prefect test
-    harness). This prevents duplicate context setup.
-
-    Returns:
-        True if running under pytest, False otherwise.
-
-    Detection methods:
-        - PYTEST_CURRENT_TEST environment variable (set by pytest)
-        - 'pytest' module in sys.modules (imported by test runner)
-
-    Note:
-        Used to avoid setting up test harness when pytest fixtures
-        already provide it.
-    """
-    return "PYTEST_CURRENT_TEST" in os.environ or "pytest" in sys.modules
-
-
-def run_cli(
-    *,
-    flows: FlowSequence,
-    options_cls: Type[TOptions],
-    initializer: InitializerFunc = None,
-    trace_name: str | None = None,
-) -> None:
-    """Execute pipeline flows from command-line arguments.
-
-    Environment setup:
-        - Initializes logging system
-        - Sets up LMNR tracing (if API key configured)
-        - Creates Prefect test harness (if no API key and not in pytest)
-        - Manages context stack for proper cleanup
-
-    Raises:
-        ValueError: If project name is empty after initialization.
-
-    Example:
-        >>> # In __main__.py
-        >>> from ai_pipeline_core import simple_runner
-        >>> from .flows import AnalysisFlow, SummaryFlow
-        >>> from .config import AnalysisOptions
-        >>>
-        >>> if __name__ == "__main__":
-        ...     simple_runner.run_cli(
-        ...         flows=[AnalysisFlow, SummaryFlow],
-        ...         options_cls=AnalysisOptions,
-        ...         trace_name="document-analysis"
-        ...     )
-
-        Command line:
-        $ python -m my_module ./output --temperature 0.5 --model gpt-5
-        $ python -m my_module ./output --start 2  # Skip first flow
-
-    Note:
-        - Field names are converted to kebab-case for CLI (max_tokens → --max-tokens)
-        - Boolean fields become flags (--verbose/--no-verbose)
-        - Field descriptions from Pydantic become help text
-        - Type hints are enforced during parsing
-        - Validation errors show helpful messages with field names
-        - Includes hints for common error types (numbers, ranges)
-        - Exits with status 1 on error
-        - Shows --help when no arguments provided
-    """
-    # Check if no arguments provided before initialization
-    if len(sys.argv) == 1:
-        # Add --help to show usage when run without arguments
-        sys.argv.append("--help")
-
-    _initialize_environment()
-
-    class _RunnerOptions(  # type: ignore[reportRedeclaration]
-        options_cls,
-        cli_parse_args=True,
-        cli_kebab_case=True,
-        cli_exit_on_error=True,  # Let it exit normally on error
-        cli_prog_name="ai-pipeline",
-        cli_use_class_docs_for_groups=True,
-    ):
-        """Internal options class combining user options with CLI arguments.
-
-        Dynamically created class that inherits from user's options_cls
-        and adds standard CLI arguments for pipeline execution.
-        """
-
-        working_directory: CliPositionalArg[Path]
-        project_name: str | None = None
-        start: int = 1
-        end: int | None = None
-
-        model_config = SettingsConfigDict(frozen=True, extra="ignore")
-
-    try:
-        opts = cast(FlowOptions, _RunnerOptions())  # type: ignore[reportCallIssue]
-    except ValidationError as e:
-        print("\nError: Invalid command line arguments\n", file=sys.stderr)
-        for error in e.errors():
-            field = " -> ".join(str(loc) for loc in error["loc"])
-            msg = error["msg"]
-            value = error.get("input", "")
-
-            # Format the field name nicely (convert from snake_case to kebab-case for CLI)
-            cli_field = field.replace("_", "-")
-
-            print(f"  --{cli_field}: {msg}", file=sys.stderr)
-            if value:
-                print(f"    Provided value: '{value}'", file=sys.stderr)
-
-            # Add helpful hints for common errors
-            if error["type"] == "float_parsing":
-                print("    Hint: Please provide a valid number (e.g., 0.7)", file=sys.stderr)
-            elif error["type"] == "int_parsing":
-                print("    Hint: Please provide a valid integer (e.g., 10)", file=sys.stderr)
-            elif error["type"] == "literal_error":
-                ctx = error.get("ctx", {})
-                expected = ctx.get("expected", "valid options")
-                print(f"    Hint: Valid options are: {expected}", file=sys.stderr)
-            elif error["type"] in [
-                "less_than_equal",
-                "greater_than_equal",
-                "less_than",
-                "greater_than",
-            ]:
-                ctx = error.get("ctx", {})
-                if "le" in ctx:
-                    print(f"    Hint: Value must be ≤ {ctx['le']}", file=sys.stderr)
-                elif "ge" in ctx:
-                    print(f"    Hint: Value must be ≥ {ctx['ge']}", file=sys.stderr)
-                elif "lt" in ctx:
-                    print(f"    Hint: Value must be < {ctx['lt']}", file=sys.stderr)
-                elif "gt" in ctx:
-                    print(f"    Hint: Value must be > {ctx['gt']}", file=sys.stderr)
-
-        print("\nRun with --help to see all available options\n", file=sys.stderr)
-        sys.exit(1)
-
-    wd: Path = cast(Path, getattr(opts, "working_directory"))
-    wd.mkdir(parents=True, exist_ok=True)
-
-    # Get project name from options or use directory basename
-    project_name = getattr(opts, "project_name", None)
-    if not project_name:  # None or empty string
-        project_name = wd.name
-
-    # Ensure project_name is not empty
-    if not project_name:
-        raise ValueError("Project name cannot be empty")
-
-    # Use initializer if provided, otherwise use defaults
-    initial_documents = DocumentList([])
-    if initializer:
-        init_result = initializer(opts)
-        # Always expect tuple format from initializer
-        _, initial_documents = init_result  # Ignore project name from initializer
-
-        # Save initial documents if starting from first step
-        if getattr(opts, "start", 1) == 1 and initial_documents and flows:
-            # Get config from the first flow
-            first_flow_config = getattr(flows[0], "config", None)
-            if first_flow_config:
-                asyncio.run(
-                    first_flow_config.save_documents(
-                        str(wd), initial_documents, validate_output_type=False
-                    )
-                )
-
-    # Setup context stack with optional test harness and tracing
-    with ExitStack() as stack:
-        if trace_name:
-            stack.enter_context(
-                Laminar.start_as_current_span(
-                    name=f"{trace_name}-{project_name}", input=[opts.model_dump_json()]
-                )
-            )
-
-        if not settings.prefect_api_key and not _running_under_pytest():
-            stack.enter_context(prefect_test_harness())
-            stack.enter_context(disable_run_logger())
-
-        asyncio.run(
-            run_pipelines(
-                project_name=project_name,
-                output_dir=wd,
-                flows=flows,
-                flow_options=opts,
-                start_step=getattr(opts, "start", 1),
-                end_step=getattr(opts, "end", None),
-            )
-        )

ai_pipeline_core/simple_runner/simple_runner.py

@@ -1,247 +0,0 @@
-"""Simple pipeline runner for local flow execution.
-
-This module provides the core functionality for running AI pipeline flows
-locally without full Prefect orchestration. It handles document I/O,
-flow sequencing, and error management.
-
-Key components:
-    - Document I/O from/to filesystem directories via FlowConfig
-    - Single and multi-flow execution
-    - Automatic document validation and passing between flows
-    - Step-based execution control (start/end steps)
-
-Directory structure:
-    working_dir/
-    ├── inputdocument/       # Documents of type InputDocument (lowercase)
-    │   ├── file1.txt
-    │   └── file1.txt.description.md   # Optional description
-    └── outputdocument/      # Documents of type OutputDocument (lowercase)
-        └── result.json
-
-Example:
-    >>> from ai_pipeline_core import simple_runner
-    >>>
-    >>> # Run single flow
-    >>> results = await simple_runner.run_pipeline(
-    ...     flow_func=MyFlow,
-    ...     config=MyConfig,
-    ...     project_name="test",
-    ...     output_dir=Path("./output"),
-    ...     flow_options=options
-    ... )
-
-Note:
-    Document directories are organized by document type names (lowercase)
-    for consistent structure and easy access.
-"""
-
-from pathlib import Path
-from typing import Any, Callable, Sequence
-
-from ai_pipeline_core.documents import DocumentList
-from ai_pipeline_core.flow.options import FlowOptions
-from ai_pipeline_core.logging import get_pipeline_logger
-
-logger = get_pipeline_logger(__name__)
-
-FlowSequence = Sequence[Callable[..., Any]]
-"""Type alias for a sequence of flow functions."""
-
-
-async def run_pipeline(
-    flow_func: Callable[..., Any],
-    project_name: str,
-    output_dir: Path,
-    flow_options: FlowOptions,
-    flow_name: str | None = None,
-) -> DocumentList:
-    """Execute a single pipeline flow with document I/O.
-
-    Runs a flow function with automatic document loading, validation,
-    and saving. The flow receives input documents from the filesystem
-    and saves its output for subsequent flows.
-
-    The execution proceeds through these steps:
-    1. Load input documents from output_dir subdirectories
-    2. Validate input documents against flow's config requirements
-    3. Execute flow function with documents and options
-    4. Validate output documents match config.OUTPUT_DOCUMENT_TYPE
-    5. Save output documents to output_dir subdirectories
-
-    Args:
-        flow_func: Async flow function decorated with @pipeline_flow.
-                  Must accept (project_name, documents, flow_options).
-                  The flow must have a config attribute set by @pipeline_flow.
-
-        project_name: Name of the project/pipeline for logging and tracking.
-
-        output_dir: Directory for loading input and saving output documents.
-                   Document subdirectories are created as needed.
-
-        flow_options: Configuration options passed to the flow function.
-                     Can be FlowOptions or any subclass.
-
-        flow_name: Optional display name for logging. If None, uses
-                  flow_func.name or flow_func.__name__.
-
-    Returns:
-        DocumentList containing the flow's output documents.
-
-    Raises:
-        RuntimeError: If required input documents are missing or if
-                     flow doesn't have a config attribute.
-
-    Example:
-        >>> from my_flows import AnalysisFlow
-        >>>
-        >>> results = await run_pipeline(
-        ...     flow_func=AnalysisFlow,
-        ...     project_name="analysis_001",
-        ...     output_dir=Path("./results"),
-        ...     flow_options=FlowOptions(temperature=0.7)
-        ... )
-        >>> print(f"Generated {len(results)} documents")
-
-    Note:
-        - Flow must be async (decorated with @pipeline_flow with config)
-        - Input documents are loaded based on flow's config.INPUT_DOCUMENT_TYPES
-        - Output is validated against config.OUTPUT_DOCUMENT_TYPE
-        - All I/O is logged for debugging
-    """
-    if flow_name is None:
-        # For Prefect Flow objects, use their name attribute
-        # For regular functions, fall back to __name__
-        flow_name = getattr(flow_func, "name", None) or getattr(flow_func, "__name__", "flow")
-
-    logger.info(f"Running Flow: {flow_name}")
-
-    # Get config from the flow function (attached by @pipeline_flow decorator)
-    config = getattr(flow_func, "config", None)
-    if config is None:
-        raise RuntimeError(
-            f"Flow {flow_name} does not have a config attribute. "
-            "Ensure it's decorated with @pipeline_flow(config=YourConfig)"
-        )
-
-    # Load input documents using FlowConfig's new async method
-    input_documents = await config.load_documents(str(output_dir))
-
-    if not config.has_input_documents(input_documents):
-        raise RuntimeError(f"Missing input documents for flow {flow_name}")
-
-    result_documents = await flow_func(project_name, input_documents, flow_options)
-
-    config.validate_output_documents(result_documents)
-
-    # Save output documents using FlowConfig's new async method
-    await config.save_documents(str(output_dir), result_documents)
-
-    logger.info(f"Completed Flow: {flow_name}")
-
-    return result_documents
-
-
-async def run_pipelines(
-    project_name: str,
-    output_dir: Path,
-    flows: FlowSequence,
-    flow_options: FlowOptions,
-    start_step: int = 1,
-    end_step: int | None = None,
-) -> None:
-    """Execute multiple pipeline flows in sequence.
-
-    Runs a series of flows where each flow's output becomes the input
-    for the next flow. Supports partial execution with start/end steps
-    for debugging and resuming failed pipelines.
-
-    Execution proceeds by:
-    1. Validating step indices
-    2. For each flow in range [start_step, end_step]:
-       a. Loading input documents from output_dir
-       b. Executing flow with documents
-       c. Saving output documents to output_dir
-       d. Output becomes input for next flow
-    3. Logging progress and any failures
-
-    Steps are 1-based for user convenience. Step 1 is the first flow,
-    Step N is the Nth flow. Use start_step > 1 to skip initial flows
-    and end_step < N to stop early.
-
-    Args:
-        project_name: Name of the overall pipeline/project.
-        output_dir: Directory for document I/O between flows.
-                   Shared by all flows in the sequence.
-        flows: Sequence of flow functions to execute in order.
-              Must all be async functions decorated with @pipeline_flow
-              with a config parameter.
-        flow_options: Options passed to all flows in the sequence.
-                     Individual flows can use different fields.
-        start_step: First flow to execute (1-based index).
-                   Default 1 starts from the beginning.
-        end_step: Last flow to execute (1-based index).
-                 None runs through the last flow.
-
-    Raises:
-        ValueError: If start_step or end_step are out of range.
-        RuntimeError: If any flow doesn't have a config attribute.
-
-    Example:
-        >>> # Run full pipeline
-        >>> await run_pipelines(
-        ...     project_name="analysis",
-        ...     output_dir=Path("./work"),
-        ...     flows=[ExtractFlow, AnalyzeFlow, SummarizeFlow],
-        ...     flow_options=options
-        ... )
-        >>>
-        >>> # Run only steps 2-3 (skip extraction)
-        >>> await run_pipelines(
-        ...     ...,
-        ...     start_step=2,
-        ...     end_step=3
-        ... )
-
-    Note:
-        - Each flow must be decorated with @pipeline_flow(config=...)
-        - Each flow's output must match the next flow's input types
-        - Failed flows stop the entire pipeline
-        - Progress is logged with step numbers for debugging
-        - Documents persist in output_dir between runs
-    """
-    num_steps = len(flows)
-    start_index = start_step - 1
-    end_index = (end_step if end_step is not None else num_steps) - 1
-
-    if (
-        not (0 <= start_index < num_steps)
-        or not (0 <= end_index < num_steps)
-        or start_index > end_index
-    ):
-        raise ValueError("Invalid start/end steps.")
-
-    logger.info(f"Starting pipeline '{project_name}' (Steps {start_step} to {end_index + 1})")
-
-    for i in range(start_index, end_index + 1):
-        flow_func = flows[i]
-        # For Prefect Flow objects, use their name attribute; for functions, use __name__
-        flow_name = getattr(flow_func, "name", None) or getattr(
-            flow_func, "__name__", f"flow_{i + 1}"
-        )
-
-        logger.info(f"--- [Step {i + 1}/{num_steps}] Running Flow: {flow_name} ---")
-
-        try:
-            await run_pipeline(
-                flow_func=flow_func,
-                project_name=project_name,
-                output_dir=output_dir,
-                flow_options=flow_options,
-                flow_name=f"[Step {i + 1}/{num_steps}] {flow_name}",
-            )
-
-        except Exception as e:
-            logger.error(
-                f"--- [Step {i + 1}/{num_steps}] Flow {flow_name} Failed: {e} ---", exc_info=True
-            )
-            raise