ai-pipeline-core 0.2.9__py3-none-any.whl → 0.3.3__py3-none-any.whl

{ai_pipeline_core-0.2.9.dist-info → ai_pipeline_core-0.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-pipeline-core
-Version: 0.2.9
+Version: 0.3.3
 Summary: Core utilities for AI-powered processing pipelines using prefect
 Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
 Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core
@@ -22,6 +22,7 @@ Requires-Dist: httpx>=0.28.1
 Requires-Dist: jinja2>=3.1.6
 Requires-Dist: lmnr>=0.7.18
 Requires-Dist: openai>=1.109.1
+Requires-Dist: pillow>=10.0.0
 Requires-Dist: prefect-gcp[cloud-storage]>=0.6.10
 Requires-Dist: prefect>=3.4.21
 Requires-Dist: pydantic-settings>=2.10.1
@@ -63,7 +64,7 @@ AI Pipeline Core is a production-ready framework that combines document processi
 - **Structured Output**: Type-safe generation with Pydantic model validation
 - **Workflow Orchestration**: Prefect-based flows and tasks with automatic retries
 - **Observability**: Built-in distributed tracing via Laminar (LMNR) with cost tracking for debugging and monitoring
-- **Local Development**: Simple runner for testing pipelines without infrastructure
+- **Deployment**: Unified pipeline execution for local, CLI, and production environments
 
 ## Installation
 
@@ -124,7 +125,7 @@ async def analyze_flow(
     for doc in documents:
         # Use AIMessages for LLM interaction
         response = await llm.generate(
-            model="gpt-5",
+            model="gpt-5.1",
             messages=AIMessages([doc])
         )
 
@@ -151,7 +152,7 @@ class Analysis(BaseModel):
 
 # Generate structured output
 response = await llm.generate_structured(
-    model="gpt-5",
+    model="gpt-5.1",
     response_format=Analysis,
     messages="Analyze this product review: ..."
 )
@@ -177,7 +178,7 @@ doc = MyDocument.create(
 # Parse back to original type
 data = doc.parse(dict)  # Returns {"key": "value"}
 
-# Document provenance tracking (new in v0.1.14)
+# Document provenance tracking
 doc_with_sources = MyDocument.create(
     name="derived.json",
     content={"result": "processed"},
@@ -224,15 +225,15 @@ if doc.is_text:
 # Parse structured data
 data = doc.as_json()  # or as_yaml(), as_pydantic_model()
 
-# Convert between document types (new in v0.2.1)
+# Convert between document types
 task_doc = flow_doc.model_convert(TaskDocument)  # Convert FlowDocument to TaskDocument
 new_doc = doc.model_convert(OtherDocType, content={"new": "data"})  # With content update
 
-# Enhanced filtering (new in v0.1.14)
+# Enhanced filtering
 filtered = documents.filter_by([Doc1, Doc2, Doc3])  # Multiple types
 named = documents.filter_by(["file1.txt", "file2.txt"])  # Multiple names
 
-# Immutable collections (new in v0.2.1)
+# Immutable collections
 frozen_docs = DocumentList(docs, frozen=True)  # Immutable document list
 frozen_msgs = AIMessages(messages, frozen=True)  # Immutable message list
 ```
@@ -246,7 +247,7 @@ from ai_pipeline_core import llm, AIMessages, ModelOptions
 
 # Simple generation
 response = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     messages="Explain quantum computing"
 )
 print(response.content)
@@ -256,21 +257,21 @@ static_context = AIMessages([large_document])
 
 # First call: caches context
 r1 = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=static_context,  # Cached for 120 seconds by default
     messages="Summarize"     # Dynamic query
 )
 
 # Second call: reuses cache
 r2 = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=static_context,  # Reused from cache!
     messages="Key points?"   # Different query
 )
 
-# Custom cache TTL (new in v0.1.14)
+# Custom cache TTL
 response = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=static_context,
     messages="Analyze",
     options=ModelOptions(cache_ttl="300s")  # Cache for 5 minutes
@@ -278,7 +279,7 @@ response = await llm.generate(
 
 # Disable caching for dynamic contexts
 response = await llm.generate(
-    model="gpt-5",
+    model="gpt-5.1",
     context=dynamic_context,
     messages="Process",
     options=ModelOptions(cache_ttl=None)  # No caching
@@ -317,12 +318,12 @@ from ai_pipeline_core import pipeline_flow, pipeline_task, set_trace_cost
 @pipeline_task  # Automatic retry, tracing, and monitoring
 async def process_chunk(data: str) -> str:
     result = await transform(data)
-    set_trace_cost(0.05)  # Track costs (new in v0.1.14)
+    set_trace_cost(0.05)  # Track costs
     return result
 
 @pipeline_flow(
     config=MyFlowConfig,
-    trace_trim_documents=True  # Trim large documents in traces (new in v0.2.1)
+    trace_trim_documents=True  # Trim large documents in traces
 )
 async def main_flow(
     project_name: str,
@@ -335,6 +336,68 @@ async def main_flow(
     return DocumentList(results)
 ```
 
+### Local Trace Debugging
+
+Save all trace spans to the local filesystem for LLM-assisted debugging:
+
+```bash
+export TRACE_DEBUG_PATH=/path/to/debug/output
+```
+
+This creates a hierarchical directory structure that mirrors the execution flow with automatic deduplication:
+
+```
+20260128_152932_abc12345_my_flow/
+├── _trace.yaml           # Trace metadata
+├── _index.yaml           # Span ID → path mapping
+├── _summary.md           # Unified summary for human inspection and LLM debugging
+├── artifacts/            # Deduplicated content storage
+│   └── sha256/
+│       └── ab/cd/        # Sharded by hash prefix
+│           └── abcdef...1234.txt  # Large content (>10KB)
+└── 0001_my_flow/         # Root span (numbered for execution order)
+    ├── _span.yaml        # Span metadata (timing, status, I/O refs)
+    ├── input.yaml        # Structured inputs (inline or refs)
+    ├── output.yaml       # Structured outputs (inline or refs)
+    ├── 0002_task_1/      # Child spans nested inside parent
+    │   ├── _span.yaml
+    │   ├── input.yaml
+    │   ├── output.yaml
+    │   └── 0003_llm_call/
+    │       ├── _span.yaml
+    │       ├── input.yaml   # LLM messages with inline/external content
+    │       └── output.yaml
+    └── 0004_task_2/
+        └── ...
+```
+
+**Key Features:**
+- **Automatic Deduplication**: Identical content (e.g., system prompts) stored once in `artifacts/`
+- **Smart Externalization**: Large content (>10KB) externalized with 2KB inline previews
+- **AI-Friendly**: Files capped at 50KB for easy LLM processing
+- **Lossless**: Full content reconstruction via `content_ref` pointers
+
+Example `input.yaml` with externalization:
+```yaml
+format_version: 3
+type: llm_messages
+messages:
+  - role: system
+    parts:
+      - type: text
+        size_bytes: 28500
+        content_ref:  # Large content → artifact
+          hash: sha256:a1b2c3d4...
+          path: artifacts/sha256/a1/b2/a1b2c3d4...txt
+        excerpt: "You are a helpful assistant...\n[TRUNCATED]"
+  - role: user
+    parts:
+      - type: text
+        content: "Hello!"  # Small content stays inline
+```
+
+Run `tree` on the output directory to visualize the entire execution hierarchy. Feed `_summary.md` to an LLM for debugging assistance - it combines high-level overview with detailed navigation for comprehensive trace analysis.
+
 ## Configuration
 
 ### Environment Variables
@@ -348,6 +411,9 @@ OPENAI_API_KEY=your-api-key
 LMNR_PROJECT_API_KEY=your-lmnr-key
 LMNR_DEBUG=true  # Enable debug traces
 
+# Optional: Local Trace Debugging
+TRACE_DEBUG_PATH=/path/to/trace/output  # Save traces locally for LLM-assisted debugging
+
 # Optional: Orchestration
 PREFECT_API_URL=http://localhost:4200/api
 PREFECT_API_KEY=your-prefect-key
@@ -458,18 +524,21 @@ For AI assistants:
 ```
 ai-pipeline-core/
 ├── ai_pipeline_core/
-│   ├── documents/      # Document abstraction system
-│   ├── flow/           # Flow configuration and options
-│   ├── llm/            # LLM client and response handling
-│   ├── logging/        # Logging infrastructure
-│   ├── tracing.py      # Distributed tracing
-│   ├── pipeline.py     # Pipeline decorators
+│   ├── deployment/      # Pipeline deployment and execution
+│   ├── documents/       # Document abstraction system
+│   ├── flow/            # Flow configuration and options
+│   ├── llm/             # LLM client and response handling
+│   ├── logging/         # Logging infrastructure
+│   ├── prompt_builder/  # Document-aware prompt construction
+│   ├── pipeline.py      # Pipeline decorators
+│   ├── progress.py      # Intra-flow progress tracking
 │   ├── prompt_manager.py # Jinja2 template management
-│   └── settings.py     # Configuration management
-├── tests/              # Comprehensive test suite
-├── examples/           # Usage examples
-├── API.md             # Complete API reference
-└── pyproject.toml     # Project configuration
+│   ├── settings.py      # Configuration management
+│   └── tracing.py       # Distributed tracing
+├── tests/               # Comprehensive test suite
+├── examples/            # Usage examples
+├── API.md               # Complete API reference
+└── pyproject.toml       # Project configuration
 ```
 
 ## Contributing

ai_pipeline_core-0.3.3.dist-info/RECORD

@@ -0,0 +1,57 @@
+ai_pipeline_core/__init__.py,sha256=2jzEQktQJp-A3bzDU-A7c9xntnY3x9I-1XwYcojYjYE,6452
+ai_pipeline_core/exceptions.py,sha256=vx-XLTw2fJSPs-vwtXVYtqoQUcOc0JeI7UmHqRqQYWU,1569
+ai_pipeline_core/pipeline.py,sha256=t9qH-V6umpKY5MhGuXFgUGfdzGyxzVlS0n9RoKLfnug,28704
+ai_pipeline_core/prefect.py,sha256=91ZgLJHsDsRUW77CpNmkKxYs3RCJuucPM3pjKmNBeDg,2199
+ai_pipeline_core/progress.py,sha256=Ppxk4OOm84Y0x3t-Y3CmHsL4PovQLNUxXMu24zRCD-Q,3621
+ai_pipeline_core/prompt_manager.py,sha256=FAtb1yK7bGuAeuIJ523LOX9bd7TrcHG-TqZ7Lz4RJC0,12087
+ai_pipeline_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ai_pipeline_core/settings.py,sha256=IMrFaX0i-WIlaOA5O53ipNSta6KQVSFHc1aJXmS3nSo,5078
+ai_pipeline_core/tracing.py,sha256=HJ_DJhCEk6W_u3skecjETMQVLyOmbuPcqcOuoMIJlPs,33194
+ai_pipeline_core/debug/__init__.py,sha256=wOc9KotFqGYzBEtZUZ7ATfJf3dXWarYm6PXs6yW9uwE,756
+ai_pipeline_core/debug/config.py,sha256=l5WC2xbd6PgC-CcuioZg696iva_MkqyZj4C9TFdwfMs,3205
+ai_pipeline_core/debug/content.py,sha256=REtA1cJnOJy3OqaGud59B3Bug8cOJszm8w1GCqdAKJs,26696
+ai_pipeline_core/debug/processor.py,sha256=Cvm1HKc6lKRm80Xx7WXi_Z8pWoKH6actVZvntP9Mons,3935
+ai_pipeline_core/debug/summary.py,sha256=pzXC7QoFOBeen_XZ-AMFAVvaOtDuf28YB-WwCbsHYdQ,8017
+ai_pipeline_core/debug/writer.py,sha256=IF5eyML10EBFBqCGqlVwcWKraFvTgfqbU8WJ_XG_RU4,33108
+ai_pipeline_core/deployment/__init__.py,sha256=FN2HVoM80x2GJuNs7o4DnccB8HWWibgM1pJesB942CM,1259
+ai_pipeline_core/deployment/base.py,sha256=JYf8XLFR73c0H24dr6atK7yUcoE0vLxbYZ8EkQpEwN4,24791
+ai_pipeline_core/deployment/contract.py,sha256=0DKt5eqNE-grcITwMNq9CuBdo5WxdopEjDeQFzFZxhU,2225
+ai_pipeline_core/deployment/helpers.py,sha256=3nRuCyABkUEDZiL0q9u19XHpjA4527B6rsxQNOGTohw,3460
+ai_pipeline_core/documents/__init__.py,sha256=WHStvGZiSyybOcMTYxSV24U6MA3Am_0_Az5p-DuMFrk,738
+ai_pipeline_core/documents/document.py,sha256=hdTh36KGEcrDollTnQmTI66DJIqYfe4X42Y0q7Cm4fY,68153
+ai_pipeline_core/documents/document_list.py,sha256=Y_NCjfM_CjkIwHRD2iyGgYBuIykN8lT2IIH_uWOiGis,16254
+ai_pipeline_core/documents/flow_document.py,sha256=QK6RxNQu449IRAosOHSk3G_5yIq5I7yLBOSQPCd3m64,4141
+ai_pipeline_core/documents/mime_type.py,sha256=JFEOq4HwlIW2snobyNfWwySdT7urZSWkobiRMVs2fSE,7959
+ai_pipeline_core/documents/task_document.py,sha256=uASmAaxNkYtuqQrBM57vutFT9DXNTbqv0wbwwF55E3I,4300
+ai_pipeline_core/documents/temporary_document.py,sha256=jaz2ZHC5CmSbVbkXdI7pOB5DGEuhH16C0Yutv-lS_UI,2708
+ai_pipeline_core/documents/utils.py,sha256=ZyJNjFN7ihWno0K7dJZed7twYmmPLA0z40UzFw1A3A8,5465
+ai_pipeline_core/flow/__init__.py,sha256=2BfWYMOPYW5teGzwo-qzpn_bom1lxxry0bPsjVgcsCk,188
+ai_pipeline_core/flow/config.py,sha256=a9FALpgrFsdz-D7HU3diVeUzbaBvLwI8hsPviuj001s,19389
+ai_pipeline_core/flow/options.py,sha256=s5GBTy5lwFa1irf8BKrWO8NMZ5s_f4tqq7Wg9WQ7TTg,2302
+ai_pipeline_core/images/__init__.py,sha256=6R6Ncif6oRyVOH7LsdwNvEuMGHuljo-_gImY8C3Z_ow,9877
+ai_pipeline_core/images/_processing.py,sha256=wKSBAFe5TO-mo64ll20nmN9huazHwvVWFfNJB6g7u2Q,4421
+ai_pipeline_core/llm/__init__.py,sha256=3B_vtEzxrzidP1qOUNQ4RxlUmxZ2MBKQcUhQiTybM9g,661
+ai_pipeline_core/llm/ai_messages.py,sha256=XR2fwzguuh7v-HQ9LEJX_xwNX1D_-6f4T7E8_iNVTS4,15680
+ai_pipeline_core/llm/client.py,sha256=777Zf5BBRA-6g1I4Og8mitpoCPdYMp66WE52wH-1I-o,24910
+ai_pipeline_core/llm/model_options.py,sha256=uRNIHfVeh2sgt1mZBiOUx6hPQ6GKjB8b7TytZJ6afKg,11768
+ai_pipeline_core/llm/model_response.py,sha256=zEANsfuSAYVRKPwKx9gFIqHbdVG_1_JNMRHNoE43_YM,13503
+ai_pipeline_core/llm/model_types.py,sha256=wx-m0up7_NncTmSYmMsL-l-RgydjjJ905u7RMEAg7tI,2710
+ai_pipeline_core/logging/__init__.py,sha256=Nz6-ghAoENsgNmLD2ma9TW9M0U2_QfxuQ5DDW6Vt6M0,651
+ai_pipeline_core/logging/logging.yml,sha256=YTW48keO_K5bkkb-KXGM7ZuaYKiquLsjsURei8Ql0V4,1353
+ai_pipeline_core/logging/logging_config.py,sha256=pV2x6GgMPXrzPH27sicCSXfw56beio4C2JKCJ3NsXrg,6207
+ai_pipeline_core/logging/logging_mixin.py,sha256=OpdR3ASiM2ZwKZYGjZRJFUloGWUCv2Grnr8RqUWlYn8,8094
+ai_pipeline_core/prompt_builder/__init__.py,sha256=-v0SKZlir07xRzxXwv75VP66aINRUiKH0VUgB-PCDmI,195
+ai_pipeline_core/prompt_builder/documents_prompt.jinja2,sha256=LPql5AaFhFWtDfhnBWvi-bWbz5vdgsWqKGzcqxWfLIM,1075
+ai_pipeline_core/prompt_builder/global_cache.py,sha256=9_9zoF6-sr3KBMxF5QLD3vxqXg9B2tT8o9ViplzUCNg,2811
+ai_pipeline_core/prompt_builder/new_core_documents_prompt.jinja2,sha256=M8uPpwf-uLpsWWJT9DY_DnjrLToGPVnrD-gVhQrQdaQ,229
+ai_pipeline_core/prompt_builder/prompt_builder.py,sha256=4TrDRPiOMFwEfi6QGfriTHfjzj_CtbEjAcgQrVfRqhw,9378
+ai_pipeline_core/prompt_builder/system_prompt.jinja2,sha256=-1jLcfvAG07Zfl-dnYrjfVcAG4PWeeoeWpaKJGY3rKQ,3945
+ai_pipeline_core/storage/__init__.py,sha256=tcIkjJ3zPBLCyetwiJDewBvS2sbRJrDlBh3gEsQm08E,184
+ai_pipeline_core/storage/storage.py,sha256=ClMr419Y-eU2RuOjZYd51dC0stWQk28Vb56PvQaoUwc,20007
+ai_pipeline_core/utils/__init__.py,sha256=TJSmEm1Quf-gKwXrxM96u2IGzVolUyeNNfLMPoLstXI,254
+ai_pipeline_core/utils/deploy.py,sha256=N3i7B97DQJs1lwgYN3sa1UgwCNjseKXfjs50ZJUMCEI,22106
+ai_pipeline_core/utils/remote_deployment.py,sha256=U7MNJ1SU1mg3RrJyLqpuN_4pwqm8LSsFZbypJvjGPoo,4630
+ai_pipeline_core-0.3.3.dist-info/METADATA,sha256=WCRBGS2kO8916jlGc6jY_YuPwzw8diXfeNSrNFKxuvk,17893
+ai_pipeline_core-0.3.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+ai_pipeline_core-0.3.3.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
+ai_pipeline_core-0.3.3.dist-info/RECORD,,

{ai_pipeline_core-0.2.9.dist-info → ai_pipeline_core-0.3.3.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any

ai_pipeline_core/simple_runner/__init__.py

@@ -1,14 +0,0 @@
-"""Simple pipeline execution for local development.
-
-Utilities for running AI pipelines locally without full Prefect orchestration.
-"""
-
-from .cli import run_cli
-from .simple_runner import FlowSequence, run_pipeline, run_pipelines
-
-__all__ = [
-    "run_cli",
-    "run_pipeline",
-    "run_pipelines",
-    "FlowSequence",
-]

ai_pipeline_core/simple_runner/cli.py

@@ -1,254 +0,0 @@
-"""Command-line interface for simple pipeline execution."""
-
-import asyncio
-import os
-import sys
-from contextlib import ExitStack
-from pathlib import Path
-from typing import Callable, Type, TypeVar, cast
-
-from lmnr import Laminar
-from pydantic import ValidationError
-from pydantic_settings import CliPositionalArg, SettingsConfigDict
-
-from ai_pipeline_core.documents import DocumentList
-from ai_pipeline_core.flow.options import FlowOptions
-from ai_pipeline_core.logging import get_pipeline_logger, setup_logging
-from ai_pipeline_core.prefect import disable_run_logger, prefect_test_harness
-from ai_pipeline_core.settings import settings
-
-from .simple_runner import FlowSequence, run_pipelines
-
-logger = get_pipeline_logger(__name__)
-
-TOptions = TypeVar("TOptions", bound=FlowOptions)
-"""Type variable for FlowOptions subclasses used in CLI."""
-
-InitializerFunc = Callable[[FlowOptions], tuple[str, DocumentList]] | None
-"""Function type for custom pipeline initialization.
-
-Initializers can create initial documents or setup project state
-before flow execution begins.
-
-Args:
-    FlowOptions: Parsed CLI options
-
-Returns:
-    Tuple of (project_name, initial_documents) or None
-"""
-
-
-def _initialize_environment() -> None:
-    """Initialize logging and observability systems.
-
-    Sets up the pipeline logging configuration and attempts to
-    initialize LMNR (Laminar) for distributed tracing. Failures
-    in LMNR initialization are logged but don't stop execution.
-
-    Side effects:
-        - Configures Python logging system
-        - Initializes Laminar SDK if API key is available
-        - Logs initialization status
-
-    Note:
-        Called automatically by run_cli before parsing arguments.
-    """
-    setup_logging()
-    try:
-        Laminar.initialize()
-        logger.info("LMNR tracing initialized.")
-    except Exception as e:
-        logger.warning(f"Failed to initialize LMNR tracing: {e}")
-
-
-def _running_under_pytest() -> bool:
-    """Check if code is running under pytest.
-
-    Detects pytest execution context to determine whether test
-    fixtures will provide necessary contexts (like Prefect test
-    harness). This prevents duplicate context setup.
-
-    Returns:
-        True if running under pytest, False otherwise.
-
-    Detection methods:
-        - PYTEST_CURRENT_TEST environment variable (set by pytest)
-        - 'pytest' module in sys.modules (imported by test runner)
-
-    Note:
-        Used to avoid setting up test harness when pytest fixtures
-        already provide it.
-    """
-    return "PYTEST_CURRENT_TEST" in os.environ or "pytest" in sys.modules
-
-
-def run_cli(
-    *,
-    flows: FlowSequence,
-    options_cls: Type[TOptions],
-    initializer: InitializerFunc = None,
-    trace_name: str | None = None,
-) -> None:
-    """Execute pipeline flows from command-line arguments.
-
-    Environment setup:
-        - Initializes logging system
-        - Sets up LMNR tracing (if API key configured)
-        - Creates Prefect test harness (if no API key and not in pytest)
-        - Manages context stack for proper cleanup
-
-    Raises:
-        ValueError: If project name is empty after initialization.
-
-    Example:
-        >>> # In __main__.py
-        >>> from ai_pipeline_core import simple_runner
-        >>> from .flows import AnalysisFlow, SummaryFlow
-        >>> from .config import AnalysisOptions
-        >>>
-        >>> if __name__ == "__main__":
-        ...     simple_runner.run_cli(
-        ...         flows=[AnalysisFlow, SummaryFlow],
-        ...         options_cls=AnalysisOptions,
-        ...         trace_name="document-analysis"
-        ...     )
-
-        Command line:
-        $ python -m my_module ./output --temperature 0.5 --model gpt-5
-        $ python -m my_module ./output --start 2  # Skip first flow
-
-    Note:
-        - Field names are converted to kebab-case for CLI (max_tokens → --max-tokens)
-        - Boolean fields become flags (--verbose/--no-verbose)
-        - Field descriptions from Pydantic become help text
-        - Type hints are enforced during parsing
-        - Validation errors show helpful messages with field names
-        - Includes hints for common error types (numbers, ranges)
-        - Exits with status 1 on error
-        - Shows --help when no arguments provided
-    """
-    # Check if no arguments provided before initialization
-    if len(sys.argv) == 1:
-        # Add --help to show usage when run without arguments
-        sys.argv.append("--help")
-
-    _initialize_environment()
-
-    class _RunnerOptions(  # type: ignore[reportRedeclaration]
-        options_cls,
-        cli_parse_args=True,
-        cli_kebab_case=True,
-        cli_exit_on_error=True,  # Let it exit normally on error
-        cli_prog_name="ai-pipeline",
-        cli_use_class_docs_for_groups=True,
-    ):
-        """Internal options class combining user options with CLI arguments.
-
-        Dynamically created class that inherits from user's options_cls
-        and adds standard CLI arguments for pipeline execution.
-        """
-
-        working_directory: CliPositionalArg[Path]
-        project_name: str | None = None
-        start: int = 1
-        end: int | None = None
-
-        model_config = SettingsConfigDict(frozen=True, extra="ignore")
-
-    try:
-        opts = cast(FlowOptions, _RunnerOptions())  # type: ignore[reportCallIssue]
-    except ValidationError as e:
-        print("\nError: Invalid command line arguments\n", file=sys.stderr)
-        for error in e.errors():
-            field = " -> ".join(str(loc) for loc in error["loc"])
-            msg = error["msg"]
-            value = error.get("input", "")
-
-            # Format the field name nicely (convert from snake_case to kebab-case for CLI)
-            cli_field = field.replace("_", "-")
-
-            print(f"  --{cli_field}: {msg}", file=sys.stderr)
-            if value:
-                print(f"    Provided value: '{value}'", file=sys.stderr)
-
-            # Add helpful hints for common errors
-            if error["type"] == "float_parsing":
-                print("    Hint: Please provide a valid number (e.g., 0.7)", file=sys.stderr)
-            elif error["type"] == "int_parsing":
-                print("    Hint: Please provide a valid integer (e.g., 10)", file=sys.stderr)
-            elif error["type"] == "literal_error":
-                ctx = error.get("ctx", {})
-                expected = ctx.get("expected", "valid options")
-                print(f"    Hint: Valid options are: {expected}", file=sys.stderr)
-            elif error["type"] in [
-                "less_than_equal",
-                "greater_than_equal",
-                "less_than",
-                "greater_than",
-            ]:
-                ctx = error.get("ctx", {})
-                if "le" in ctx:
-                    print(f"    Hint: Value must be ≤ {ctx['le']}", file=sys.stderr)
-                elif "ge" in ctx:
-                    print(f"    Hint: Value must be ≥ {ctx['ge']}", file=sys.stderr)
-                elif "lt" in ctx:
-                    print(f"    Hint: Value must be < {ctx['lt']}", file=sys.stderr)
-                elif "gt" in ctx:
-                    print(f"    Hint: Value must be > {ctx['gt']}", file=sys.stderr)
-
-        print("\nRun with --help to see all available options\n", file=sys.stderr)
-        sys.exit(1)
-
-    wd: Path = cast(Path, getattr(opts, "working_directory"))
-    wd.mkdir(parents=True, exist_ok=True)
-
-    # Get project name from options or use directory basename
-    project_name = getattr(opts, "project_name", None)
-    if not project_name:  # None or empty string
-        project_name = wd.name
-
-    # Ensure project_name is not empty
-    if not project_name:
-        raise ValueError("Project name cannot be empty")
-
-    # Use initializer if provided, otherwise use defaults
-    initial_documents = DocumentList([])
-    if initializer:
-        init_result = initializer(opts)
-        # Always expect tuple format from initializer
-        _, initial_documents = init_result  # Ignore project name from initializer
-
-        # Save initial documents if starting from first step
-        if getattr(opts, "start", 1) == 1 and initial_documents and flows:
-            # Get config from the first flow
-            first_flow_config = getattr(flows[0], "config", None)
-            if first_flow_config:
-                asyncio.run(
-                    first_flow_config.save_documents(
-                        str(wd), initial_documents, validate_output_type=False
-                    )
-                )
-
-    # Setup context stack with optional test harness and tracing
-    with ExitStack() as stack:
-        if trace_name:
-            stack.enter_context(
-                Laminar.start_as_current_span(
-                    name=f"{trace_name}-{project_name}", input=[opts.model_dump_json()]
-                )
-            )
-
-        if not settings.prefect_api_key and not _running_under_pytest():
-            stack.enter_context(prefect_test_harness())
-            stack.enter_context(disable_run_logger())
-
-        asyncio.run(
-            run_pipelines(
-                project_name=project_name,
-                output_dir=wd,
-                flows=flows,
-                flow_options=opts,
-                start_step=getattr(opts, "start", 1),
-                end_step=getattr(opts, "end", None),
-            )
-        )