ai-pipeline-core 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

ai_pipeline_core/pipeline.py

@@ -0,0 +1,418 @@
+"""Pipeline decorators that combine Prefect functionality with tracing support.
+
+These decorators extend the base Prefect decorators with automatic tracing capabilities.
+"""
+
+import datetime
+import functools
+import inspect
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Coroutine,
+    Dict,
+    Iterable,
+    Optional,
+    TypeVar,
+    Union,
+    cast,
+    overload,
+)
+
+from prefect.assets import Asset
+from prefect.cache_policies import CachePolicy
+from prefect.context import TaskRunContext
+from prefect.flows import Flow, FlowStateHook
+from prefect.futures import PrefectFuture
+from prefect.results import ResultSerializer, ResultStorage
+from prefect.task_runners import TaskRunner
+from prefect.tasks import (
+    RetryConditionCallable,
+    StateHookCallable,
+    Task,
+    TaskRunNameValueOrCallable,
+)
+from prefect.utilities.annotations import NotSet
+from typing_extensions import Concatenate, ParamSpec
+
+from ai_pipeline_core.documents import DocumentList
+from ai_pipeline_core.flow.options import FlowOptions
+from ai_pipeline_core.prefect import flow, task
+from ai_pipeline_core.tracing import TraceLevel, trace
+
+if TYPE_CHECKING:
+    pass
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+# ============================================================================
+# PIPELINE TASK DECORATOR
+# ============================================================================
+
+
+@overload
+def pipeline_task(__fn: Callable[P, R], /) -> Task[P, R]: ...
+
+
+@overload
+def pipeline_task(
+    *,
+    # Tracing parameters
+    trace_level: TraceLevel = "always",
+    trace_ignore_input: bool = False,
+    trace_ignore_output: bool = False,
+    trace_ignore_inputs: list[str] | None = None,
+    trace_input_formatter: Optional[Callable[..., str]] = None,
+    trace_output_formatter: Optional[Callable[..., str]] = None,
+    # Prefect parameters
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    tags: Optional[Iterable[str]] = None,
+    version: Optional[str] = None,
+    cache_policy: Union[CachePolicy, type[NotSet]] = NotSet,
+    cache_key_fn: Optional[Callable[[TaskRunContext, Dict[str, Any]], Optional[str]]] = None,
+    cache_expiration: Optional[datetime.timedelta] = None,
+    task_run_name: Optional[TaskRunNameValueOrCallable] = None,
+    retries: Optional[int] = None,
+    retry_delay_seconds: Optional[
+        Union[float, int, list[float], Callable[[int], list[float]]]
+    ] = None,
+    retry_jitter_factor: Optional[float] = None,
+    persist_result: Optional[bool] = None,
+    result_storage: Optional[Union[ResultStorage, str]] = None,
+    result_serializer: Optional[Union[ResultSerializer, str]] = None,
+    result_storage_key: Optional[str] = None,
+    cache_result_in_memory: bool = True,
+    timeout_seconds: Union[int, float, None] = None,
+    log_prints: Optional[bool] = False,
+    refresh_cache: Optional[bool] = None,
+    on_completion: Optional[list[StateHookCallable]] = None,
+    on_failure: Optional[list[StateHookCallable]] = None,
+    retry_condition_fn: Optional[RetryConditionCallable] = None,
+    viz_return_value: Optional[bool] = None,
+    asset_deps: Optional[list[Union[str, Asset]]] = None,
+) -> Callable[[Callable[P, R]], Task[P, R]]: ...
+
+
+def pipeline_task(
+    __fn: Optional[Callable[P, R]] = None,
+    /,
+    *,
+    # Tracing parameters
+    trace_level: TraceLevel = "always",
+    trace_ignore_input: bool = False,
+    trace_ignore_output: bool = False,
+    trace_ignore_inputs: list[str] | None = None,
+    trace_input_formatter: Optional[Callable[..., str]] = None,
+    trace_output_formatter: Optional[Callable[..., str]] = None,
+    # Prefect parameters
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    tags: Optional[Iterable[str]] = None,
+    version: Optional[str] = None,
+    cache_policy: Union[CachePolicy, type[NotSet]] = NotSet,
+    cache_key_fn: Optional[Callable[[TaskRunContext, Dict[str, Any]], Optional[str]]] = None,
+    cache_expiration: Optional[datetime.timedelta] = None,
+    task_run_name: Optional[TaskRunNameValueOrCallable] = None,
+    retries: Optional[int] = None,
+    retry_delay_seconds: Optional[
+        Union[float, int, list[float], Callable[[int], list[float]]]
+    ] = None,
+    retry_jitter_factor: Optional[float] = None,
+    persist_result: Optional[bool] = None,
+    result_storage: Optional[Union[ResultStorage, str]] = None,
+    result_serializer: Optional[Union[ResultSerializer, str]] = None,
+    result_storage_key: Optional[str] = None,
+    cache_result_in_memory: bool = True,
+    timeout_seconds: Union[int, float, None] = None,
+    log_prints: Optional[bool] = False,
+    refresh_cache: Optional[bool] = None,
+    on_completion: Optional[list[StateHookCallable]] = None,
+    on_failure: Optional[list[StateHookCallable]] = None,
+    retry_condition_fn: Optional[RetryConditionCallable] = None,
+    viz_return_value: Optional[bool] = None,
+    asset_deps: Optional[list[Union[str, Asset]]] = None,
+) -> Union[Task[P, R], Callable[[Callable[P, R]], Task[P, R]]]:
+    """
+    Pipeline task decorator that combines Prefect task functionality with automatic tracing.
+
+    This decorator applies tracing before the Prefect task decorator, allowing you to
+    monitor task execution with LMNR while maintaining all Prefect functionality.
+
+    Args:
+        trace_level: Control tracing ("always", "debug", "off")
+        trace_ignore_input: Whether to ignore input in traces
+        trace_ignore_output: Whether to ignore output in traces
+        trace_ignore_inputs: List of input parameter names to ignore
+        trace_input_formatter: Custom formatter for inputs
+        trace_output_formatter: Custom formatter for outputs
+
+        Plus all standard Prefect task parameters...
+    """
+
+    def decorator(fn: Callable[P, R]) -> Task[P, R]:
+        # Apply tracing first if enabled
+        if trace_level != "off":
+            traced_fn = trace(
+                level=trace_level,
+                name=name or fn.__name__,
+                ignore_input=trace_ignore_input,
+                ignore_output=trace_ignore_output,
+                ignore_inputs=trace_ignore_inputs,
+                input_formatter=trace_input_formatter,
+                output_formatter=trace_output_formatter,
+            )(fn)
+        else:
+            traced_fn = fn
+
+        # Then apply Prefect task decorator
+        return task(  # pyright: ignore[reportCallIssue,reportUnknownVariableType]
+            traced_fn,  # pyright: ignore[reportArgumentType]
+            name=name,
+            description=description,
+            tags=tags,
+            version=version,
+            cache_policy=cache_policy,
+            cache_key_fn=cache_key_fn,
+            cache_expiration=cache_expiration,
+            task_run_name=task_run_name,
+            retries=retries or 0,
+            retry_delay_seconds=retry_delay_seconds,
+            retry_jitter_factor=retry_jitter_factor,
+            persist_result=persist_result,
+            result_storage=result_storage,
+            result_serializer=result_serializer,
+            result_storage_key=result_storage_key,
+            cache_result_in_memory=cache_result_in_memory,
+            timeout_seconds=timeout_seconds,
+            log_prints=log_prints,
+            refresh_cache=refresh_cache,
+            on_completion=on_completion,
+            on_failure=on_failure,
+            retry_condition_fn=retry_condition_fn,
+            viz_return_value=viz_return_value,
+            asset_deps=asset_deps,
+        )
+
+    if __fn:
+        return decorator(__fn)
+    return decorator
+
+
+# ============================================================================
+# PIPELINE FLOW DECORATOR WITH DOCUMENT PROCESSING
+# ============================================================================
+
+# Type aliases for document flow signatures
+DocumentsFlowSig = Callable[
+    Concatenate[str, DocumentList, FlowOptions, P],
+    Union[DocumentList, Coroutine[Any, Any, DocumentList]],
+]
+
+DocumentsFlowResult = Flow[Concatenate[str, DocumentList, FlowOptions, P], DocumentList]
+
+
+@overload
+def pipeline_flow(
+    __fn: DocumentsFlowSig[P],
+    /,
+) -> DocumentsFlowResult[P]: ...
+
+
+@overload
+def pipeline_flow(
+    *,
+    # Tracing parameters
+    trace_level: TraceLevel = "always",
+    trace_ignore_input: bool = False,
+    trace_ignore_output: bool = False,
+    trace_ignore_inputs: list[str] | None = None,
+    trace_input_formatter: Optional[Callable[..., str]] = None,
+    trace_output_formatter: Optional[Callable[..., str]] = None,
+    # Prefect parameters
+    name: Optional[str] = None,
+    version: Optional[str] = None,
+    flow_run_name: Optional[Union[Callable[[], str], str]] = None,
+    retries: Optional[int] = None,
+    retry_delay_seconds: Optional[Union[int, float]] = None,
+    task_runner: Optional[TaskRunner[PrefectFuture[Any]]] = None,
+    description: Optional[str] = None,
+    timeout_seconds: Union[int, float, None] = None,
+    validate_parameters: bool = True,
+    persist_result: Optional[bool] = None,
+    result_storage: Optional[Union[ResultStorage, str]] = None,
+    result_serializer: Optional[Union[ResultSerializer, str]] = None,
+    cache_result_in_memory: bool = True,
+    log_prints: Optional[bool] = None,
+    on_completion: Optional[list["FlowStateHook[..., Any]"]] = None,
+    on_failure: Optional[list["FlowStateHook[..., Any]"]] = None,
+    on_cancellation: Optional[list["FlowStateHook[..., Any]"]] = None,
+    on_crashed: Optional[list["FlowStateHook[..., Any]"]] = None,
+    on_running: Optional[list["FlowStateHook[..., Any]"]] = None,
+) -> Callable[[DocumentsFlowSig[P]], DocumentsFlowResult[P]]: ...
+
+
+def pipeline_flow(
+    __fn: Optional[DocumentsFlowSig[P]] = None,
+    /,
+    *,
+    # Tracing parameters
+    trace_level: TraceLevel = "always",
+    trace_ignore_input: bool = False,
+    trace_ignore_output: bool = False,
+    trace_ignore_inputs: list[str] | None = None,
+    trace_input_formatter: Optional[Callable[..., str]] = None,
+    trace_output_formatter: Optional[Callable[..., str]] = None,
+    # Prefect parameters
+    name: Optional[str] = None,
+    version: Optional[str] = None,
+    flow_run_name: Optional[Union[Callable[[], str], str]] = None,
+    retries: Optional[int] = None,
+    retry_delay_seconds: Optional[Union[int, float]] = None,
+    task_runner: Optional[TaskRunner[PrefectFuture[Any]]] = None,
+    description: Optional[str] = None,
+    timeout_seconds: Union[int, float, None] = None,
+    validate_parameters: bool = True,
+    persist_result: Optional[bool] = None,
+    result_storage: Optional[Union[ResultStorage, str]] = None,
+    result_serializer: Optional[Union[ResultSerializer, str]] = None,
+    cache_result_in_memory: bool = True,
+    log_prints: Optional[bool] = None,
+    on_completion: Optional[list["FlowStateHook[..., Any]"]] = None,
+    on_failure: Optional[list["FlowStateHook[..., Any]"]] = None,
+    on_cancellation: Optional[list["FlowStateHook[..., Any]"]] = None,
+    on_crashed: Optional[list["FlowStateHook[..., Any]"]] = None,
+    on_running: Optional[list["FlowStateHook[..., Any]"]] = None,
+) -> Union[DocumentsFlowResult[P], Callable[[DocumentsFlowSig[P]], DocumentsFlowResult[P]]]:
+    """
+    Pipeline flow for document processing with standardized signature.
+
+    This decorator enforces a specific signature for document processing flows:
+    - First parameter: project_name (str)
+    - Second parameter: documents (DocumentList)
+    - Third parameter: flow_options (FlowOptions or subclass)
+    - Additional parameters allowed
+    - Must return DocumentList
+
+    It includes automatic tracing and all Prefect flow functionality.
+
+    Args:
+        trace_level: Control tracing ("always", "debug", "off")
+        trace_ignore_input: Whether to ignore input in traces
+        trace_ignore_output: Whether to ignore output in traces
+        trace_ignore_inputs: List of input parameter names to ignore
+        trace_input_formatter: Custom formatter for inputs
+        trace_output_formatter: Custom formatter for outputs
+
+        Plus all standard Prefect flow parameters...
+    """
+
+    def decorator(func: DocumentsFlowSig[P]) -> DocumentsFlowResult[P]:
+        sig = inspect.signature(func)
+        params = list(sig.parameters.values())
+
+        if len(params) < 3:
+            raise TypeError(
+                f"@pipeline_flow '{func.__name__}' must accept at least 3 arguments: "
+                "(project_name, documents, flow_options)"
+            )
+
+        # Validate parameter types (optional but recommended)
+        # We check names as a convention, not strict type checking at decoration time
+        expected_names = ["project_name", "documents", "flow_options"]
+        for i, expected in enumerate(expected_names):
+            if i < len(params) and params[i].name != expected:
+                print(
+                    f"Warning: Parameter {i + 1} of '{func.__name__}' is named '{params[i].name}' "
+                    f"but convention suggests '{expected}'"
+                )
+
+        # Create wrapper that ensures return type
+        if inspect.iscoroutinefunction(func):
+
+            @functools.wraps(func)
+            async def wrapper(  # pyright: ignore[reportRedeclaration]
+                project_name: str,
+                documents: DocumentList,
+                flow_options: FlowOptions,
+                *args,  # pyright: ignore[reportMissingParameterType]
+                **kwargs,  # pyright: ignore[reportMissingParameterType]
+            ) -> DocumentList:
+                result = await func(project_name, documents, flow_options, *args, **kwargs)
+                # Runtime type checking
+                DL = DocumentList  # Avoid recomputation
+                if not isinstance(result, DL):
+                    raise TypeError(
+                        f"Flow '{func.__name__}' must return a DocumentList, "
+                        f"but returned {type(result).__name__}"
+                    )
+                return result
+        else:
+
+            @functools.wraps(func)
+            def wrapper(  # pyright: ignore[reportRedeclaration]
+                project_name: str,
+                documents: DocumentList,
+                flow_options: FlowOptions,
+                *args,  # pyright: ignore[reportMissingParameterType]
+                **kwargs,  # pyright: ignore[reportMissingParameterType]
+            ) -> DocumentList:
+                result = func(project_name, documents, flow_options, *args, **kwargs)
+                # Runtime type checking
+                DL = DocumentList  # Avoid recomputation
+                if not isinstance(result, DL):
+                    raise TypeError(
+                        f"Flow '{func.__name__}' must return a DocumentList, "
+                        f"but returned {type(result).__name__}"
+                    )
+                return result
+
+        # Apply tracing first if enabled
+        if trace_level != "off":
+            traced_wrapper = trace(
+                level=trace_level,
+                name=name or func.__name__,
+                ignore_input=trace_ignore_input,
+                ignore_output=trace_ignore_output,
+                ignore_inputs=trace_ignore_inputs,
+                input_formatter=trace_input_formatter,
+                output_formatter=trace_output_formatter,
+            )(wrapper)
+        else:
+            traced_wrapper = wrapper
+
+        # Then apply Prefect flow decorator
+        return cast(
+            DocumentsFlowResult[P],
+            flow(  # pyright: ignore[reportCallIssue,reportUnknownVariableType]
+                traced_wrapper,  # pyright: ignore[reportArgumentType]
+                name=name,
+                version=version,
+                flow_run_name=flow_run_name,
+                retries=retries,
+                retry_delay_seconds=retry_delay_seconds,
+                task_runner=task_runner,
+                description=description,
+                timeout_seconds=timeout_seconds,
+                validate_parameters=validate_parameters,
+                persist_result=persist_result,
+                result_storage=result_storage,
+                result_serializer=result_serializer,
+                cache_result_in_memory=cache_result_in_memory,
+                log_prints=log_prints,
+                on_completion=on_completion,
+                on_failure=on_failure,
+                on_cancellation=on_cancellation,
+                on_crashed=on_crashed,
+                on_running=on_running,
+            ),
+        )
+
+    if __fn:
+        return decorator(__fn)
+    return decorator
+
+
+__all__ = ["pipeline_task", "pipeline_flow"]

ai_pipeline_core/prefect.py

@@ -0,0 +1,7 @@
+"""Prefect core features."""
+
+from prefect import flow, task
+from prefect.logging import disable_run_logger
+from prefect.testing.utilities import prefect_test_harness
+
+__all__ = ["task", "flow", "disable_run_logger", "prefect_test_harness"]

ai_pipeline_core/simple_runner/__init__.py

@@ -0,0 +1,19 @@
+from .cli import run_cli
+from .simple_runner import (
+    ConfigSequence,
+    FlowSequence,
+    load_documents_from_directory,
+    run_pipeline,
+    run_pipelines,
+    save_documents_to_directory,
+)
+
+__all__ = [
+    "run_cli",
+    "run_pipeline",
+    "run_pipelines",
+    "load_documents_from_directory",
+    "save_documents_to_directory",
+    "FlowSequence",
+    "ConfigSequence",
+]

ai_pipeline_core/simple_runner/cli.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from typing import Callable, Type, TypeVar, cast
+
+from lmnr import Laminar
+from pydantic_settings import CliPositionalArg, SettingsConfigDict
+
+from ai_pipeline_core.documents import DocumentList
+from ai_pipeline_core.flow.options import FlowOptions
+from ai_pipeline_core.logging import get_pipeline_logger, setup_logging
+
+from .simple_runner import ConfigSequence, FlowSequence, run_pipelines, save_documents_to_directory
+
+logger = get_pipeline_logger(__name__)
+
+TOptions = TypeVar("TOptions", bound=FlowOptions)
+InitializerFunc = Callable[[FlowOptions], tuple[str, DocumentList]] | None
+
+
+def _initialize_environment() -> None:
+    setup_logging()
+    try:
+        Laminar.initialize()
+        logger.info("LMNR tracing initialized.")
+    except Exception as e:
+        logger.warning(f"Failed to initialize LMNR tracing: {e}")
+
+
+def run_cli(
+    *,
+    flows: FlowSequence,
+    flow_configs: ConfigSequence,
+    options_cls: Type[TOptions],
+    initializer: InitializerFunc = None,
+) -> None:
+    """
+    Parse CLI+env into options, then run the pipeline.
+
+    - working_directory: required positional arg
+    - --project-name: optional, defaults to directory name
+    - --start/--end: optional, 1-based step bounds
+    - all other flags come from options_cls (fields & Field descriptions)
+    """
+    _initialize_environment()
+
+    class _RunnerOptions(  # type: ignore[reportRedeclaration]
+        options_cls,
+        cli_parse_args=True,
+        cli_kebab_case=True,
+        cli_exit_on_error=False,
+    ):
+        working_directory: CliPositionalArg[Path]
+        project_name: str | None = None
+        start: int = 1
+        end: int | None = None
+
+        model_config = SettingsConfigDict(frozen=True, extra="ignore")
+
+    opts = cast(FlowOptions, _RunnerOptions())  # type: ignore[reportCallIssue]
+
+    wd: Path = cast(Path, getattr(opts, "working_directory"))
+    wd.mkdir(parents=True, exist_ok=True)
+
+    # Get project name from options or use directory basename
+    project_name = getattr(opts, "project_name", None)
+    if not project_name:  # None or empty string
+        project_name = wd.name
+
+    # Ensure project_name is not empty
+    if not project_name:
+        raise ValueError("Project name cannot be empty")
+
+    # Use initializer if provided, otherwise use defaults
+    initial_documents = DocumentList([])
+    if initializer:
+        init_result = initializer(opts)
+        # Always expect tuple format from initializer
+        _, initial_documents = init_result  # Ignore project name from initializer
+
+        if getattr(opts, "start", 1) == 1 and initial_documents:
+            save_documents_to_directory(wd, initial_documents)
+
+    asyncio.run(
+        run_pipelines(
+            project_name=project_name,
+            output_dir=wd,
+            flows=flows,
+            flow_configs=flow_configs,
+            flow_options=opts,
+            start_step=getattr(opts, "start", 1),
+            end_step=getattr(opts, "end", None),
+        )
+    )

ai_pipeline_core/simple_runner/simple_runner.py

@@ -0,0 +1,147 @@
+from pathlib import Path
+from typing import Any, Callable, Sequence, Type
+
+from ai_pipeline_core.documents import Document, DocumentList, FlowDocument
+from ai_pipeline_core.flow.config import FlowConfig
+from ai_pipeline_core.flow.options import FlowOptions
+from ai_pipeline_core.logging import get_pipeline_logger
+
+logger = get_pipeline_logger(__name__)
+
+FlowSequence = Sequence[Callable[..., Any]]
+ConfigSequence = Sequence[Type[FlowConfig]]
+
+
+def load_documents_from_directory(
+    base_dir: Path, document_types: Sequence[Type[FlowDocument]]
+) -> DocumentList:
+    """Loads documents using canonical_name."""
+    documents = DocumentList()
+
+    for doc_class in document_types:
+        dir_name = doc_class.canonical_name()
+        type_dir = base_dir / dir_name
+
+        if not type_dir.exists() or not type_dir.is_dir():
+            continue
+
+        logger.info(f"Loading documents from {type_dir.relative_to(base_dir)}")
+
+        for file_path in type_dir.iterdir():
+            if not file_path.is_file() or file_path.name.endswith(Document.DESCRIPTION_EXTENSION):
+                continue
+
+            try:
+                content = file_path.read_bytes()
+                doc = doc_class(name=file_path.name, content=content)
+
+                desc_file = file_path.with_name(file_path.name + Document.DESCRIPTION_EXTENSION)
+                if desc_file.exists():
+                    object.__setattr__(doc, "description", desc_file.read_text(encoding="utf-8"))
+
+                documents.append(doc)
+            except Exception as e:
+                logger.error(
+                    f"  Failed to load {file_path.name} as {doc_class.__name__}: {e}", exc_info=True
+                )
+
+    return documents
+
+
+def save_documents_to_directory(base_dir: Path, documents: DocumentList) -> None:
+    """Saves documents using canonical_name."""
+    for document in documents:
+        if not isinstance(document, FlowDocument):
+            continue
+
+        dir_name = document.canonical_name()
+        document_dir = base_dir / dir_name
+        document_dir.mkdir(parents=True, exist_ok=True)
+
+        file_path = document_dir / document.name
+        file_path.write_bytes(document.content)
+        logger.info(f"Saved: {dir_name}/{document.name}")
+
+        if document.description:
+            desc_file = file_path.with_name(file_path.name + Document.DESCRIPTION_EXTENSION)
+            desc_file.write_text(document.description, encoding="utf-8")
+
+
+async def run_pipeline(
+    flow_func: Callable[..., Any],
+    config: Type[FlowConfig],
+    project_name: str,
+    output_dir: Path,
+    flow_options: FlowOptions,
+    flow_name: str | None = None,
+) -> DocumentList:
+    """Execute a single pipeline flow."""
+    if flow_name is None:
+        flow_name = getattr(flow_func, "name", getattr(flow_func, "__name__", "flow"))
+
+    logger.info(f"Running Flow: {flow_name}")
+
+    input_documents = load_documents_from_directory(output_dir, config.INPUT_DOCUMENT_TYPES)
+
+    if not config.has_input_documents(input_documents):
+        raise RuntimeError(f"Missing input documents for flow {flow_name}")
+
+    result_documents = await flow_func(project_name, input_documents, flow_options)
+
+    config.validate_output_documents(result_documents)
+
+    save_documents_to_directory(output_dir, result_documents)
+
+    logger.info(f"Completed Flow: {flow_name}")
+
+    return result_documents
+
+
+async def run_pipelines(
+    project_name: str,
+    output_dir: Path,
+    flows: FlowSequence,
+    flow_configs: ConfigSequence,
+    flow_options: FlowOptions,
+    start_step: int = 1,
+    end_step: int | None = None,
+) -> None:
+    """Executes multiple pipeline flows sequentially."""
+    if len(flows) != len(flow_configs):
+        raise ValueError("The number of flows and flow configs must match.")
+
+    num_steps = len(flows)
+    start_index = start_step - 1
+    end_index = (end_step if end_step is not None else num_steps) - 1
+
+    if (
+        not (0 <= start_index < num_steps)
+        or not (0 <= end_index < num_steps)
+        or start_index > end_index
+    ):
+        raise ValueError("Invalid start/end steps.")
+
+    logger.info(f"Starting pipeline '{project_name}' (Steps {start_step} to {end_index + 1})")
+
+    for i in range(start_index, end_index + 1):
+        flow_func = flows[i]
+        config = flow_configs[i]
+        flow_name = getattr(flow_func, "name", getattr(flow_func, "__name__", f"flow_{i + 1}"))
+
+        logger.info(f"--- [Step {i + 1}/{num_steps}] Running Flow: {flow_name} ---")
+
+        try:
+            await run_pipeline(
+                flow_func=flow_func,
+                config=config,
+                project_name=project_name,
+                output_dir=output_dir,
+                flow_options=flow_options,
+                flow_name=f"[Step {i + 1}/{num_steps}] {flow_name}",
+            )
+
+        except Exception as e:
+            logger.error(
+                f"--- [Step {i + 1}/{num_steps}] Flow {flow_name} Failed: {e} ---", exc_info=True
+            )
+            raise