pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/runners/prefect_pipeline_flow.py

@@ -0,0 +1,904 @@
+"""Prefect pipeline flow — the generic @flow that interprets DAG parameters.
+
+This is the ONLY Prefect flow for the new pipeline orchestration system.
+The PrefectTranslator submits flow runs that execute this flow. It reads
+the serialized DAG from flow parameters, creates @task invocations for
+each step, handles fan-out/fan-in, and dispatches containers via the
+appropriate compute backend.
+
+Entrypoint: pulse_engine.runners.prefect_pipeline_flow:pipeline_flow
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from typing import Any
+
+import httpx
+
+from pulse_engine.secrets import fetch_secret
+
+logger = logging.getLogger(__name__)
+
+# Prefect imports are deferred so that pure helper functions
+# (_build_execution_order, _resolve_module_env)
+# can be tested without installing Prefect.
+try:
+    from prefect import flow as _prefect_flow
+    from prefect import task as _prefect_task
+    from prefect.futures import PrefectFuture  # noqa: F401
+except ImportError:  # pragma: no cover
+    _prefect_flow = None
+    _prefect_task = None
+
+
+# ---------------------------------------------------------------------------
+# Infrastructure provisioner (singleton per process)
+# ---------------------------------------------------------------------------
+
+_provisioner = None
+
+
+def _get_provisioner() -> Any:
+    """Lazy-load the InfraProvisioner from engine settings.
+
+    Returns None if pipeline infrastructure settings are not configured
+    (e.g. local dev without AWS).
+    """
+    global _provisioner
+    if _provisioner is not None:
+        return _provisioner
+
+    try:
+        from pulse_engine.config import get_settings
+        from pulse_engine.deployment.infra_provisioner import InfraProvisioner
+
+        settings = get_settings()
+        if not settings.pipeline_cluster_name:
+            logger.warning(
+                "Pipeline infrastructure settings not configured — "
+                "dispatching will use defaults (may fail without pre-provisioned infra)"
+            )
+            return None
+
+        _provisioner = InfraProvisioner(
+            region=settings.aws_region,
+            pipeline_cluster_name=settings.pipeline_cluster_name,
+            pipeline_execution_role_arn=settings.pipeline_execution_role_arn,
+            pipeline_task_role_arn=settings.pipeline_task_role_arn,
+            pipeline_log_group=settings.pipeline_log_group,
+            pipeline_subnets=settings.pipeline_subnet_list,
+            pipeline_security_groups=settings.pipeline_sg_list,
+            lambda_execution_role_arn=settings.lambda_execution_role_arn,
+            lambda_subnets=settings.lambda_subnet_list,
+            lambda_security_groups=settings.lambda_sg_list,
+            lambda_log_group=settings.lambda_log_group,
+        )
+        return _provisioner
+    except Exception:
+        logger.warning("Failed to initialize InfraProvisioner", exc_info=True)
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Container dispatch helpers
+# ---------------------------------------------------------------------------
+
+
+async def _dispatch_ecs(
+    image: str,
+    env: dict[str, str],
+    timeout: int,
+    cluster: str | None = None,
+    subnets: list[str] | None = None,
+    security_groups: list[str] | None = None,
+    cpu_units: int = 256,
+    results_backend: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Launch an ECS Fargate task and wait for completion.
+
+    If InfraProvisioner is configured, auto-provisions the task definition
+    and uses cluster/networking from settings. Otherwise falls back to
+    hardcoded defaults for backward compatibility.
+    """
+    import boto3
+
+    provisioner = _get_provisioner()
+
+    # Resolve settings — provisioner path uses Terraform-created infra
+    try:
+        from pulse_engine.config import get_settings
+
+        settings = get_settings()
+        task_def_family = settings.pipeline_task_definition
+    except Exception:
+        task_def_family = "pulse-pipeline-step"
+
+    if provisioner:
+        # Auto-provision task definition if it doesn't exist
+        provisioner.ensure_ecs_task_definition(family=task_def_family)
+        cluster = cluster or provisioner._cluster
+        subnets = subnets or provisioner._subnets
+        security_groups = security_groups or provisioner._security_groups
+        region = provisioner._region
+    else:
+        cluster = cluster or "pulse-pipeline"
+        region = boto3.session.Session().region_name or "us-east-1"
+
+    ecs = boto3.client("ecs", region_name=region)
+
+    # containerOverrides only supports: name, command, environment, cpu, memory
+    # The container has entrypoint.py which exports run(). We invoke it via
+    # a self-contained Python script that reads env vars, calls run(), and
+    # writes the output JSON to S3 so the orchestrator can read it back.
+    step_name = env.get("STEP_NAME", "unknown")
+    pipeline_run_id_val = env.get("PIPELINE_RUN_ID", "unknown")
+
+    # Resolve S3 bucket/prefix from results_backend override or engine settings
+    try:
+        from pulse_engine.config import get_settings
+
+        default_bucket = get_settings().pulse_s3_bucket
+    except Exception:
+        default_bucket = env.get("PULSE_S3_BUCKET", "pulse-data")
+
+    rb = results_backend or {}
+    s3_bucket = rb.get("bucket") or default_bucket
+    s3_prefix = rb.get("prefix", "pipeline-runs/").rstrip("/")
+    output_s3_key = f"{s3_prefix}/{pipeline_run_id_val}/output/{step_name}.json"
+
+    # Inject the output S3 location into the container environment
+    env["OUTPUT_S3_BUCKET"] = s3_bucket
+    env["OUTPUT_S3_KEY"] = output_s3_key
+
+    # Always offload INPUTS to S3 — avoids ECS containerOverrides 8192-byte
+    # limit regardless of payload size, and gives a consistent data path.
+    inputs_raw = env.pop("INPUTS", None)
+    if inputs_raw is not None:
+        import boto3 as _boto3
+
+        inputs_s3_key = f"{s3_prefix}/{pipeline_run_id_val}/input/{step_name}.json"
+        _boto3.client("s3").put_object(
+            Bucket=s3_bucket,
+            Key=inputs_s3_key,
+            Body=inputs_raw.encode(),
+            ContentType="application/json",
+        )
+        env["INPUTS_S3_KEY"] = inputs_s3_key
+        logger.info(
+            "Stored INPUTS at s3://%s/%s (%d bytes)",
+            s3_bucket,
+            inputs_s3_key,
+            len(inputs_raw),
+        )
+
+    runner_script = (
+        "import os, json, sys, boto3; "
+        "sys.path.insert(0, '/app'); "
+        "sys.path.insert(0, '/app/src'); "
+        "from entrypoint import run; "
+        "args = json.loads(os.environ.get('ARGS', '{}')); "
+        "s3k = os.environ.get('INPUTS_S3_KEY'); "
+        "b = os.environ['OUTPUT_S3_BUCKET']; "
+        "raw = boto3.client('s3').get_object(Bucket=b,Key=s3k)['Body'].read().decode()"
+        " if s3k else os.environ.get('INPUTS', '{}'); "
+        "inputs = json.loads(raw); "
+        "result = run("
+        "job_id=os.environ.get('PIPELINE_RUN_ID',''), "
+        "args=args, "
+        "inputs=inputs, "
+        "pipeline_run_id=os.environ.get('PIPELINE_RUN_ID',''), "
+        "pulse_engine_url=os.environ.get('PULSE_ENGINE_URL',''), "
+        "pulse_api_token=os.environ.get('PULSE_API_TOKEN','')); "
+        "boto3.client('s3').put_object("
+        "Bucket=os.environ['OUTPUT_S3_BUCKET'], "
+        "Key=os.environ['OUTPUT_S3_KEY'], "
+        "Body=json.dumps(result).encode(), "
+        "ContentType='application/json'); "
+        "print(json.dumps(result))"
+    )
+    # The task definition sets entryPoint: ["python", "-c"] so that
+    # containerOverrides.command is passed as args to python -c.
+    container_override = {
+        "name": "step",
+        "environment": [{"name": k, "value": v} for k, v in env.items()],
+        "command": [runner_script],
+    }
+
+    network_config = {}
+    if subnets:
+        network_config = {
+            "awsvpcConfiguration": {
+                "subnets": subnets,
+                "securityGroups": security_groups or [],
+                "assignPublicIp": "DISABLED",
+            }
+        }
+
+    # Register a one-off task definition with the correct image
+    # since containerOverrides cannot change the image
+    if provisioner:
+        task_def_arn = provisioner.ensure_ecs_task_definition(
+            family=task_def_family, image=image, cpu=str(cpu_units)
+        )
+    else:
+        task_def_arn = task_def_family
+
+    response = ecs.run_task(
+        cluster=cluster,
+        taskDefinition=task_def_arn,
+        launchType="FARGATE",
+        overrides={
+            "containerOverrides": [container_override],
+        },
+        networkConfiguration=network_config,
+        count=1,
+    )
+
+    task_arn = response["tasks"][0]["taskArn"]
+    logger.info("ECS task started: %s", task_arn)
+
+    # Poll for completion
+    waiter = ecs.get_waiter("tasks_stopped")
+    waiter.wait(
+        cluster=cluster,
+        tasks=[task_arn],
+        WaiterConfig={"Delay": 10, "MaxAttempts": max(timeout // 10, 1)},
+    )
+
+    desc = ecs.describe_tasks(cluster=cluster, tasks=[task_arn])
+    container_info = desc["tasks"][0]["containers"][0]
+    exit_code = container_info.get("exitCode", -1)
+
+    if exit_code != 0:
+        reason = container_info.get("reason", "unknown")
+        raise RuntimeError(f"ECS task failed with exit code {exit_code}: {reason}")
+
+    # Read the module's output from S3
+    try:
+        import boto3 as _boto3
+
+        s3_client = _boto3.client("s3", region_name=region)
+        obj = s3_client.get_object(Bucket=s3_bucket, Key=output_s3_key)
+        output_data: dict[str, Any] = json.loads(obj["Body"].read().decode("utf-8"))
+        logger.info("Read step output from s3://%s/%s", s3_bucket, output_s3_key)
+        return output_data
+    except Exception:
+        logger.warning(
+            "Could not read step output from S3, returning task metadata",
+            exc_info=True,
+        )
+        return {"task_arn": task_arn, "exit_code": exit_code}
+
+
+_LAMBDA_MAX_TIMEOUT = 900  # AWS Lambda hard limit: 15 minutes
+
+
+async def _dispatch_lambda(
+    image: str,
+    env: dict[str, str],
+    timeout: int,
+    memory_mb: int = 3008,
+    results_backend: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Invoke an AWS Lambda function synchronously.
+
+    If InfraProvisioner is configured, auto-provisions the Lambda function
+    from the container image before invoking. The Lambda handler reads env
+    vars from the event payload, calls run(), and returns the result.
+    INPUTS are offloaded to S3 to stay within Lambda's 6 MB payload limit.
+    """
+    import boto3
+
+    provisioner = _get_provisioner()
+
+    # Lambda function name derived from image (convention: last segment before :tag)
+    # e.g., 123.ecr/sample-product-extractor:latest -> sample-product-extractor
+    func_name = image.split("/")[-1].split(":")[0]
+
+    # Lambda hard limit — clamp module timeout to avoid AWS rejection
+    lambda_timeout = min(timeout, _LAMBDA_MAX_TIMEOUT)
+
+    if provisioner:
+        provisioner.ensure_lambda_function(
+            func_name, image, timeout=lambda_timeout, memory_size=memory_mb
+        )
+        region = provisioner._region
+    else:
+        region = boto3.session.Session().region_name or "us-east-1"
+
+    step_name = env.get("STEP_NAME", "unknown")
+    pipeline_run_id_val = env.get("PIPELINE_RUN_ID", "unknown")
+
+    try:
+        from pulse_engine.config import get_settings
+
+        default_bucket = get_settings().pulse_s3_bucket
+    except Exception:
+        default_bucket = env.get("PULSE_S3_BUCKET", "pulse-data")
+
+    rb = results_backend or {}
+    s3_bucket = rb.get("bucket") or default_bucket
+    s3_prefix = rb.get("prefix", "pipeline-runs/").rstrip("/")
+    output_s3_key = f"{s3_prefix}/{pipeline_run_id_val}/output/{step_name}.json"
+
+    env["OUTPUT_S3_BUCKET"] = s3_bucket
+    env["OUTPUT_S3_KEY"] = output_s3_key
+
+    # Offload INPUTS to S3 — Lambda 6 MB payload limit mirrors the ECS 8 KB limit
+    inputs_raw = env.pop("INPUTS", None)
+    if inputs_raw is not None:
+        import boto3 as _boto3
+
+        inputs_s3_key = f"{s3_prefix}/{pipeline_run_id_val}/input/{step_name}.json"
+        _boto3.client("s3").put_object(
+            Bucket=s3_bucket,
+            Key=inputs_s3_key,
+            Body=inputs_raw.encode(),
+            ContentType="application/json",
+        )
+        env["INPUTS_S3_KEY"] = inputs_s3_key
+        logger.info(
+            "Stored INPUTS at s3://%s/%s (%d bytes)",
+            s3_bucket,
+            inputs_s3_key,
+            len(inputs_raw),
+        )
+
+    lambda_client = boto3.client("lambda", region_name=region)
+    payload = json.dumps(env)
+
+    response = lambda_client.invoke(
+        FunctionName=func_name,
+        InvocationType="RequestResponse",
+        Payload=payload.encode(),
+    )
+
+    status_code = response["StatusCode"]
+    response_payload = json.loads(response["Payload"].read().decode())
+
+    if status_code != 200 or "FunctionError" in response:
+        raise RuntimeError(
+            f"Lambda invocation failed ({status_code}): {response_payload}"
+        )
+
+    # Prefer the direct response payload (Lambda returns module output)
+    if isinstance(response_payload, dict) and response_payload:
+        return response_payload
+
+    # Fallback: read output from S3
+    try:
+        s3_client = boto3.client("s3", region_name=region)
+        obj = s3_client.get_object(Bucket=s3_bucket, Key=output_s3_key)
+        output_data: dict[str, Any] = json.loads(obj["Body"].read().decode("utf-8"))
+        return output_data
+    except Exception:
+        logger.warning("Could not read Lambda output from S3", exc_info=True)
+        return {}
+
+
+async def _dispatch_container(
+    compute: str,
+    image: str,
+    step_env: dict[str, str],
+    timeout: int,
+    memory_mb: int = 3008,
+    cpu_units: int = 256,
+    results_backend: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Dispatch a container to the appropriate compute backend."""
+    if compute == "ecs":
+        return await _dispatch_ecs(
+            image,
+            step_env,
+            timeout,
+            cpu_units=cpu_units,
+            results_backend=results_backend,
+        )
+    elif compute == "lambda":
+        return await _dispatch_lambda(
+            image,
+            step_env,
+            timeout,
+            memory_mb=memory_mb,
+            results_backend=results_backend,
+        )
+    else:
+        raise ValueError(f"Unknown compute backend: {compute}")
+
+
+# ---------------------------------------------------------------------------
+# Engine callback helpers
+# ---------------------------------------------------------------------------
+
+
+async def _report_step_status(
+    pulse_engine_url: str,
+    pulse_api_token: str,
+    pipeline_run_id: str,
+    step_name: str,
+    status: str,
+    output_ref: dict[str, Any] | None = None,
+    error_message: str | None = None,
+) -> None:
+    """Report step status back to the Pulse Engine."""
+    async with httpx.AsyncClient(timeout=15.0) as client:
+        response = await client.post(
+            f"{pulse_engine_url}/api/v1/pipelines/{pipeline_run_id}/steps/{step_name}/status",
+            json={
+                "status": status,
+                "output_ref": output_ref,
+                "error_message": error_message,
+            },
+            headers={"Authorization": f"Bearer {pulse_api_token}"},
+        )
+        response.raise_for_status()
+
+
+# ---------------------------------------------------------------------------
+# Prefect task for executing a single step
+# ---------------------------------------------------------------------------
+
+
+def _run_async(coro: Any) -> Any:
+    """Run an async coroutine from a synchronous context.
+
+    Handles the case where an event loop may or may not already be running.
+    """
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = None
+
+    if loop and loop.is_running():
+        # We are inside an already-running loop (e.g. Prefect's async internals).
+        # Create a new loop in a thread to avoid nested run_until_complete.
+        import concurrent.futures
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
+            return pool.submit(asyncio.run, coro).result()
+    else:
+        return asyncio.run(coro)
+
+
+def _task_decorator(**kwargs: Any) -> Any:
+    """Apply @task only when Prefect is installed."""
+    if _prefect_task is not None:
+        return _prefect_task(**kwargs)
+    return lambda fn: fn
+
+
+async def _resolve_module_env(raw_env: dict[str, Any]) -> dict[str, str]:
+    """Resolve module env — expand SecretRef dicts via Secrets Manager."""
+    result: dict[str, str] = {}
+    for key, val in raw_env.items():
+        if isinstance(val, dict) and "from_secret" in val:
+            result[key] = await fetch_secret(val["from_secret"])
+        else:
+            result[key] = str(val)
+    return result
+
+
+@_task_decorator(name="pipeline-step")  # type: ignore[misc]
+async def run_step(
+    step_config: dict[str, Any],
+    module_images: dict[str, str],
+    task_args: dict[str, Any],
+    inputs: Any,
+    global_config: dict[str, Any],
+    pipeline_run_id: str,
+    tenant_id: str,
+    pulse_engine_url: str,
+    pulse_api_token: str,
+    results_backend: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Execute a single pipeline step by dispatching a container."""
+    step_name = step_config["step"]
+    module_type = step_config["module_type"]  # → MODULE_NAME env var
+    timeout = step_config.get("timeout", 3600)
+
+    image = module_images.get(module_type)
+    if not image:
+        raise RuntimeError(f"No image registered for module type '{module_type}'")
+
+    # Merge static args: yaml config + module args (module args take precedence)
+    static_args = {**global_config, **task_args}
+
+    step_env: dict[str, str] = {
+        "MODULE_NAME": module_type,
+        "ARGS": json.dumps(static_args),
+        "PIPELINE_RUN_ID": pipeline_run_id,
+        "STEP_NAME": step_name,
+        "TENANT_ID": tenant_id,
+        "PULSE_ENGINE_URL": pulse_engine_url,
+        "PULSE_API_TOKEN": pulse_api_token,
+    }
+    if inputs is not None:
+        step_env["INPUTS"] = json.dumps(inputs)
+
+    # Resolve module env (expand SecretRef entries)
+    raw_env = step_config.get("env", {})
+    if raw_env:
+        step_env.update(await _resolve_module_env(raw_env))
+
+    resources = step_config.get("resources", {})
+    compute = resources.get("compute", "ecs")
+    memory_mb = resources.get("memory", 3008)
+    cpu_units = resources.get("cpu_units", 256)
+    try:
+        from pulse_engine.config import get_settings
+
+        compute = get_settings().pipeline_default_compute or compute
+    except Exception:
+        pass
+
+    logger.info(
+        "Dispatching step '%s' (compute=%s, module_type=%s, image=%s)",
+        step_name,
+        compute,
+        module_type,
+        image,
+    )
+
+    await _report_step_status(
+        pulse_engine_url, pulse_api_token, pipeline_run_id, step_name, "running"
+    )
+
+    try:
+        result = await _dispatch_container(
+            compute, image, step_env, timeout, memory_mb, cpu_units, results_backend
+        )
+        output_ref = (
+            result if isinstance(result, dict | list) else {"status": "completed"}
+        )
+
+        await _report_step_status(
+            pulse_engine_url,
+            pulse_api_token,
+            pipeline_run_id,
+            step_name,
+            "completed",
+            output_ref,
+        )
+        return output_ref
+
+    except Exception as exc:
+        logger.exception("Step '%s' failed", step_name)
+        await _report_step_status(
+            pulse_engine_url,
+            pulse_api_token,
+            pipeline_run_id,
+            step_name,
+            "failed",
+            error_message=str(exc),
+        )
+        raise
+
+
+def _when_step_deps(when_expr: str, step_names: set[str]) -> set[str]:
+    """Extract step names referenced by $steps.X.* in a when expression."""
+    import re
+
+    refs = re.findall(r"\$steps\.([\w-]+)\.", when_expr)
+    return {r for r in refs if r in step_names}
+
+
+def _build_deps_map(dag: list[dict[str, Any]]) -> dict[str, set[str]]:
+    """Return step_name → set of upstream dep step names."""
+    from pulse_engine.pipeline.expression import parse_expression
+
+    step_names = {s["step"] for s in dag}
+    deps_map: dict[str, set[str]] = {s["step"]: set() for s in dag}
+
+    for step in dag:
+        deps: set[str] = set()
+
+        collect_from = step.get("collect_from", [])
+        if collect_from:
+            deps.update(r for r in collect_from if r in step_names)
+
+        for_each = step.get("for_each")
+        if for_each:
+            try:
+                expr = parse_expression(for_each)
+                if expr.source == "steps" and expr.step and expr.step in step_names:
+                    deps.add(expr.step)
+            except ValueError:
+                pass
+
+        when_expr = step.get("when")
+        if when_expr and "$item." not in when_expr:
+            deps.update(_when_step_deps(when_expr, step_names))
+
+        for dep_ref in step.get("depends_on") or []:
+            dep_step = dep_ref.get("step") if isinstance(dep_ref, dict) else dep_ref
+            if dep_step is not None and dep_step in step_names:
+                deps.add(dep_step)
+
+        deps_map[step["step"]] = deps
+
+    return deps_map
+
+
+def _should_run(
+    deps: set[str],
+    step_statuses: dict[str, str],
+    trigger_rule: str | None,
+) -> bool:
+    """Evaluate trigger_rule against upstream statuses.
+
+    Returns True if step should run.
+    """
+    if not deps:
+        return True
+    rule = trigger_rule or "all_success"
+    dep_statuses = [step_statuses.get(d, "succeeded") for d in deps]
+
+    if rule == "all_success":
+        return all(s == "succeeded" for s in dep_statuses)
+    if rule == "all_done":
+        return all(s in {"succeeded", "failed", "skipped"} for s in dep_statuses)
+    if rule == "all_skipped":
+        return all(s == "skipped" for s in dep_statuses)
+    if rule == "one_succeeded":
+        return any(s == "succeeded" for s in dep_statuses)
+    if rule == "one_failed":
+        return any(s == "failed" for s in dep_statuses)
+    if rule == "none_failed":
+        return "failed" not in dep_statuses
+    if rule == "none_skipped":
+        return "skipped" not in dep_statuses
+    return True
+
+
+def _build_execution_order(dag: list[dict[str, Any]]) -> list[list[str]]:
+    """Topological sort of v2 DAG into parallel execution layers.
+
+    Dependency edges come from:
+      - collect_from: [s1, s2]  → depends on s1, s2
+      - for_each: $steps.S.*   → depends on S
+    """
+    from pulse_engine.pipeline.expression import parse_expression
+
+    in_degree: dict[str, int] = {s["step"]: 0 for s in dag}
+    dependents: dict[str, list[str]] = {s["step"]: [] for s in dag}
+    step_names = set(in_degree)
+
+    for step in dag:
+        deps: set[str] = set()
+
+        collect_from = step.get("collect_from", [])
+        if collect_from:
+            deps.update(r for r in collect_from if r in step_names)
+
+        for_each = step.get("for_each")
+        if for_each:
+            try:
+                expr = parse_expression(for_each)
+                if expr.source == "steps" and expr.step and expr.step in step_names:
+                    deps.add(expr.step)
+            except ValueError as e:
+                logger.warning(
+                    "Could not parse for_each expression '%s': %s", for_each, e
+                )
+
+        for dep_ref in step.get("depends_on") or []:
+            dep_step = dep_ref.get("step") if isinstance(dep_ref, dict) else dep_ref
+            if dep_step is not None and dep_step in step_names:
+                deps.add(dep_step)
+
+        when_expr = step.get("when")
+        if when_expr and "$item." not in when_expr:
+            deps.update(_when_step_deps(when_expr, step_names))
+
+        for dep in deps:
+            in_degree[step["step"]] += 1
+            dependents[dep].append(step["step"])
+
+    layers: list[list[str]] = []
+    queue = [name for name, deg in in_degree.items() if deg == 0]
+
+    while queue:
+        layers.append(sorted(queue))
+        next_queue: list[str] = []
+        for name in queue:
+            for dep_name in dependents[name]:
+                in_degree[dep_name] -= 1
+                if in_degree[dep_name] == 0:
+                    next_queue.append(dep_name)
+        queue = next_queue
+
+    return layers
+
+
+def _flow_decorator(**kwargs: Any) -> Any:
+    """Apply @flow only when Prefect is installed."""
+    if _prefect_flow is not None:
+        return _prefect_flow(**kwargs)
+    return lambda fn: fn
+
+
+@_flow_decorator(name="pulse-pipeline-runner", retries=0)  # type: ignore[misc]
+def _make_submitter(
+    module_images: dict[str, str],
+    global_config: dict[str, Any],
+    pipeline_run_id: str,
+    tenant_id: str,
+    pulse_engine_url: str,
+    pulse_api_token: str,
+    results_backend: dict[str, Any] | None = None,
+) -> Any:
+    """Return a _submit callable bound to the current pipeline run context."""
+
+    def _submit(
+        task_name: str, step_cfg: dict[str, Any], t_args: dict[str, Any], inp: Any
+    ) -> Any:
+        kwargs = {
+            "step_config": step_cfg,
+            "module_images": module_images,
+            "task_args": t_args,
+            "inputs": inp,
+            "global_config": global_config,
+            "pipeline_run_id": pipeline_run_id,
+            "tenant_id": tenant_id,
+            "pulse_engine_url": pulse_engine_url,
+            "pulse_api_token": pulse_api_token,
+            "results_backend": results_backend,
+        }
+        if _prefect_task is not None and hasattr(run_step, "with_options"):
+            return run_step.with_options(
+                name=task_name,
+                retries=step_cfg.get("retries", 0),
+                retry_delay_seconds=step_cfg.get("retry_delay", 0),
+                timeout_seconds=step_cfg.get("timeout", 3600),
+            ).submit(**kwargs)
+        return run_step(**kwargs)
+
+    return _submit
+
+
+def _execute_layer(
+    layer: list[str],
+    step_map: dict[str, Any],
+    step_results: dict[str, Any],
+    step_statuses: dict[str, str],
+    deps_map: dict[str, set[str]],
+    submit: Any,
+) -> None:
+    """Submit all steps in a layer, evaluating trigger_rule and when conditions."""
+    from pulse_engine.pipeline.expression import (
+        build_task_args,
+        collect_inputs,
+        evaluate_when,
+        parse_expression,
+        resolve_for_each_items,
+    )
+
+    layer_futures: list[tuple[str, Any]] = []
+    layer_fanout: list[tuple[str, list[Any]]] = []
+
+    for step_name in layer:
+        step_config = step_map[step_name]
+        module_args: dict[str, Any] = step_config.get("args", {})
+        collect_from: list[str] = step_config.get("collect_from", [])
+        trigger_rule: str | None = step_config.get("trigger_rule")
+        when_expr: str | None = step_config.get("when")
+        deps = deps_map.get(step_name, set())
+
+        # Gate 1: trigger_rule
+        if not _should_run(deps, step_statuses, trigger_rule):
+            logger.info(
+                "Step '%s' skipped — trigger_rule='%s' not satisfied",
+                step_name,
+                trigger_rule or "all_success",
+            )
+            step_statuses[step_name] = "skipped"
+            step_results[step_name] = {}
+            continue
+
+        # Gate 2: step-level when (only when expression doesn't reference $item.*)
+        if when_expr and "$item." not in when_expr:
+            if not evaluate_when(when_expr, step_results, step_statuses):
+                logger.info("Step '%s' skipped — when condition false", step_name)
+                step_statuses[step_name] = "skipped"
+                step_results[step_name] = {}
+                continue
+
+        if step_config.get("for_each"):
+            expr = parse_expression(step_config["for_each"])
+            items = resolve_for_each_items(
+                expr, module_args, step_results, collect_from or None
+            )
+
+            # Item-level when filter (fan-out only)
+            if when_expr and "$item." in when_expr:
+                items = [
+                    it
+                    for it in items
+                    if evaluate_when(when_expr, step_results, step_statuses, it)
+                ]
+
+            futures = [
+                # Give each fan-out task a unique step name so S3 input/output
+                # keys don't collide (all tasks share the same base step_config).
+                submit(
+                    f"{step_name}[{i}]",
+                    {**step_config, "step": f"{step_name}[{i}]"},
+                    *build_task_args(expr, module_args, item),
+                )
+                for i, item in enumerate(items)
+            ]
+            logger.info("Fan-out step '%s': %d tasks submitted", step_name, len(items))
+            layer_fanout.append((step_name, futures))
+        elif collect_from:
+            inputs_val = collect_inputs(collect_from, step_results)
+            layer_futures.append(
+                (step_name, submit(step_name, step_config, module_args, inputs_val))
+            )
+        else:
+            layer_futures.append(
+                (step_name, submit(step_name, step_config, module_args, None))
+            )
+
+    for step_name, fut in layer_futures:
+        try:
+            step_results[step_name] = fut.result() if hasattr(fut, "result") else fut
+            step_statuses[step_name] = "succeeded"
+        except Exception:
+            step_results[step_name] = {}
+            step_statuses[step_name] = "failed"
+            logger.exception("Step '%s' failed", step_name)
+
+    for step_name, fan_futures in layer_fanout:
+        outputs = []
+        fan_failed = False
+        for f in fan_futures:
+            try:
+                outputs.append(f.result() if hasattr(f, "result") else f)
+            except Exception:
+                fan_failed = True
+                logger.exception("Fan-out task in step '%s' failed", step_name)
+        step_results[step_name] = outputs
+        step_statuses[step_name] = "failed" if fan_failed else "succeeded"
+
+
+def pipeline_flow(
+    pipeline_run_id: str,
+    tenant_id: str,
+    dag: list[dict[str, Any]],
+    module_images: dict[str, str],
+    global_config: dict[str, Any],
+    pulse_engine_url: str,
+    pulse_api_token: str,
+    results_backend: dict[str, Any] | None = None,
+) -> None:
+    """Generic v2 pipeline flow — interprets DAG, dispatches containers per step."""
+    logger.info(
+        "Starting v2 pipeline flow: run_id=%s, steps=%d", pipeline_run_id, len(dag)
+    )
+
+    step_map = {s["step"]: s for s in dag}
+    step_results: dict[str, Any] = {}
+    step_statuses: dict[str, str] = {}
+    deps_map = _build_deps_map(dag)
+
+    submit = _make_submitter(
+        module_images,
+        global_config,
+        pipeline_run_id,
+        tenant_id,
+        pulse_engine_url,
+        pulse_api_token,
+        results_backend,
+    )
+
+    for layer in _build_execution_order(dag):
+        _execute_layer(layer, step_map, step_results, step_statuses, deps_map, submit)
+
+    failed = [s for s, st in step_statuses.items() if st == "failed"]
+    if failed:
+        raise RuntimeError(f"Pipeline {pipeline_run_id} failed — steps: {failed}")
+
+    logger.info("Pipeline flow completed: run_id=%s", pipeline_run_id)