pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/pipeline/config_parser.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import yaml  # type: ignore[import-untyped]
+from pydantic import ValidationError
+
+from pulse_engine.pipeline.expression import parse_expression
+from pulse_engine.pipeline.schemas import PipelineConfig
+
+
+class PipelineConfigError(Exception):
+    """Raised when pipeline config is invalid."""
+
+    def __init__(self, errors: list[str]) -> None:
+        self.errors = errors
+        super().__init__(f"Pipeline config validation failed: {'; '.join(errors)}")
+
+
+def parse_pipeline_config(yaml_content: str) -> PipelineConfig:
+    """Parse YAML string into validated PipelineConfig."""
+    try:
+        raw = yaml.safe_load(yaml_content)
+    except yaml.YAMLError as e:
+        raise PipelineConfigError([f"YAML parse error: {e}"]) from e
+
+    if not isinstance(raw, dict):
+        raise PipelineConfigError(["YAML content must be a mapping"])
+
+    try:
+        config = PipelineConfig(**raw)
+    except ValidationError as e:
+        errors = [f"{err['loc']}: {err['msg']}" for err in e.errors()]
+        raise PipelineConfigError(errors) from e
+
+    validate_dag(config)
+    return config
+
+
+def validate_dag(config: PipelineConfig) -> None:
+    """Validate DAG structure for v2 pipelines."""
+    errors: list[str] = []
+    module_names = {m.name for m in config.modules}
+    module_map = {m.name: m for m in config.modules}
+    step_names: set[str] = set()
+
+    # Pass 1: duplicate steps + module reference check
+    for s in config.dag:
+        if s.step in step_names:
+            errors.append(f"Duplicate step: '{s.step}'")
+        step_names.add(s.step)
+
+        if s.module not in module_names:
+            errors.append(f"Step '{s.step}' references unknown module '{s.module}'")
+
+    # Pass 2: expression + collect_from validation
+    for s in config.dag:
+        # Validate collect_from references
+        if s.collect_from is not None:
+            refs = (
+                [s.collect_from]
+                if isinstance(s.collect_from, str)
+                else list(s.collect_from)
+            )
+            for ref in refs:
+                if ref not in step_names:
+                    errors.append(
+                        f"Step '{s.step}' collect_from references unknown step '{ref}'"
+                    )
+
+        # Validate for_each expression
+        if s.for_each is not None:
+            module = module_map.get(s.module)
+            try:
+                expr = parse_expression(s.for_each)
+
+                if expr.source == "args":
+                    # $args.X requires module to have arg X
+                    if module is not None and expr.field not in module.args:
+                        errors.append(
+                            f"Step '{s.step}' for_each '{s.for_each}' but "
+                            f"module has no arg '{expr.field}'"
+                        )
+                    # $args fan-out cannot be combined with collect_from
+                    if s.collect_from is not None:
+                        errors.append(
+                            f"Step '{s.step}' cannot combine '$args' fan-out "
+                            f"with collect_from"
+                        )
+
+                elif expr.source == "steps":
+                    if expr.step not in step_names:
+                        errors.append(
+                            f"Step '{s.step}' for_each references unknown step "
+                            f"'{expr.step}'"
+                        )
+
+                elif expr.source == "collect":
+                    if s.collect_from is None:
+                        errors.append(
+                            f"Step '{s.step}' uses '$collect.{expr.field}' "
+                            f"but has no collect_from"
+                        )
+
+            except ValueError as exc:
+                errors.append(f"Step '{s.step}' has invalid for_each expression: {exc}")
+
+    # Cycle detection via Kahn's algorithm
+    # Edges: collect_from deps + for_each $steps.S.* deps
+    in_degree: dict[str, int] = {s.step: 0 for s in config.dag}
+    adjacency: dict[str, list[str]] = {s.step: [] for s in config.dag}
+
+    for s in config.dag:
+        deps: set[str] = set()
+
+        if s.collect_from is not None:
+            refs = (
+                [s.collect_from]
+                if isinstance(s.collect_from, str)
+                else list(s.collect_from)
+            )
+            deps.update(r for r in refs if r in step_names)
+
+        if s.for_each is not None:
+            try:
+                expr = parse_expression(s.for_each)
+                if expr.source == "steps" and expr.step and expr.step in step_names:
+                    deps.add(expr.step)
+            except ValueError:
+                pass
+
+        for dep in deps:
+            adjacency[dep].append(s.step)
+            in_degree[s.step] += 1
+
+    queue = [name for name, deg in in_degree.items() if deg == 0]
+    visited = 0
+    while queue:
+        node = queue.pop(0)
+        visited += 1
+        for neighbor in adjacency[node]:
+            in_degree[neighbor] -= 1
+            if in_degree[neighbor] == 0:
+                queue.append(neighbor)
+
+    if visited != len(config.dag):
+        errors.append("Circular dependency detected in DAG")
+
+    if errors:
+        raise PipelineConfigError(errors)

pulse_engine/pipeline/expression.py

@@ -0,0 +1,268 @@
+"""Pure expression parser and resolver for v2 pipeline for_each expressions.
+
+No Prefect, no AWS. All functions are synchronous and unit-testable in isolation.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class ParsedExpression:
+    source: str  # "args", "steps", or "collect"
+    step: str | None  # for source="steps": the referenced step name
+    # for "args": arg key; for "steps"/"collect": optional output field
+    field: str | None
+
+
+def parse_expression(expr: str) -> ParsedExpression:
+    """Parse a for_each expression string.
+
+    Valid forms:
+      $args.FIELD
+      $steps.STEP_NAME.output
+      $steps.STEP_NAME.output.FIELD
+      $collect.FIELD
+
+    Step names may contain hyphens (e.g. $steps.batch-yt.output.batches).
+    """
+    m = re.match(r"^\$args\.(\w+)$", expr)
+    if m:
+        return ParsedExpression(source="args", step=None, field=m.group(1))
+
+    m = re.match(r"^\$steps\.([\w-]+)\.output(?:\.(\w+))?$", expr)
+    if m:
+        return ParsedExpression(source="steps", step=m.group(1), field=m.group(2))
+
+    m = re.match(r"^\$collect\.(\w+)$", expr)
+    if m:
+        return ParsedExpression(source="collect", step=None, field=m.group(1))
+
+    raise ValueError(f"Invalid for_each expression: {expr!r}")
+
+
+def resolve_for_each_items(
+    expr: ParsedExpression,
+    module_args: dict[str, Any],
+    step_results: dict[str, Any],
+    collect_from: list[str] | None,
+) -> list[Any]:
+    """Return the list of items to fan out over for the given expression.
+
+    Raises ValueError if an $args source is not a list.
+    Returns [] for missing/empty sources rather than raising.
+    """
+    if expr.source == "args":
+        assert expr.field is not None  # guaranteed by parse_expression for "args"
+        items = module_args.get(expr.field, [])
+        if not isinstance(items, list):
+            raise ValueError(
+                f"module.args['{expr.field}'] must be a list for fan-out, "
+                f"got {type(items).__name__}"
+            )
+        return items
+
+    if expr.source == "steps":
+        assert expr.step is not None  # guaranteed by parse_expression for "steps"
+        output = step_results.get(expr.step)
+        if output is None:
+            return []
+        if expr.field:
+            return list(output.get(expr.field, [])) if isinstance(output, dict) else []
+        return output if isinstance(output, list) else [output]
+
+    if expr.source == "collect":
+        field = expr.field
+        assert field is not None  # guaranteed by parse_expression for "collect"
+        sources = collect_from or []
+        collected: list[Any] = []
+        for step_name in sources:
+            result = step_results.get(step_name)
+            if result is None:
+                continue
+            if isinstance(result, list):
+                for r in result:
+                    if isinstance(r, dict):
+                        collected.extend(r.get(field, []))
+            elif isinstance(result, dict):
+                collected.extend(result.get(field, []))
+        return collected
+
+    return []
+
+
+def build_task_args(
+    expr: ParsedExpression,
+    module_args: dict[str, Any],
+    item: Any,
+) -> tuple[dict[str, Any], Any]:
+    """Build (task_args, inputs) for a single fan-out task.
+
+    task_args is always module_args unchanged — args stays static across all shards.
+    inputs is the fan-out item — the runtime value this shard processes.
+    """
+    return dict(module_args), item
+
+
+def _unwrap_item(item: Any) -> list[Any]:
+    """Unwrap a step output item into a flat list of data items.
+
+    If item is a dict with exactly ONE key and that value is a list, return
+    that list — this handles the common pattern where a module returns
+    {"videos": [...]} or {"speeches": [...]} and the caller wants the inner list.
+
+    Dicts with multiple keys are never unwrapped to avoid accidentally extracting
+    a list field (e.g. "segments") from a rich data object like a video item.
+    """
+    if isinstance(item, dict) and len(item) == 1:
+        (val,) = item.values()
+        if isinstance(val, list):
+            return val
+    return [item]
+
+
+def collect_inputs(
+    collect_from: list[str],
+    step_results: dict[str, Any],
+) -> list[Any]:
+    """Flatten outputs from multiple steps into a single list for inputs (collect_from).
+
+    Each collected item is unwrapped via _unwrap_item so that modules returning
+    {"videos": [...]} or {"speeches": [...]} contribute their inner list items
+    rather than the wrapper dict.
+    """
+    result: list[Any] = []
+    for step_name in collect_from:
+        val = step_results.get(step_name)
+        if val is None:
+            continue
+        items = val if isinstance(val, list) else [val]
+        for item in items:
+            result.extend(_unwrap_item(item))
+    return result
+
+
+# ---------------------------------------------------------------------------
+# when: expression evaluator
+# ---------------------------------------------------------------------------
+
+
+def _resolve_when_references(
+    expr: str,
+    step_results: dict[str, Any],
+    step_statuses: dict[str, str],
+    item: Any,
+) -> str:
+    """Replace $steps.* and $item.* with repr() of their resolved values."""
+
+    def _steps_sub(m: re.Match[str]) -> str:
+        step = m.group(1)
+        rest = m.group(2)
+        if rest == "status":
+            return repr(step_statuses.get(step, "unknown"))
+        if rest == "skipped":
+            return repr(step_statuses.get(step) == "skipped")
+        if rest.startswith("output"):
+            output = step_results.get(step, {})
+            field_path = rest[len("output") :].lstrip(".")
+            if field_path and isinstance(output, dict):
+                val = output.get(field_path)
+            else:
+                val = output
+            return repr(val)
+        return repr(None)
+
+    def _item_sub(m: re.Match[str]) -> str:
+        field = m.group(1)
+        if isinstance(item, dict):
+            return repr(item.get(field))
+        return repr(None)
+
+    expr = re.sub(r"\$steps\.([\w-]+)\.([\w.]+)", _steps_sub, expr)
+    expr = re.sub(r"\$item\.(\w+)", _item_sub, expr)
+    expr = re.sub(r"\bnull\b", "None", expr)
+    expr = re.sub(r"\btrue\b", "True", expr)
+    expr = re.sub(r"\bfalse\b", "False", expr)
+    return expr
+
+
+class _SafeEval(ast.NodeVisitor):
+    """Evaluate a boolean AST over constants, comparisons, and bool ops only."""
+
+    def visit(self, node: ast.AST) -> Any:
+        return super().visit(node)
+
+    def visit_Expression(self, node: ast.Expression) -> Any:
+        return self.visit(node.body)
+
+    def visit_Constant(self, node: ast.Constant) -> Any:
+        return node.value
+
+    def visit_UnaryOp(self, node: ast.UnaryOp) -> Any:
+        if isinstance(node.op, ast.Not):
+            return not self.visit(node.operand)
+        raise ValueError(f"Unsupported unary op: {type(node.op).__name__}")
+
+    def visit_BoolOp(self, node: ast.BoolOp) -> Any:
+        vals = [self.visit(v) for v in node.values]
+        if isinstance(node.op, ast.And):
+            return all(vals)
+        if isinstance(node.op, ast.Or):
+            return any(vals)
+        raise ValueError(f"Unsupported bool op: {type(node.op).__name__}")
+
+    def visit_Compare(self, node: ast.Compare) -> Any:
+        left = self.visit(node.left)
+        for op, comp in zip(node.ops, node.comparators):
+            right = self.visit(comp)
+            try:
+                if isinstance(op, ast.Eq):
+                    result = left == right
+                elif isinstance(op, ast.NotEq):
+                    result = left != right
+                elif isinstance(op, ast.Lt):
+                    result = left < right
+                elif isinstance(op, ast.LtE):
+                    result = left <= right
+                elif isinstance(op, ast.Gt):
+                    result = left > right
+                elif isinstance(op, ast.GtE):
+                    result = left >= right
+                elif isinstance(op, ast.Is):
+                    result = left is right
+                elif isinstance(op, ast.IsNot):
+                    result = left is not right
+                else:
+                    raise ValueError(f"Unsupported compare op: {type(op).__name__}")
+            except TypeError:
+                return False
+            if not result:
+                return False
+            left = right
+        return True
+
+    def generic_visit(self, node: ast.AST) -> Any:
+        raise ValueError(f"Unsupported AST node: {type(node).__name__}")
+
+
+def evaluate_when(
+    expr: str,
+    step_results: dict[str, Any],
+    step_statuses: dict[str, str],
+    item: Any = None,
+) -> bool:
+    """Evaluate a when: expression. Returns True if the step/item should proceed.
+
+    On eval error (malformed expression, type mismatch) returns True so that
+    steps are never silently skipped due to a bad expression string.
+    """
+    try:
+        resolved = _resolve_when_references(expr, step_results, step_statuses, item)
+        tree = ast.parse(resolved, mode="eval")
+        return bool(_SafeEval().visit(tree))
+    except Exception:
+        return True

pulse_engine/pipeline/models.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import JSON, DateTime, Index, String, Text, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from pulse_engine.database import Base
+
+
+class ModuleRegistryModel(Base):
+    __tablename__ = "module_registry"
+
+    id: Mapped[str] = mapped_column(
+        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+    )
+    tenant_id: Mapped[str] = mapped_column(String(255), nullable=False)
+    product: Mapped[str] = mapped_column(String(255), nullable=False)
+    module_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    image: Mapped[str] = mapped_column(Text, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
+    )
+
+    __table_args__ = (
+        Index(
+            "uq_module_registry_product_module",
+            "tenant_id",
+            "product",
+            "module_name",
+            unique=True,
+        ),
+    )
+
+
+class PipelineRunModel(Base):
+    __tablename__ = "pipeline_runs"
+
+    id: Mapped[str] = mapped_column(
+        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+    )
+    tenant_id: Mapped[str] = mapped_column(String(255), nullable=False)
+    product: Mapped[str] = mapped_column(String(255), nullable=False)
+    orchestrator: Mapped[str] = mapped_column(String(50), nullable=False)
+    orchestrator_run_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    config_snapshot: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+    global_config: Mapped[dict[str, Any]] = mapped_column(
+        JSON, nullable=False, default=dict
+    )
+    status: Mapped[str] = mapped_column(String(50), nullable=False, default="pending")
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
+    )
+
+    __table_args__ = (
+        Index("ix_pipeline_runs_tenant_status", "tenant_id", "status"),
+        Index("ix_pipeline_runs_tenant_product", "tenant_id", "product"),
+    )
+
+
+class PipelineStepRunModel(Base):
+    __tablename__ = "pipeline_step_runs"
+
+    id: Mapped[str] = mapped_column(
+        String(36), primary_key=True, default=lambda: str(uuid.uuid4())
+    )
+    pipeline_run_id: Mapped[str] = mapped_column(String(36), nullable=False)
+    tenant_id: Mapped[str] = mapped_column(String(255), nullable=False)
+    step_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    module_type: Mapped[str] = mapped_column(String(255), nullable=False, default="")
+    status: Mapped[str] = mapped_column(String(50), nullable=False, default="pending")
+    started_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
+    completed_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+    output_ref: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
+    )
+
+    __table_args__ = (
+        Index("ix_pipeline_step_runs_run_id", "pipeline_run_id"),
+        Index("ix_pipeline_step_runs_tenant_status", "tenant_id", "status"),
+    )