pyoco 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

pyoco/dsl/syntax.py

@@ -1,122 +1,268 @@
-from typing import Callable, Union, List, Tuple
-from ..core.models import Task, Flow
-from ..core import engine
+from __future__ import annotations
 
-# Global context
-_active_flow: Flow = None
+from dataclasses import dataclass
+from typing import Callable, Iterable, List, Sequence, Tuple, Union
+
+from ..core.models import Task
+from .expressions import Expression, ensure_expression
+from .nodes import (
+    CaseNode,
+    DSLNode,
+    ForEachNode,
+    RepeatNode,
+    SubFlowNode,
+    SwitchNode,
+    TaskNode,
+    UntilNode,
+    DEFAULT_CASE_VALUE,
+)
+RESERVED_CTX_KEYS = {"params", "results", "scratch", "loop", "loops", "env", "artifacts"}
+
+
+class FlowFragment:
+    """
+    Represents a fragment of a flow (sequence of DSL nodes). Every DSL
+    operator returns a FlowFragment so sub-flows can be composed before
+    being attached to a Flow.
+    """
+
+    __slots__ = ("_nodes",)
+
+    def __init__(self, nodes: Sequence[DSLNode]):
+        if not isinstance(nodes, (list, tuple)):
+            raise TypeError("FlowFragment expects a list/tuple of nodes.")
+        self._nodes: Tuple[DSLNode, ...] = tuple(nodes)
+
+    # Sequence composition -------------------------------------------------
+    def __rshift__(self, other: Union["FlowFragment", "TaskWrapper", Task]) -> "FlowFragment":
+        right = ensure_fragment(other)
+        self._link_to(right)
+        return FlowFragment(self._nodes + right._nodes)
+
+    # Loop support ---------------------------------------------------------
+    def __getitem__(self, selector: Union[int, str, Expression]) -> "FlowFragment":
+        """
+        Implements [] operator for repeat / for-each loops.
+        """
+
+        body = self.to_subflow()
+        if isinstance(selector, int):
+            if selector < 0:
+                raise ValueError("Repeat count must be non-negative.")
+            node = RepeatNode(body=body, count=selector)
+            return FlowFragment([node])
+
+        if isinstance(selector, Expression):
+            node = RepeatNode(body=body, count=selector)
+            return FlowFragment([node])
+
+        if isinstance(selector, str):
+            source, alias = parse_foreach_selector(selector)
+            node = ForEachNode(body=body, source=ensure_expression(source), alias=alias)
+            return FlowFragment([node])
+
+        raise TypeError(f"Unsupported loop selector: {selector!r}")
+
+    def __mod__(self, value: Union[str, Expression, Tuple[Union[str, Expression], int]]) -> "FlowFragment":
+        """
+        Implements the % operator for until loops.
+        """
+
+        max_iter = None
+        expr_value: Union[str, Expression]
+
+        if isinstance(value, tuple):
+            if len(value) != 2:
+                raise ValueError("Until tuple selector must be (expression, max_iter).")
+            expr_value, max_iter = value
+        else:
+            expr_value = value
+
+        node = UntilNode(
+            body=self.to_subflow(),
+            condition=ensure_expression(expr_value),
+            max_iter=max_iter,
+        )
+        return FlowFragment([node])
+
+    # Switch/case ----------------------------------------------------------
+    def __rrshift__(self, other: Union[str, int, float, bool]) -> CaseNode:
+        """
+        Enables `"X" >> fragment` syntax by reversing the operands.
+        """
+
+        if isinstance(other, str) and other == "*":
+            value = DEFAULT_CASE_VALUE
+        else:
+            value = other
+        return CaseNode(value=value, target=self.to_subflow())
+
+    # Helpers --------------------------------------------------------------
+    def to_subflow(self) -> SubFlowNode:
+        return SubFlowNode(list(self._nodes))
+
+    def task_nodes(self) -> List[Task]:
+        tasks: List[Task] = []
+        for node in self._nodes:
+            tasks.extend(_collect_tasks(node))
+        return tasks
+
+    def _first_task(self) -> Task | None:
+        for node in self._nodes:
+            tasks = _collect_tasks(node)
+            if tasks:
+                return tasks[0]
+        return None
+
+    def _last_task(self) -> Task | None:
+        for node in reversed(self._nodes):
+            tasks = _collect_tasks(node)
+            if tasks:
+                return tasks[-1]
+        return None
+
+    def _link_to(self, other: "FlowFragment"):
+        left_task = self._last_task()
+        right_task = other._first_task()
+        if left_task and right_task and left_task is not right_task:
+            right_task.dependencies.add(left_task)
+            left_task.dependents.add(right_task)
+
+    def has_control_flow(self) -> bool:
+        return any(not isinstance(node, TaskNode) for node in self._nodes)
+
+
+class TaskWrapper(FlowFragment):
+    """
+    Wraps a Task to handle DSL operators, while exposing the underlying
+    task for legacy access (e.g., `task.task.inputs = ...`).
+    """
+
+    __slots__ = ("task",)
+
+    def __init__(self, task: Task):
+        self.task = task
+        super().__init__([TaskNode(task)])
+
+    def __call__(self, *args, **kwargs) -> "TaskWrapper":
+        return self
+
+    def __and__(self, other):
+        return Parallel([self, other])
+
+    def __or__(self, other):
+        return Branch([self, other])
 
-def task(func: Callable) -> Task:
-    t = Task(func=func, name=func.__name__)
-    return t
 
 class Branch(list):
-    """Represents a branch of tasks (OR-split/join logic placeholder)."""
+    """Represents `A | B` OR-branches (legacy)."""
+
     def __rshift__(self, other):
-        # (A | B) >> C
-        # C depends on A and B.
-        # AND C.trigger_policy = "ANY"
-        
-        targets = []
-        if hasattr(other, 'task'):
-            targets = [other.task]
-        elif isinstance(other, (list, tuple)):
-             for item in other:
-                if hasattr(item, 'task'):
-                    targets.append(item.task)
-        
+        targets = _collect_target_tasks(other)
         for target in targets:
             target.trigger_policy = "ANY"
             for source in self:
-                if hasattr(source, 'task'):
+                if hasattr(source, "task"):
                     target.dependencies.add(source.task)
                     source.task.dependents.add(target)
-        
         return other
 
+
 class Parallel(list):
-    """Represents a parallel group of tasks (AND-split/join)."""
+    """Represents `A & B` parallel branches (legacy)."""
+
     def __rshift__(self, other):
-        # (A & B) >> C
-        # C depends on A AND B.
-        
-        targets = []
-        if hasattr(other, 'task'):
-            targets = [other.task]
-        elif isinstance(other, (list, tuple)):
-             for item in other:
-                if hasattr(item, 'task'):
-                    targets.append(item.task)
-        
+        targets = _collect_target_tasks(other)
         for target in targets:
             for source in self:
-                if hasattr(source, 'task'):
+                if hasattr(source, "task"):
                     target.dependencies.add(source.task)
                     source.task.dependents.add(target)
-        
         return other
 
-class TaskWrapper:
-    """
-    Wraps a Task to handle DSL operators and registration.
-    """
-    def __init__(self, task: Task):
-        self.task = task
-    
-    def __call__(self, *args, **kwargs):
-        # In this new spec, calling a task might not be strictly necessary for registration
-        # if we assume tasks are added to flow explicitly or via >>
-        # But let's keep the pattern: calling it returns a wrapper that can be chained
-        # We might need to store args/kwargs if we want to support them
-        return self
 
-    def __rshift__(self, other):
-        # self >> other
-        if isinstance(other, TaskWrapper):
-            other.task.dependencies.add(self.task)
-            self.task.dependents.add(other.task)
-            return other
-        elif isinstance(other, (list, tuple)):
-            # self >> (A & B) or self >> (A | B)
-            # If it's a Branch (from |), does it imply something different?
-            # Spec says: "Update Flow to handle Branch >> Task (set trigger_policy=ANY)"
-            # But here we are doing Task >> Branch.
-            # Task >> (A | B) means Task triggers both A and B?
-            # Usually >> means "follows".
-            # A >> (B | C) -> A triggers B and C?
-            # Or does it mean B and C depend on A? Yes.
-            # The difference between & and | is usually how they JOIN later, or how they are triggered?
-            # In Airflow, >> [A, B] means A and B depend on upstream.
-            # If we have (A | B) >> C, then C depends on A OR B.
-            # So if 'other' is a Branch, we just add dependencies as usual.
-            # The "OR" logic is relevant when 'other' connects to downstream.
-            
-            for item in other:
-                if isinstance(item, TaskWrapper):
-                    item.task.dependencies.add(self.task)
-                    self.task.dependents.add(item.task)
-            return other
-        return other
+def switch(expression: Union[str, Expression]) -> "SwitchBuilder":
+    return SwitchBuilder(expression=ensure_expression(expression))
 
-    def __and__(self, other):
-        # self & other (Parallel)
-        return Parallel([self, other])
 
-    def __or__(self, other):
-        # self | other (Branch)
-        # Return a Branch object containing both
-        return Branch([self, other])
+@dataclass
+class SwitchBuilder:
+    expression: Expression
+
+    def __getitem__(self, cases: Union[CaseNode, Sequence[CaseNode]]) -> FlowFragment:
+        if isinstance(cases, CaseNode):
+            case_list = [cases]
+        elif isinstance(cases, Sequence):
+            case_list = list(cases)
+        else:
+            raise TypeError("switch()[...] expects CaseNode(s)")
+
+        if not case_list:
+            raise ValueError("switch() requires at least one case.")
+        return FlowFragment([SwitchNode(expression=self.expression, cases=case_list)])
+
+
+# Helper utilities ---------------------------------------------------------
+def ensure_fragment(value: Union[FlowFragment, TaskWrapper, Task]) -> FlowFragment:
+    if isinstance(value, FlowFragment):
+        return value
+    if isinstance(value, TaskWrapper):
+        return value
+    if hasattr(value, "task"):
+        return FlowFragment([TaskNode(value.task)])
+    if isinstance(value, Task):
+        return TaskWrapper(value)
+    raise TypeError(f"Cannot treat {value!r} as a flow fragment.")
+
+
+def parse_foreach_selector(selector: str) -> Tuple[str, Union[str, None]]:
+    token = selector.strip()
+    alias = None
+    if " as " in token:
+        expr, alias = token.split(" as ", 1)
+        token = expr.strip()
+        alias = alias.strip()
+        if not alias or not alias.isidentifier() or alias in RESERVED_CTX_KEYS:
+            raise ValueError(f"Invalid foreach alias '{alias}'.")
+
+    return token, alias
+
+
+def _collect_tasks(obj) -> List[Task]:
+    if isinstance(obj, TaskNode):
+        return [obj.task]
+    if isinstance(obj, SubFlowNode):
+        tasks: List[Task] = []
+        for step in obj.steps:
+            tasks.extend(_collect_tasks(step))
+        return tasks
+    if isinstance(obj, RepeatNode):
+        return _collect_tasks(obj.body)
+    if isinstance(obj, ForEachNode):
+        return _collect_tasks(obj.body)
+    if isinstance(obj, UntilNode):
+        return _collect_tasks(obj.body)
+    if isinstance(obj, SwitchNode):
+        tasks: List[Task] = []
+        for case in obj.cases:
+            tasks.extend(_collect_tasks(case.target))
+        return tasks
+    return []
+
+
+def _collect_target_tasks(other) -> List[Task]:
+    targets = []
+    if hasattr(other, "task"):
+        targets = [other.task]
+    elif isinstance(other, (list, tuple)):
+        for item in other:
+            if hasattr(item, "task"):
+                targets.append(item.task)
+    return targets
 
-# We need to adapt the DSL to match the spec:
-# @task
-# def A(ctx, x:int)->int: ...
-# flow = Flow() >> A >> (B & C)
 
-# So A, B, C must be usable in the expression.
-# The @task decorator should return something that supports >>, &, |
+def task(func: Callable) -> TaskWrapper:
+    return TaskWrapper(Task(func=func, name=func.__name__))
 
-def task_decorator(func: Callable):
-    t = Task(func=func, name=func.__name__)
-    return TaskWrapper(t)
 
-# Re-export as task
-task = task_decorator
+__all__ = ["task", "FlowFragment", "switch", "TaskWrapper", "Branch", "Parallel"]

pyoco/dsl/validator.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List
+
+from ..core.models import Flow
+from .nodes import (
+    CaseNode,
+    ForEachNode,
+    RepeatNode,
+    SubFlowNode,
+    SwitchNode,
+    TaskNode,
+    UntilNode,
+    DSLNode,
+    DEFAULT_CASE_VALUE,
+)
+
+
+@dataclass
+class ValidationReport:
+    warnings: List[str] = field(default_factory=list)
+    errors: List[str] = field(default_factory=list)
+
+    @property
+    def status(self) -> str:
+        if self.errors:
+            return "error"
+        if self.warnings:
+            return "warning"
+        return "ok"
+
+    def to_dict(self) -> dict:
+        return {
+            "status": self.status,
+            "warnings": list(self.warnings),
+            "errors": list(self.errors),
+        }
+
+
+class FlowValidator:
+    """
+    Traverses a Flow's SubFlow definition and produces warnings/errors for
+    problematic control-flow constructs (unbounded loops, duplicate cases, etc.).
+    """
+
+    def __init__(self, flow: Flow):
+        self.flow = flow
+        self.report = ValidationReport()
+
+    def validate(self) -> ValidationReport:
+        program = self.flow.build_program()
+        self._visit_subflow(program, "flow")
+        return self.report
+
+    # Traversal helpers --------------------------------------------------
+    def _visit_subflow(self, subflow: SubFlowNode, path: str):
+        for idx, node in enumerate(subflow.steps):
+            self._visit_node(node, f"{path}.step[{idx}]")
+
+    def _visit_node(self, node: DSLNode, path: str):
+        if isinstance(node, TaskNode):
+            return
+        if isinstance(node, RepeatNode):
+            self._visit_subflow(node.body, f"{path}.repeat")
+        elif isinstance(node, ForEachNode):
+            self._visit_subflow(node.body, f"{path}.foreach")
+        elif isinstance(node, UntilNode):
+            self._validate_until(node, path)
+            self._visit_subflow(node.body, f"{path}.until")
+        elif isinstance(node, SwitchNode):
+            self._validate_switch(node, path)
+        elif isinstance(node, SubFlowNode):
+            self._visit_subflow(node, path)
+        else:
+            self.report.errors.append(f"{path}: Unknown node type {type(node).__name__}")
+
+    # Validators ---------------------------------------------------------
+    def _validate_until(self, node: UntilNode, path: str):
+        if node.max_iter is None:
+            self.report.warnings.append(f"{path}: Until loop missing max_iter (defaults to 1000).")
+
+    def _validate_switch(self, node: SwitchNode, path: str):
+        seen_values = set()
+        default_count = 0
+        for idx, case in enumerate(node.cases):
+            case_path = f"{path}.case[{idx}]"
+            if case.value == DEFAULT_CASE_VALUE:
+                default_count += 1
+                if default_count > 1:
+                    self.report.errors.append(f"{case_path}: Multiple default (*) cases are not allowed.")
+            else:
+                try:
+                    key = case.value
+                    if key in seen_values:
+                        self.report.errors.append(f"{case_path}: Duplicate switch value '{case.value}'.")
+                    else:
+                        seen_values.add(key)
+                except TypeError:
+                    self.report.errors.append(f"{case_path}: Unhashable switch value '{case.value}'.")
+            self._visit_subflow(case.target, f"{case_path}.target")
+
+        if default_count == 0:
+            self.report.warnings.append(f"{path}: Switch has no default (*) case.")

pyoco/server/api.py

@@ -1,37 +1,43 @@
-from fastapi import FastAPI, HTTPException
-from typing import List, Optional
+from fastapi import FastAPI, HTTPException, Query
+from fastapi.responses import PlainTextResponse
+from typing import List, Optional, Dict, Any
 from .store import StateStore
 from .models import (
-    RunSubmitRequest, RunResponse, 
+    RunSubmitRequest, RunResponse,
     WorkerPollRequest, WorkerPollResponse,
     WorkerHeartbeatRequest, WorkerHeartbeatResponse
 )
-from ..core.models import RunContext, RunStatus
+from ..core.models import RunStatus
+from .metrics import metrics, metrics_content_type
 
 app = FastAPI(title="Pyoco Kanban Server")
 store = StateStore()
 
 @app.post("/runs", response_model=RunResponse)
-def submit_run(req: RunSubmitRequest):
+async def submit_run(req: RunSubmitRequest):
     run_ctx = store.create_run(req.flow_name, req.params)
     return RunResponse(run_id=run_ctx.run_id, status=run_ctx.status)
 
-@app.get("/runs", response_model=List[RunContext])
-def list_runs(status: Optional[RunStatus] = None):
-    runs = store.list_runs()
-    if status:
-        runs = [r for r in runs if r.status == status]
-    return runs
+@app.get("/runs")
+async def list_runs(
+    status: Optional[str] = None,
+    flow: Optional[str] = None,
+    limit: Optional[int] = Query(default=None, ge=1, le=200),
+):
+    status_enum = _parse_status(status)
+    limit_value = limit if isinstance(limit, int) else None
+    runs = store.list_runs(status=status_enum, flow=flow, limit=limit_value)
+    return [store.export_run(r) for r in runs]
 
-@app.get("/runs/{run_id}", response_model=RunContext)
-def get_run(run_id: str):
+@app.get("/runs/{run_id}")
+async def get_run(run_id: str):
     run = store.get_run(run_id)
     if not run:
         raise HTTPException(status_code=404, detail="Run not found")
-    return run
+    return store.export_run(run)
 
 @app.post("/runs/{run_id}/cancel")
-def cancel_run(run_id: str):
+async def cancel_run(run_id: str):
     run = store.get_run(run_id)
     if not run:
         raise HTTPException(status_code=404, detail="Run not found")
@@ -39,7 +45,7 @@ def cancel_run(run_id: str):
     return {"status": "CANCELLING"}
 
 @app.post("/workers/poll", response_model=WorkerPollResponse)
-def poll_work(req: WorkerPollRequest):
+async def poll_work(req: WorkerPollRequest):
     # In v0.3.0, we ignore worker_id and tags for simplicity
     run = store.dequeue()
     if run:
@@ -58,14 +64,49 @@ def poll_work(req: WorkerPollRequest):
     return WorkerPollResponse()
 
 @app.post("/runs/{run_id}/heartbeat", response_model=WorkerHeartbeatResponse)
-def heartbeat(run_id: str, req: WorkerHeartbeatRequest):
+async def heartbeat(run_id: str, req: WorkerHeartbeatRequest):
     run = store.get_run(run_id)
     if not run:
         raise HTTPException(status_code=404, detail="Run not found")
         
-    store.update_run(run_id, status=req.run_status, task_states=req.task_states)
+    store.update_run(
+        run_id,
+        status=req.run_status,
+        task_states=req.task_states,
+        task_records=req.task_records,
+        logs=req.logs
+    )
     
     # Check if cancellation was requested
     cancel_requested = (run.status == RunStatus.CANCELLING)
     
     return WorkerHeartbeatResponse(cancel_requested=cancel_requested)
+
+@app.get("/runs/{run_id}/logs")
+async def get_logs(run_id: str, task: Optional[str] = None, tail: Optional[int] = None):
+    run = store.get_run(run_id)
+    if not run:
+        raise HTTPException(status_code=404, detail="Run not found")
+    logs = run.logs
+    if task:
+        logs = [entry for entry in logs if entry["task"] == task]
+    if tail:
+        logs = logs[-tail:]
+    return {"run_status": run.status, "logs": logs}
+
+
+@app.get("/metrics")
+async def prometheus_metrics():
+    payload = metrics.render_latest()
+    return PlainTextResponse(payload, media_type=metrics_content_type())
+
+
+def _parse_status(value: Optional[str]) -> Optional[RunStatus]:
+    if not value:
+        return None
+    if isinstance(value, RunStatus):
+        return value
+    try:
+        return RunStatus(value)
+    except ValueError:
+        raise HTTPException(status_code=400, detail=f"Invalid status '{value}'")

pyoco/server/metrics.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from prometheus_client import (
+    CollectorRegistry,
+    CONTENT_TYPE_LATEST,
+    Counter,
+    Gauge,
+    Histogram,
+    generate_latest,
+)
+
+from ..core.models import RunStatus
+
+
+_DEFAULT_BUCKETS = (
+    0.05,
+    0.1,
+    0.25,
+    0.5,
+    1.0,
+    2.5,
+    5.0,
+    10.0,
+    30.0,
+    60.0,
+)
+
+
+class MetricsSink:
+    """
+    Small wrapper that owns a CollectorRegistry so tests can reset easily.
+    """
+
+    def __init__(self) -> None:
+        self.registry = CollectorRegistry()
+        self._init_metrics()
+
+    def _init_metrics(self) -> None:
+        self.runs_total = Counter(
+            "pyoco_runs_total",
+            "Total runs observed by status transitions.",
+            ["status"],
+            registry=self.registry,
+        )
+        self.runs_in_progress = Gauge(
+            "pyoco_runs_in_progress",
+            "Number of runs currently executing (RUNNING).",
+            registry=self.registry,
+        )
+        self.task_duration = Histogram(
+            "pyoco_task_duration_seconds",
+            "Observed task durations.",
+            ["task"],
+            buckets=_DEFAULT_BUCKETS,
+            registry=self.registry,
+        )
+        self.run_duration = Histogram(
+            "pyoco_run_duration_seconds",
+            "Observed end-to-end run durations.",
+            ["flow"],
+            buckets=_DEFAULT_BUCKETS,
+            registry=self.registry,
+        )
+
+    def reset(self) -> None:
+        self.__init__()
+
+    def record_status_transition(
+        self,
+        previous: Optional[RunStatus],
+        new_status: RunStatus,
+    ) -> None:
+        status_value = new_status.value if hasattr(new_status, "value") else str(new_status)
+        self.runs_total.labels(status=status_value).inc()
+
+        prev_value = previous.value if hasattr(previous, "value") else previous
+        if status_value == RunStatus.RUNNING.value:
+            if prev_value != RunStatus.RUNNING.value:
+                self.runs_in_progress.inc()
+        elif prev_value == RunStatus.RUNNING.value:
+            self.runs_in_progress.dec()
+
+    def record_task_duration(self, task_name: str, duration_ms: Optional[float]) -> None:
+        if duration_ms is None:
+            return
+        if duration_ms < 0:
+            return
+        self.task_duration.labels(task=task_name).observe(duration_ms / 1000.0)
+
+    def record_run_duration(
+        self,
+        flow_name: str,
+        start_time: Optional[float],
+        end_time: Optional[float],
+    ) -> None:
+        if start_time is None or end_time is None:
+            return
+        duration = end_time - start_time
+        if duration < 0:
+            return
+        self.run_duration.labels(flow=flow_name).observe(duration)
+
+    def render_latest(self) -> bytes:
+        return generate_latest(self.registry)
+
+
+metrics = MetricsSink()
+
+
+def metrics_content_type() -> str:
+    return CONTENT_TYPE_LATEST