pyoco 0.1.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

pyoco/core/models.py

@@ -1,8 +1,18 @@
-from typing import Any, Callable, Dict, List, Optional, Set, Union, ForwardRef
+from typing import Any, Callable, Dict, List, Optional, Set, Union
 from dataclasses import dataclass, field
+from enum import Enum
+import time
+import uuid
+import json
 
 @dataclass
 class Task:
+    """
+    Represents a single unit of work in the workflow.
+    
+    Designed to be lightweight and serializable.
+    Contains metadata about the task, its dependencies, and execution policies.
+    """
     func: Callable
     name: str
     dependencies: Set['Task'] = field(default_factory=set)
@@ -32,13 +42,137 @@ class Task:
     def __repr__(self):
         return f"<Task {self.name}>"
 
+class TaskState(Enum):
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    SUCCEEDED = "SUCCEEDED"
+    FAILED = "FAILED"
+    CANCELLED = "CANCELLED"
+
+class RunStatus(Enum):
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+    CANCELLING = "CANCELLING"
+    CANCELLED = "CANCELLED"
+
+@dataclass
+class TaskRecord:
+    state: TaskState = TaskState.PENDING
+    started_at: Optional[float] = None
+    ended_at: Optional[float] = None
+    duration_ms: Optional[float] = None
+    error: Optional[str] = None
+    traceback: Optional[str] = None
+    inputs: Dict[str, Any] = field(default_factory=dict)
+    output: Any = None
+    artifacts: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class RunContext:
+    """
+    Holds the state of a single workflow execution.
+    """
+    run_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    flow_name: str = "main"
+    params: Dict[str, Any] = field(default_factory=dict)
+    status: RunStatus = RunStatus.RUNNING
+    tasks: Dict[str, TaskState] = field(default_factory=dict)
+    task_records: Dict[str, TaskRecord] = field(default_factory=dict)
+    start_time: float = field(default_factory=time.time)
+    end_time: Optional[float] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    logs: List[Dict[str, Any]] = field(default_factory=list)
+    _pending_logs: List[Dict[str, Any]] = field(default_factory=list, repr=False)
+    _log_seq: int = field(default=0, repr=False)
+    log_bytes: Dict[str, int] = field(default_factory=dict)
+    metrics_recorded_tasks: Set[str] = field(default_factory=set, repr=False)
+    metrics_run_observed: bool = field(default=False, repr=False)
+    webhook_notified_status: Optional[str] = field(default=None, repr=False)
+
+    def ensure_task_record(self, task_name: str) -> TaskRecord:
+        if task_name not in self.task_records:
+            self.task_records[task_name] = TaskRecord()
+        return self.task_records[task_name]
+
+    def append_log(self, task_name: str, stream: str, payload: str):
+        if not payload:
+            return
+        entry = {
+            "seq": self._log_seq,
+            "task": task_name,
+            "stream": stream,
+            "text": payload,
+            "timestamp": time.time(),
+        }
+        self._log_seq += 1
+        self.logs.append(entry)
+        self._pending_logs.append(entry)
+
+    def drain_logs(self) -> List[Dict[str, Any]]:
+        drained = list(self._pending_logs)
+        self._pending_logs.clear()
+        return drained
+
+    def serialize_task_records(self) -> Dict[str, Any]:
+        serialized: Dict[str, Any] = {}
+        for name, record in self.task_records.items():
+            serialized[name] = {
+                "state": record.state.value if hasattr(record.state, "value") else record.state,
+                "started_at": record.started_at,
+                "ended_at": record.ended_at,
+                "duration_ms": record.duration_ms,
+                "error": record.error,
+                "traceback": record.traceback,
+                "inputs": {k: self._safe_value(v) for k, v in record.inputs.items()},
+                "output": self._safe_value(record.output),
+                "artifacts": record.artifacts,
+            }
+        return serialized
+
+    def _safe_value(self, value: Any) -> Any:
+        if isinstance(value, (str, int, float, bool)) or value is None:
+            return value
+        try:
+            json.dumps(value)
+            return value
+        except Exception:
+            return repr(value)
+
 @dataclass
 class Flow:
+    """
+    Represents a Directed Acyclic Graph (DAG) of tasks.
+    
+    Manages the collection of tasks and their dependencies.
+    Optimized for single-machine execution without complex scheduling overhead.
+    """
     name: str = "main"
     tasks: Set[Task] = field(default_factory=set)
     _tail: Set[Task] = field(default_factory=set)
+    _definition: List[Any] = field(default_factory=list, repr=False)
+    _has_control_flow: bool = False
     
     def __rshift__(self, other):
+        from ..dsl.syntax import TaskWrapper, FlowFragment, ensure_fragment
+
+        if isinstance(other, TaskWrapper):
+            fragment = other
+            self._record_fragment(fragment)
+            self._append_task(fragment.task)
+            return self
+
+        if hasattr(other, "to_subflow"):
+            fragment = other if isinstance(other, FlowFragment) else ensure_fragment(other)
+            self._record_fragment(fragment)
+            if not self._has_control_flow and not fragment.has_control_flow():
+                self._append_linear_fragment(fragment)
+            else:
+                self._has_control_flow = True
+            return self
+
         # Flow >> Task/List/Branch
         new_tasks = []
         is_branch = False
@@ -114,3 +248,39 @@ class Flow:
     def add_task(self, task: Task):
         self.tasks.add(task)
 
+    def has_control_flow(self) -> bool:
+        return self._has_control_flow
+
+    def build_program(self):
+        from ..dsl.nodes import SubFlowNode
+        return SubFlowNode(list(self._definition))
+
+    def _record_fragment(self, fragment):
+        from ..dsl.nodes import TaskNode
+        subflow = fragment.to_subflow()
+        self._definition.extend(subflow.steps)
+        for task in fragment.task_nodes():
+            self.add_task(task)
+        if any(not isinstance(step, TaskNode) for step in subflow.steps):
+            self._has_control_flow = True
+
+    def _append_linear_fragment(self, fragment):
+        subflow = fragment.to_subflow()
+        for step in subflow.steps:
+            if hasattr(step, "task"):
+                self._append_task(step.task)
+
+    def _append_task(self, task: Task):
+        self.add_task(task)
+        if self._has_control_flow:
+            self._tail = {task}
+            return
+
+        if not self._tail:
+            self._tail = {task}
+            return
+
+        for tail_task in self._tail:
+            tail_task.dependents.add(task)
+            task.dependencies.add(tail_task)
+        self._tail = {task}

pyoco/discovery/loader.py

@@ -1,9 +1,10 @@
 import importlib
 import pkgutil
 import sys
-from typing import Dict, List, Any
+from typing import Dict, List, Any, Set
 from ..core.models import Task
 from ..dsl.syntax import TaskWrapper
+from .plugins import PluginRegistry, iter_entry_points
 
 class TaskLoader:
     def __init__(self, config: Any, strict: bool = False):
@@ -11,6 +12,7 @@ class TaskLoader:
         self.strict = strict
         self.tasks: Dict[str, Task] = {}
         self._explicit_tasks: Set[str] = set()
+        self.plugin_reports: List[Dict[str, Any]] = []
 
     def load(self):
         # Load explicitly defined tasks in config FIRST (Higher priority)
@@ -31,6 +33,8 @@ class TaskLoader:
         for pattern in self.config.discovery.glob_modules:
             self._load_glob_modules(pattern)
 
+        self._load_entry_point_plugins()
+
     def _register_task(self, name: str, task: Task):
         if name in self.tasks:
             if name in self._explicit_tasks:
@@ -45,7 +49,7 @@ class TaskLoader:
                 print(f"Warning: {msg} Overwriting.")
         
         # Apply config overlay if exists
-        if name in self.config.tasks:
+        if self.config and name in self.config.tasks:
             conf = self.config.tasks[name]
             if not conf.callable:
                 if conf.inputs:
@@ -97,6 +101,29 @@ class TaskLoader:
             module_name = rel_path.replace(os.sep, ".")[:-3] # strip .py
             self._load_module(module_name)
 
+    def _load_entry_point_plugins(self):
+        entries = iter_entry_points()
+        for ep in entries:
+            info = {
+                "name": ep.name,
+                "value": ep.value,
+                "module": getattr(ep, "module", ""),
+                "tasks": [],
+            }
+            registry = PluginRegistry(self, ep.name)
+            try:
+                hook = ep.load()
+                if not callable(hook):
+                    raise TypeError("Entry point must be callable")
+                hook(registry)
+                info["tasks"] = list(registry.registered_names)
+            except Exception as exc:
+                info["error"] = str(exc)
+                if self.strict:
+                    raise
+                print(f"Warning: Plugin '{ep.name}' failed to load: {exc}")
+            self.plugin_reports.append(info)
+
     def _scan_module(self, module: Any):
         for name, obj in vars(module).items():
             if isinstance(obj, TaskWrapper):
@@ -124,7 +151,6 @@ class TaskLoader:
             # Create a Task wrapper
             t = Task(func=real_func, name=name)
             t.inputs = conf.inputs
-            t.inputs = conf.inputs
             t.outputs = conf.outputs
             self.tasks[name] = t
         except (ImportError, AttributeError) as e:

pyoco/discovery/plugins.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from importlib import metadata as importlib_metadata
+from typing import Any, Callable, Dict, List, Optional
+
+from ..core.models import Task
+from ..dsl.syntax import TaskWrapper
+
+
+def iter_entry_points(group: str = "pyoco.tasks"):
+    eps = importlib_metadata.entry_points()
+    if hasattr(eps, "select"):
+        return list(eps.select(group=group))
+    return list(eps.get(group, []))
+
+
+def list_available_plugins() -> List[Dict[str, Any]]:
+    plugins = []
+    for ep in iter_entry_points():
+        plugins.append(
+            {
+                "name": ep.name,
+                "module": getattr(ep, "module", ""),
+                "value": ep.value,
+            }
+        )
+    return plugins
+
+
+class PluginRegistry:
+    def __init__(self, loader: Any, provider_name: str) -> None:
+        self.loader = loader
+        self.provider_name = provider_name
+        self.registered_names: List[str] = []
+
+    def task(
+        self,
+        func: Optional[Callable] = None,
+        *,
+        name: Optional[str] = None,
+        inputs: Optional[Dict[str, Any]] = None,
+        outputs: Optional[List[str]] = None,
+    ):
+        if func is not None:
+            self.register_callable(
+                func,
+                name=name,
+                inputs=inputs or {},
+                outputs=outputs or [],
+            )
+            return func
+
+        def decorator(inner: Callable):
+            self.register_callable(
+                inner,
+                name=name,
+                inputs=inputs or {},
+                outputs=outputs or [],
+            )
+            return inner
+
+        return decorator
+
+    def register_callable(
+        self,
+        func: Callable,
+        *,
+        name: Optional[str] = None,
+        inputs: Optional[Dict[str, Any]] = None,
+        outputs: Optional[List[str]] = None,
+    ) -> Task:
+        task_name = name or getattr(func, "__name__", f"{self.provider_name}_task")
+        task = Task(func=func, name=task_name)
+        if inputs:
+            task.inputs.update(inputs)
+        if outputs:
+            task.outputs.extend(outputs)
+        self.loader._register_task(task_name, task)
+        self.registered_names.append(task_name)
+        return task
+
+    def add(self, obj: Any, *, name: Optional[str] = None) -> None:
+        if isinstance(obj, TaskWrapper):
+            self.loader._register_task(name or obj.task.name, obj.task)
+            self.registered_names.append(name or obj.task.name)
+        elif isinstance(obj, Task):
+            self.loader._register_task(name or obj.name, obj)
+            self.registered_names.append(name or obj.name)
+        elif callable(obj):
+            self.register_callable(obj, name=name)
+        else:
+            raise TypeError(f"Unsupported task object: {obj!r}")

pyoco/dsl/expressions.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, Mapping, Optional, Tuple, Union
+
+
+class ExpressionSyntaxError(ValueError):
+    pass
+
+
+class ExpressionEvaluationError(RuntimeError):
+    pass
+
+
+DOT_PATH_RE = re.compile(r"^[A-Za-z_][\w.]*$")
+
+
+@dataclass(frozen=True)
+class Expression:
+    source: str
+    _python: str = field(init=False, repr=False)
+    _code: object = field(init=False, repr=False)
+
+    def __post_init__(self):
+        if not isinstance(self.source, str):
+            raise TypeError("Expression source must be a string.")
+        python_expr = translate(self.source)
+        object.__setattr__(self, "_python", python_expr)
+        object.__setattr__(self, "_code", compile_safely(python_expr))
+
+    def evaluate(
+        self,
+        ctx: Optional[Mapping[str, Any]] = None,
+        env: Optional[Mapping[str, Any]] = None,
+        extras: Optional[Mapping[str, Any]] = None,
+    ) -> Any:
+        scope = build_eval_scope(ctx or {}, env or {}, extras or {})
+        try:
+            return eval(self._code, {"__builtins__": {}}, scope)  # noqa: S307
+        except Exception as exc:
+            raise ExpressionEvaluationError(
+                f"Failed to evaluate expression '{self.source}': {exc}"
+            ) from exc
+
+
+def ensure_expression(value: Union[str, Expression]) -> Expression:
+    if isinstance(value, Expression):
+        return value
+    if isinstance(value, str):
+        return Expression(value.strip())
+    raise TypeError(f"Unsupported expression value: {value!r}")
+
+
+def translate(expr: str) -> str:
+    if "_ctx" in expr or "_env" in expr:
+        raise ExpressionSyntaxError("Use $ctx/$env references instead of _ctx/_env.")
+    def replace_token(match: re.Match[str]) -> str:
+        token = match.group(0)
+        if token.startswith("$ctx."):
+            path = token[len("$ctx.") :]
+            return f"_ctx('{path}')"
+        if token.startswith("$env."):
+            path = token[len("$env.") :]
+            return f"_env('{path}')"
+        raise ExpressionSyntaxError(f"Unsupported token '{token}'")
+
+    token_re = re.compile(r"\$(?:ctx|env)\.[A-Za-z_][\w.]*")
+    translated = token_re.sub(replace_token, expr.strip())
+    if "$" in translated:
+        raise ExpressionSyntaxError("All references must use $ctx.xxx or $env.xxx form.")
+    return translated
+
+
+ALLOWED_NODES = {
+    ast.Expression,
+    ast.BoolOp,
+    ast.BinOp,
+    ast.UnaryOp,
+    ast.Compare,
+    ast.And,
+    ast.Or,
+    ast.Not,
+    ast.Eq,
+    ast.NotEq,
+    ast.Gt,
+    ast.GtE,
+    ast.Lt,
+    ast.LtE,
+    ast.In,
+    ast.NotIn,
+    ast.Add,
+    ast.Sub,
+    ast.Mult,
+    ast.Div,
+    ast.Mod,
+    ast.Pow,
+    ast.USub,
+    ast.UAdd,
+    ast.Constant,
+    ast.Name,
+    ast.Load,
+    ast.Call,
+}
+
+
+def compile_safely(python_expr: str):
+    try:
+        tree = ast.parse(python_expr, mode="eval")
+    except SyntaxError as exc:
+        raise ExpressionSyntaxError(str(exc)) from exc
+
+    for node in ast.walk(tree):
+        if not isinstance(node, tuple(ALLOWED_NODES)):
+            raise ExpressionSyntaxError(f"Unsupported syntax: {type(node).__name__}")
+        if isinstance(node, ast.Name) and node.id not in {"_ctx", "_env"}:
+            raise ExpressionSyntaxError(f"Unknown identifier '{node.id}' in expression.")
+        if isinstance(node, ast.Call):
+            if not isinstance(node.func, ast.Name) or node.func.id not in {"_ctx", "_env"}:
+                raise ExpressionSyntaxError("Only $ctx/$env references are allowed.")
+            if len(node.args) != 1 or not isinstance(node.args[0], ast.Constant):
+                raise ExpressionSyntaxError("Context references must be constant strings.")
+    return compile(tree, "<expression>", "eval")
+
+
+def build_eval_scope(
+    ctx: Mapping[str, Any], env: Mapping[str, Any], extras: Mapping[str, Any]
+) -> Dict[str, Callable[[str], Any]]:
+    scope = {
+        "_ctx": lambda path: resolve_path(ctx, path, "$ctx"),
+        "_env": lambda path: resolve_path(env, path, "$env"),
+    }
+    scope.update(extras)
+    return scope
+
+
+def resolve_path(data: Mapping[str, Any], path: str, root: str):
+    if not DOT_PATH_RE.match(path):
+        raise ExpressionEvaluationError(f"Invalid path '{path}' for {root}.")
+    parts = path.split(".")
+    current: Any = data
+    for part in parts:
+        if isinstance(current, Mapping):
+            if part not in current:
+                raise ExpressionEvaluationError(f"{root}.{path} not found.")
+            current = current[part]
+        else:
+            if not hasattr(current, part):
+                raise ExpressionEvaluationError(f"{root}.{path} not found.")
+            current = getattr(current, part)
+    return current
+
+
+__all__ = [
+    "Expression",
+    "ensure_expression",
+    "ExpressionSyntaxError",
+    "ExpressionEvaluationError",
+]

pyoco/dsl/nodes.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Sequence as TypingSequence, Union
+
+from ..core.models import Task
+from .expressions import Expression, ensure_expression
+
+
+DEFAULT_CASE_VALUE = "__default__"
+
+
+class DSLNode:
+    """Base class for all DSL AST nodes."""
+
+
+@dataclass
+class TaskNode(DSLNode):
+    task: Task
+
+
+@dataclass
+class SubFlowNode(DSLNode):
+    steps: List[DSLNode] = field(default_factory=list)
+
+
+@dataclass
+class RepeatNode(DSLNode):
+    body: SubFlowNode
+    count: Union[int, Expression]
+
+
+@dataclass
+class ForEachNode(DSLNode):
+    body: SubFlowNode
+    source: Expression
+    alias: Optional[str] = None
+
+
+@dataclass
+class UntilNode(DSLNode):
+    body: SubFlowNode
+    condition: Expression
+    max_iter: Optional[int] = None
+
+
+@dataclass
+class CaseNode(DSLNode):
+    value: Union[str, int, float, bool]
+    target: SubFlowNode
+
+
+@dataclass
+class SwitchNode(DSLNode):
+    expression: Expression
+    cases: List[CaseNode] = field(default_factory=list)