orchestration-engine 0.9.0__py3-none-any.whl

orchestration_engine/__init__.py

@@ -0,0 +1,54 @@
+import importlib.metadata
+import logging
+
+from .builder import WorkflowBuilder, workflow_to_dict
+from .errors import (
+    OrchestrationError,
+    PersistenceError,
+    WorkflowDefinitionError,
+    WorkflowValidationError,
+)
+from .loader import (
+    load_workflow_from_json,
+    load_workflow_from_yaml,
+    run_workflow_from_file,
+)
+from .models import Hooks, RetryPolicy, Task, TaskResult, TaskState, Workflow, WorkflowStatus
+from .persistence import get_connection, get_run_history, get_workflow_history, list_runs
+from .runner import resume_run, run_workflow
+from .status import get_run_status, get_workflow_status
+
+logging.getLogger("orchestration_engine").addHandler(logging.NullHandler())
+
+try:
+    __version__ = importlib.metadata.version("orchestration-engine")
+except importlib.metadata.PackageNotFoundError:
+    __version__ = "0.9.0"
+
+__all__ = [
+    "__version__",
+    "Task",
+    "TaskResult",
+    "Workflow",
+    "RetryPolicy",
+    "TaskState",
+    "WorkflowStatus",
+    "Hooks",
+    "WorkflowBuilder",
+    "workflow_to_dict",
+    "run_workflow",
+    "run_workflow_from_file",
+    "resume_run",
+    "load_workflow_from_json",
+    "load_workflow_from_yaml",
+    "get_workflow_status",
+    "get_run_status",
+    "get_workflow_history",
+    "get_run_history",
+    "list_runs",
+    "get_connection",
+    "OrchestrationError",
+    "WorkflowDefinitionError",
+    "WorkflowValidationError",
+    "PersistenceError",
+]

orchestration_engine/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from orchestration_engine.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

orchestration_engine/builder.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+from typing import Any, Self
+
+from orchestration_engine.errors import WorkflowValidationError
+from orchestration_engine.models import RetryPolicy, Task, TaskState, Workflow
+from orchestration_engine.validation import get_workflow_validation_error
+
+
+class WorkflowBuilder:
+    """
+    Fluent builder for constructing and validating Workflow instances.
+    """
+
+    def __init__(
+        self,
+        workflow_id: str,
+        name: str = "",
+        description: str = "",
+        default_timeout_seconds: float | None = None,
+    ) -> None:
+        self._workflow_id = workflow_id
+        self._name = name or workflow_id
+        self._description = description
+        self._default_timeout_seconds = default_timeout_seconds
+        self._tasks: list[Task] = []
+
+    def name(self, name: str) -> Self:
+        self._name = name
+        return self
+
+    def description(self, description: str) -> Self:
+        self._description = description
+        return self
+
+    def default_timeout(self, seconds: float | None) -> Self:
+        self._default_timeout_seconds = seconds
+        return self
+
+    def task(
+        self,
+        task_id: str,
+        name: str = "",
+        description: str = "",
+        dependencies: list[str] | None = None,
+        max_attempts: int = 1,
+        delay_seconds: float = 1.0,
+        timeout_seconds: float | None = 30.0,
+        retry_policy: RetryPolicy | None = None,
+    ) -> Self:
+        if retry_policy is None:
+            policy = RetryPolicy(
+                max_attempts=max_attempts, delay_seconds=delay_seconds
+            )
+        else:
+            policy = retry_policy
+
+        effective_timeout = timeout_seconds
+        if (
+            timeout_seconds == 30.0
+            and self._default_timeout_seconds is not None
+        ):
+            effective_timeout = self._default_timeout_seconds
+
+        task = Task(
+            task_id=task_id,
+            workflow_id=self._workflow_id,
+            name=name or task_id,
+            description=description,
+            dependencies=list(dependencies or []),
+            state=TaskState.PENDING,
+            retry_policy=policy,
+            timeout_seconds=effective_timeout,
+        )
+        self._tasks.append(task)
+        return self
+
+    def build(self) -> Workflow:
+        workflow = Workflow(
+            workflow_id=self._workflow_id,
+            name=self._name,
+            description=self._description,
+            tasks=list(self._tasks),
+            default_timeout_seconds=self._default_timeout_seconds,
+        )
+        error = get_workflow_validation_error(workflow)
+        if error:
+            raise WorkflowValidationError(f"Invalid workflow: {error}")
+        return workflow
+
+
+def workflow_to_dict(workflow: Workflow) -> dict[str, Any]:
+    """
+    Serialize a Workflow instance to a JSON-compatible dictionary.
+
+    Round-trips losslessly with load_workflow_from_json.
+    """
+    return {
+        "workflow_id": workflow.workflow_id,
+        "name": workflow.name,
+        "description": workflow.description,
+        "default_timeout_seconds": workflow.default_timeout_seconds,
+        "tasks": [
+            {
+                "task_id": t.task_id,
+                "name": t.name,
+                "description": t.description,
+                "dependencies": list(t.dependencies),
+                "retry_policy": {
+                    "max_attempts": t.retry_policy.max_attempts,
+                    "delay_seconds": t.retry_policy.delay_seconds,
+                },
+                "timeout_seconds": t.timeout_seconds,
+            }
+            for t in workflow.tasks
+        ],
+    }

orchestration_engine/cli.py

@@ -0,0 +1,454 @@
+"""Command-line interface for the orchestration engine."""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import logging
+import os
+import sys
+import time
+from collections.abc import Callable
+from typing import Any, cast
+
+from orchestration_engine.errors import (
+    OrchestrationError,
+    PersistenceError,
+    WorkflowDefinitionError,
+    WorkflowValidationError,
+)
+from orchestration_engine.loader import load_workflow_from_json
+from orchestration_engine.models import Workflow, WorkflowStatus
+from orchestration_engine.persistence import (
+    get_connection,
+    get_run_history,
+    list_runs,
+)
+from orchestration_engine.runner import resume_run, run_workflow
+from orchestration_engine.status import get_run_status
+
+logger = logging.getLogger("orchestration_engine")
+
+
+def _load_runner(runner_spec: str) -> Callable[..., Any]:
+    """
+    Resolve a task runner callable from a 'module:function' string via importlib.
+
+    Note: This dynamically imports user-specified code and prepends os.getcwd()
+    to sys.path. It operates under the same trust model as executing a Python script.
+    """
+    if ":" not in runner_spec:
+        raise ValueError(
+            f"Invalid runner specification '{runner_spec}'. Expected format 'module:function'."
+        )
+    module_name, func_name = runner_spec.split(":", 1)
+    if not module_name.strip() or not func_name.strip():
+        raise ValueError(
+            f"Invalid runner specification '{runner_spec}'. Expected format 'module:function'."
+        )
+
+    cwd = os.getcwd()
+    if cwd not in sys.path:
+        sys.path.insert(0, cwd)
+
+    try:
+        mod = importlib.import_module(module_name)
+    except Exception as exc:
+        raise ImportError(f"Could not import module '{module_name}': {exc}") from exc
+
+    if not hasattr(mod, func_name):
+        raise AttributeError(f"Module '{module_name}' has no attribute '{func_name}'")
+
+    runner = getattr(mod, func_name)
+    if not callable(runner):
+        raise TypeError(f"Runner '{runner_spec}' is not callable.")
+
+    return cast(Callable[..., Any], runner)
+
+
+def _load_workflow_file(path: str) -> Workflow:
+    """Load a workflow definition from a JSON or YAML file."""
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Workflow file not found: '{path}'")
+
+    if os.path.getsize(path) > 10 * 1024 * 1024:
+        raise WorkflowDefinitionError(f"Workflow file '{path}' exceeds maximum size of 10 MB")
+
+    ext = os.path.splitext(path)[1].lower()
+    if ext in (".yaml", ".yml"):
+        try:
+            from orchestration_engine.loader import load_workflow_from_yaml
+        except ImportError as exc:
+            raise WorkflowValidationError(
+                "YAML support requires the 'yaml' extra. "
+                "Install with: pip install 'orchestration-engine[yaml]'"
+            ) from exc
+        return load_workflow_from_yaml(path, collect_errors=True)
+
+    with open(path, encoding="utf-8") as f:
+        try:
+            data = json.load(f)
+        except json.JSONDecodeError as exc:
+            raise WorkflowDefinitionError(f"Invalid JSON in '{path}': {exc}") from exc
+
+    if not isinstance(data, dict):
+        raise WorkflowDefinitionError(f"Root of workflow JSON in '{path}' must be an object")
+
+    return load_workflow_from_json(data, collect_errors=True)
+
+
+def _cmd_validate(args: argparse.Namespace) -> int:
+    try:
+        workflow = _load_workflow_file(args.workflow_file)
+    except (WorkflowValidationError, WorkflowDefinitionError, FileNotFoundError, ValueError) as exc:
+        print(f"Validation error: {exc}", file=sys.stderr)
+        return 2
+    except Exception as exc:
+        print(f"Internal error: {exc}", file=sys.stderr)
+        return 3
+
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    "workflow_id": workflow.workflow_id,
+                    "tasks_count": len(workflow.tasks),
+                    "valid": True,
+                }
+            )
+        )
+    else:
+        print(f"Workflow '{workflow.workflow_id}' is valid ({len(workflow.tasks)} tasks).")
+    return 0
+
+
+def _cmd_run(args: argparse.Namespace) -> int:
+    try:
+        workflow = _load_workflow_file(args.workflow_file)
+        runner = _load_runner(args.runner)
+        if args.max_concurrency < 1:
+            raise ValueError(f"max_concurrency must be >= 1, got {args.max_concurrency}")
+    except (
+        WorkflowValidationError,
+        WorkflowDefinitionError,
+        FileNotFoundError,
+        ValueError,
+        ImportError,
+        AttributeError,
+        TypeError,
+    ) as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 2
+    except Exception as exc:
+        print(f"Internal error: {exc}", file=sys.stderr)
+        return 3
+
+    try:
+        conn = get_connection(args.db) if args.db else get_connection(":memory:")
+    except PersistenceError as exc:
+        print(f"Database error: {exc}", file=sys.stderr)
+        return 3
+
+    try:
+        actual_run_id = run_workflow(
+            workflow,
+            task_states={},
+            time_source=time.time,
+            task_runner=runner,
+            conn=conn,
+            run_id=args.run_id,
+            max_concurrency=args.max_concurrency,
+            executor=args.executor,
+        )
+        st = get_run_status(conn, actual_run_id)
+    except (
+        WorkflowValidationError,
+        WorkflowDefinitionError,
+        OrchestrationError,
+        ValueError,
+    ) as exc:
+        print(f"Execution error: {exc}", file=sys.stderr)
+        return 2
+    except PersistenceError as exc:
+        print(f"Database error during run: {exc}", file=sys.stderr)
+        return 3
+    except Exception as exc:
+        print(f"Internal error during run: {exc}", file=sys.stderr)
+        return 3
+
+    status_str = st.value if st is not None else "unknown"
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    "run_id": actual_run_id,
+                    "workflow_id": workflow.workflow_id,
+                    "status": status_str,
+                }
+            )
+        )
+    else:
+        print(f"Workflow run '{actual_run_id}' completed with status: {status_str}")
+
+    return 0 if st == WorkflowStatus.SUCCEEDED else 1
+
+
+def _cmd_resume(args: argparse.Namespace) -> int:
+    try:
+        runner = _load_runner(args.runner)
+        if args.max_concurrency < 1:
+            raise ValueError(f"max_concurrency must be >= 1, got {args.max_concurrency}")
+    except (ValueError, ImportError, AttributeError, TypeError) as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 2
+    except Exception as exc:
+        print(f"Internal error: {exc}", file=sys.stderr)
+        return 3
+
+    try:
+        conn = get_connection(args.db)
+    except PersistenceError as exc:
+        print(f"Database error: {exc}", file=sys.stderr)
+        return 3
+
+    try:
+        st = resume_run(
+            conn=conn,
+            run_id=args.run_id,
+            task_runner=runner,
+            time_source=time.time,
+            max_concurrency=args.max_concurrency,
+            executor=args.executor,
+        )
+    except (
+        WorkflowValidationError,
+        WorkflowDefinitionError,
+        OrchestrationError,
+        ValueError,
+    ) as exc:
+        print(f"Resume error: {exc}", file=sys.stderr)
+        return 2
+    except PersistenceError as exc:
+        print(f"Database error during resume: {exc}", file=sys.stderr)
+        return 3
+    except Exception as exc:
+        print(f"Internal error during resume: {exc}", file=sys.stderr)
+        return 3
+
+    status_str = st.value
+    if args.json:
+        print(json.dumps({"run_id": args.run_id, "status": status_str}))
+    else:
+        print(f"Resumed run '{args.run_id}' completed with status: {status_str}")
+
+    return 0 if st == WorkflowStatus.SUCCEEDED else 1
+
+
+def _cmd_status(args: argparse.Namespace) -> int:
+    try:
+        conn = get_connection(args.db)
+        st = get_run_status(conn, args.run_id)
+    except PersistenceError as exc:
+        print(f"Database error: {exc}", file=sys.stderr)
+        return 3
+    except OrchestrationError as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 2
+    except Exception as exc:
+        print(f"Internal error: {exc}", file=sys.stderr)
+        return 3
+
+    if args.json:
+        print(json.dumps({"run_id": args.run_id, "status": st.value}))
+    else:
+        print(f"Run {args.run_id}: {st.value}")
+    return 0
+
+
+def _cmd_history(args: argparse.Namespace) -> int:
+    try:
+        conn = get_connection(args.db)
+        get_run_status(conn, args.run_id)
+        events = get_run_history(conn, args.run_id)
+    except PersistenceError as exc:
+        print(f"Database error: {exc}", file=sys.stderr)
+        return 3
+    except OrchestrationError as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 2
+    except Exception as exc:
+        print(f"Internal error: {exc}", file=sys.stderr)
+        return 3
+
+    if args.json:
+        output_events = [
+            {
+                "id": row[0],
+                "workflow_id": row[1],
+                "task_id": row[2],
+                "old_state": row[3],
+                "new_state": row[4],
+                "timestamp": row[5],
+                "attempt_count": row[6],
+            }
+            for row in events
+        ]
+        print(json.dumps(output_events, indent=2))
+    else:
+        if not events:
+            print(f"No history events recorded for run '{args.run_id}'.")
+        else:
+            for row in events:
+                print(f"[{row[5]:.2f}] Task '{row[2]}': {row[3]} -> {row[4]} (attempt {row[6]})")
+    return 0
+
+
+def _cmd_runs(args: argparse.Namespace) -> int:
+    try:
+        conn = get_connection(args.db)
+        runs_list = list_runs(conn, workflow_id=args.workflow)
+    except PersistenceError as exc:
+        print(f"Database error: {exc}", file=sys.stderr)
+        return 3
+    except Exception as exc:
+        print(f"Internal error: {exc}", file=sys.stderr)
+        return 3
+
+    if args.json:
+        runs_data = [
+            {
+                "run_id": row[0],
+                "workflow_id": row[1],
+                "started_at": row[2],
+                "finished_at": row[3],
+                "status": row[4],
+            }
+            for row in runs_list
+        ]
+        print(json.dumps(runs_data, indent=2))
+    else:
+        if not runs_list:
+            print("No runs found.")
+        else:
+            for row in runs_list:
+                print(
+                    f"{row[0]}  workflow={row[1]}  status={row[4]}  "
+                    f"started={row[2]}  finished={row[3]}"
+                )
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument(
+        "--log-level",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+        type=str.upper,
+        help="Configure log level for orchestration_engine.",
+    )
+
+    parser = argparse.ArgumentParser(
+        prog="orchestration-engine",
+        description="Fault-tolerant, dependency-aware workflow orchestration engine.",
+        parents=[common],
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
+
+    # validate
+    val_p = subparsers.add_parser(
+        "validate", parents=[common], help="Validate a workflow definition file."
+    )
+    val_p.add_argument("workflow_file", help="Path to workflow JSON or YAML file.")
+    val_p.add_argument("--json", action="store_true", help="Output result as JSON.")
+    val_p.set_defaults(func=_cmd_validate)
+
+    # run
+    run_p = subparsers.add_parser("run", parents=[common], help="Run a workflow to completion.")
+    run_p.add_argument("workflow_file", help="Path to workflow JSON or YAML file.")
+    run_p.add_argument(
+        "--runner", required=True, help="Task runner callable in 'module:function' format."
+    )
+    run_p.add_argument(
+        "--db", default=None, help="Path to SQLite database (defaults to in-memory)."
+    )
+    run_p.add_argument("--run-id", default=None, help="Custom identifier for this run.")
+    run_p.add_argument(
+        "--max-concurrency", type=int, default=1, help="Maximum concurrent tasks (default: 1)."
+    )
+    run_p.add_argument(
+        "--executor",
+        choices=["thread", "process"],
+        default="thread",
+        help="Coordinator executor backend.",
+    )
+    run_p.add_argument("--json", action="store_true", help="Output result as JSON.")
+    run_p.set_defaults(func=_cmd_run)
+
+    # resume
+    res_p = subparsers.add_parser(
+        "resume", parents=[common], help="Resume an interrupted run."
+    )
+    res_p.add_argument("run_id", help="Run identifier to resume.")
+    res_p.add_argument(
+        "--runner", required=True, help="Task runner callable in 'module:function' format."
+    )
+    res_p.add_argument("--db", required=True, help="Path to SQLite database.")
+    res_p.add_argument(
+        "--max-concurrency", type=int, default=1, help="Maximum concurrent tasks (default: 1)."
+    )
+    res_p.add_argument(
+        "--executor",
+        choices=["thread", "process"],
+        default="thread",
+        help="Coordinator executor backend.",
+    )
+    res_p.add_argument("--json", action="store_true", help="Output result as JSON.")
+    res_p.set_defaults(func=_cmd_resume)
+
+    # status
+    stat_p = subparsers.add_parser("status", parents=[common], help="Check run status.")
+    stat_p.add_argument("run_id", help="Run identifier.")
+    stat_p.add_argument("--db", required=True, help="Path to SQLite database.")
+    stat_p.add_argument("--json", action="store_true", help="Output result as JSON.")
+    stat_p.set_defaults(func=_cmd_status)
+
+    # history
+    hist_p = subparsers.add_parser(
+        "history", parents=[common], help="Show task transition history."
+    )
+    hist_p.add_argument("run_id", help="Run identifier.")
+    hist_p.add_argument("--db", required=True, help="Path to SQLite database.")
+    hist_p.add_argument("--json", action="store_true", help="Output result as JSON.")
+    hist_p.set_defaults(func=_cmd_history)
+
+    # runs
+    runs_p = subparsers.add_parser("runs", parents=[common], help="List workflow runs.")
+    runs_p.add_argument("--db", required=True, help="Path to SQLite database.")
+    runs_p.add_argument("--workflow", default=None, help="Filter runs by workflow identifier.")
+    runs_p.add_argument("--json", action="store_true", help="Output result as JSON.")
+    runs_p.set_defaults(func=_cmd_runs)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Entry point for the orchestration-engine CLI."""
+    parser = build_parser()
+    try:
+        args = parser.parse_args(argv if argv is not None else sys.argv[1:])
+    except SystemExit as exc:
+        return int(exc.code) if isinstance(exc.code, int) else 2
+
+    if args.log_level:
+        logging.basicConfig(
+            level=getattr(logging, args.log_level.upper()),
+            format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        )
+
+    if not hasattr(args, "func"):
+        parser.print_help(sys.stderr)
+        return 2
+
+    return int(args.func(args))

orchestration_engine/errors.py

@@ -0,0 +1,23 @@
+"""
+Error hierarchy for orchestration engine.
+"""
+
+
+class OrchestrationError(Exception):
+    """Base class for all orchestration engine exceptions."""
+
+
+class WorkflowDefinitionError(OrchestrationError, ValueError):
+    """Raised when a workflow definition is invalid, incomplete, or malformed."""
+
+    def __init__(self, message: str, errors: list[str] | None = None) -> None:
+        super().__init__(message)
+        self.errors: list[str] = list(errors or [])
+
+
+class WorkflowValidationError(OrchestrationError, ValueError):
+    """Raised when a workflow fails validation (cycles, invalid IDs, self-dependencies)."""
+
+
+class PersistenceError(OrchestrationError):
+    """Raised when a persistence or database operation fails."""