operaton-tasks 1.0.0a1__py3-none-any.whl

operaton/tasks/__init__.py

@@ -0,0 +1,20 @@
+from operaton.tasks.api import external_task_worker
+from operaton.tasks.api import handlers
+from operaton.tasks.api import operaton_session
+from operaton.tasks.api import router
+from operaton.tasks.api import serve
+from operaton.tasks.api import settings
+from operaton.tasks.api import task
+from operaton.tasks.api import task as register
+
+
+__all__ = [
+    "external_task_worker",
+    "handlers",
+    "operaton_session",
+    "register",
+    "router",
+    "serve",
+    "settings",
+    "task",
+]

operaton/tasks/api.py

@@ -0,0 +1,91 @@
+from operaton.tasks.config import handlers
+from operaton.tasks.config import logger
+from operaton.tasks.config import router
+from operaton.tasks.config import settings
+from operaton.tasks.types import ExternalTaskHandler
+from operaton.tasks.types import ExternalTaskTopic
+from operaton.tasks.utils import operaton_session
+from operaton.tasks.worker import external_task_worker
+from typing import Any
+from typing import Callable
+from typing import Optional
+import sys
+
+
+try:
+    import typer
+    import uvicorn
+
+    HAS_CLI = True
+except ImportError:
+    typer: Any = None  # type: ignore
+    uvicorn: Any = None  # type: ignore
+
+    HAS_CLI = False
+
+
+def task(
+    topic: str,
+    localVariables: bool = True,
+) -> Callable[[ExternalTaskHandler], ExternalTaskHandler]:
+    """Register function as a service task."""
+
+    def decorator(func: ExternalTaskHandler) -> ExternalTaskHandler:
+        handlers[topic] = ExternalTaskTopic(handler=func, localVariables=localVariables)
+        return func
+
+    return decorator
+
+
+if HAS_CLI:
+    cli = typer.Typer()
+
+    @cli.command(name="serve")
+    def cli_serve(
+        base_url: str = "http://localhost:8080/engine-rest",
+        authorization: Optional[str] = None,
+        timeout: int = 20,
+        poll_ttl: int = 10,
+        lock_ttl: int = 30,
+        worker_id: str = "operaton-tasks-client",
+        log_level: str = "INFO",
+        args: Optional[list[str]] = typer.Argument(
+            default=None, help="arguments passed to uvicorn"
+        ),
+    ) -> None:
+        """CLI."""
+        settings.ENGINE_REST_BASE_URL = base_url
+        settings.ENGINE_REST_AUTHORIZATION = authorization
+        settings.ENGINE_REST_TIMEOUT_SECONDS = timeout
+        settings.ENGINE_REST_POLL_TTL_SECONDS = poll_ttl
+        settings.ENGINE_REST_LOCK_TTL_SECONDS = lock_ttl
+        settings.LOG_LEVEL = log_level
+        settings.TASKS_WORKER_ID = worker_id
+        settings.TASKS_MODULE = None
+
+        sys.argv = [sys.argv[0], "operaton.tasks.main:app"]
+        if args and "--no-proxy-headers" not in args:
+            sys.argv.append("--proxy-headers")
+        if args:
+            sys.argv.extend(args)
+        uvicorn.main()
+
+
+def serve() -> None:
+    """Run Operaton External Service Task Worker."""
+    if HAS_CLI:
+        cli()
+    else:
+        logger.error("operaton-tasks[cli] required")
+        exit(1)
+
+
+__all__ = [
+    "external_task_worker",
+    "handlers",
+    "operaton_session",
+    "router",
+    "serve",
+    "settings",
+    "task",
+]

operaton/tasks/config.py

@@ -0,0 +1,54 @@
+from fastapi import APIRouter
+from operaton.tasks.types import ExternalTaskTopic
+from pydantic_settings import BaseSettings
+from typing import Dict
+from typing import Optional
+import logging
+
+
+# https://pydantic-docs.helpmanual.io/usage/settings/
+class Settings(BaseSettings):
+    ENGINE_REST_BASE_URL: str = "http://localhost:8080/engine-rest"
+    ENGINE_REST_AUTHORIZATION: Optional[str] = None
+
+    ENGINE_REST_TIMEOUT_SECONDS: int = 20
+    ENGINE_REST_POLL_TTL_SECONDS: int = 10
+    ENGINE_REST_LOCK_TTL_SECONDS: int = 30
+
+    TASKS_HEARTBEAT_TOPIC: str = "operaton.tasks.heartbeat"
+    TASKS_WORKER_ID: str = "operaton-tasks-client"
+    TASKS_MODULE: Optional[str] = None
+
+    LOG_LEVEL: str = "DEBUG"
+
+
+settings = Settings()
+
+formatter = logging.Formatter(
+    "%(asctime)s | %(levelname)s | %(name)s:%(lineno)d | %(message)s",
+    "%d-%m-%Y %H:%M:%S",
+)
+
+stream_handler = logging.StreamHandler()
+stream_handler.setFormatter(formatter)
+stream_handler.setLevel(settings.LOG_LEVEL)
+
+logger = logging.getLogger("operaton.tasks")
+logger.addHandler(stream_handler)
+logger.setLevel(settings.LOG_LEVEL)
+logger.propagate = False
+
+
+def update() -> None:
+    stream_handler.setLevel(settings.LOG_LEVEL)
+    logger.setLevel(settings.LOG_LEVEL)
+
+
+# Built-in FastAPI router
+router = APIRouter()
+
+# All topics registered using the task decorator
+handlers: Dict[str, ExternalTaskTopic] = {}
+
+
+__all__ = ["logger", "settings", "router", "handlers", "update"]

operaton/tasks/healthz.py

@@ -0,0 +1,71 @@
+from operaton.tasks import router
+from operaton.tasks import task
+from operaton.tasks.api import operaton_session
+from operaton.tasks.config import settings
+from operaton.tasks.types import CompleteExternalTaskDto
+from operaton.tasks.types import ExternalTaskComplete
+from operaton.tasks.types import LockedExternalTaskDto
+from operaton.tasks.types import VariableValueDto
+from operaton.tasks.utils import verify_response_status
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic.dataclasses import dataclass
+from starlette.exceptions import HTTPException
+from typing import Optional
+import datetime
+
+
+class Heartbeat(BaseModel):
+    """Health check response."""
+
+    timestamp: Optional[str] = Field(
+        None,
+        description="UTC timestamp of the last recorded heartbeat.",
+    )
+
+
+@dataclass
+class State:
+    """Service health check state."""
+
+    timestamp: Optional[str] = None
+
+
+state = State()
+
+
+@task(topic=settings.TASKS_HEARTBEAT_TOPIC)
+async def handler(task: LockedExternalTaskDto) -> ExternalTaskComplete:
+    """Update health check timestamp."""
+    state.timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
+    return ExternalTaskComplete(
+        task=task,
+        response=CompleteExternalTaskDto(
+            workerId=task.workerId,
+            variables={
+                "timestamp": VariableValueDto(value=state.timestamp, type="string"),
+            },
+        ),
+    )
+
+
+@router.get(
+    "/healthz", response_model=Heartbeat, summary="Service health status", tags=["Meta"]
+)
+async def healthz() -> Heartbeat:
+    """Service health status."""
+
+    # Without heartbeat external task triggered
+    if state.timestamp is None:
+        async with operaton_session() as session:
+            get = await session.get(settings.ENGINE_REST_BASE_URL + "/engine")
+            await verify_response_status(get, (200,))
+        timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
+        return Heartbeat(timestamp=timestamp)
+
+    # With heartbeat external task triggered at least once
+    now = datetime.datetime.now(datetime.timezone.utc)
+    if (now - datetime.timedelta(seconds=45)).isoformat() < state.timestamp:
+        return Heartbeat(timestamp=state.timestamp)
+    age = (now - datetime.datetime.fromisoformat(state.timestamp)).total_seconds()
+    raise HTTPException(status_code=500, detail=f"No heartbeat for {age} seconds")

operaton/tasks/main.py

@@ -0,0 +1,143 @@
+"""Operaton External Service Task Client"""
+
+from contextlib import asynccontextmanager
+from fastapi.applications import FastAPI
+from operaton.tasks import healthz
+from operaton.tasks.config import handlers
+from operaton.tasks.config import router
+from operaton.tasks.config import settings
+from operaton.tasks.config import update
+from operaton.tasks.worker import external_task_worker
+from pathlib import Path
+from starlette.requests import Request
+from starlette.responses import Response
+from typing import Any
+from typing import AsyncGenerator
+from typing import Awaitable
+from typing import Callable
+from typing import Optional
+import asyncio
+import hashlib
+import importlib.util
+import logging
+import sys
+import tempfile
+
+
+try:
+    import typer
+    import uvicorn
+
+    HAS_CLI = True
+except ImportError:
+    typer: Any = None  # type: ignore
+    uvicorn: Any = None  # type: ignore
+
+    HAS_CLI = False
+
+
+assert healthz  # import registers healthz task handler
+
+
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncGenerator[Any, Any]:
+    """Start external task worker on FastAPI startup."""
+    update()
+    if settings.TASKS_MODULE:
+        module_name = hashlib.sha256(settings.TASKS_MODULE.encode("utf-8")).hexdigest()
+        spec = importlib.util.spec_from_file_location(
+            module_name, settings.TASKS_MODULE
+        )
+        if spec:
+            module = importlib.util.module_from_spec(spec)
+            if spec.loader:
+                spec.loader.exec_module(module)
+    asyncio.ensure_future(external_task_worker(handlers))
+    logger.info("Event loop: %s", asyncio.get_event_loop())
+    yield
+
+
+app = FastAPI(
+    title="Operaton Tasks Client",
+    description="Operaton External Service Task Client",
+    version="0.1.0",
+    lifespan=lifespan,
+)
+
+
+app.include_router(router)
+
+
+@app.middleware("http")
+async def cache_headers(
+    request: Request, call_next: Callable[[Request], Awaitable[Response]]
+) -> Response:
+    """Set cache headers."""
+    response = await call_next(request)
+    response.headers["Cache-Control"] = "no-store, max-age=0"
+    return response
+
+
+if HAS_CLI:
+    cli = typer.Typer()
+
+    @cli.command()
+    def serve(
+        module: Path,
+        base_url: str = "http://localhost:8080/engine-rest",
+        authorization: Optional[str] = None,
+        timeout: int = 20,
+        poll_ttl: int = 10,
+        lock_ttl: int = 30,
+        worker_id: str = "operaton-tasks-client",
+        log_level: str = "INFO",
+        args: Optional[list[str]] = typer.Argument(
+            default=None, help="arguments passed to uvicorn"
+        ),
+    ) -> None:
+        """CLI."""
+        settings.ENGINE_REST_BASE_URL = base_url
+        settings.ENGINE_REST_AUTHORIZATION = authorization
+        settings.ENGINE_REST_TIMEOUT_SECONDS = timeout
+        settings.ENGINE_REST_POLL_TTL_SECONDS = poll_ttl
+        settings.ENGINE_REST_LOCK_TTL_SECONDS = lock_ttl
+        settings.TASKS_WORKER_ID = worker_id
+        settings.LOG_LEVEL = log_level
+        settings.TASKS_MODULE = f"{module.absolute()}"
+
+        sys.argv = [sys.argv[0], "operaton.tasks.main:app"]
+        if args and "--no-proxy-headers" not in args:
+            sys.argv.append("--proxy-headers")
+        if args:
+            sys.argv.extend(args)
+        if args and "--reload" in args:
+            with tempfile.NamedTemporaryFile(mode="w+", delete=True) as temp_file:
+                temp_file.writelines(
+                    [
+                        f"ENGINE_REST_BASE_URL={base_url}\n",
+                        f"ENGINE_REST_AUTHORIZATION={authorization}\n",
+                        f"ENGINE_REST_TIMEOUT_SECONDS={timeout}\n",
+                        f"ENGINE_REST_POLL_TTL_SECONDS={poll_ttl}\n",
+                        f"ENGINE_REST_LOCK_TTL_SECONDS={lock_ttl}\n",
+                        f"TASKS_WORKER_ID={worker_id}\n",
+                        f"TASKS_MODULE={module}\n",
+                        f"LOG_LEVEL={log_level}",
+                    ]
+                )
+                temp_file.flush()
+                sys.argv.extend(["--env-file", temp_file.name])
+                uvicorn.main()
+        else:
+            uvicorn.main()
+
+
+def main() -> None:
+    """Main."""
+    if HAS_CLI:
+        cli()
+    else:
+        logger.error("operaton-tasks[cli] required")
+        exit(1)