orchestrator-lso 2.3.1__tar.gz → 2.4.1__tar.gz

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-lso
-Version: 2.3.1
+Version: 2.4.1
 Summary: LSO, an API for remotely running Ansible playbooks.
 Author: GÉANT Orchestration and Automation Team
 Author-email: GÉANT Orchestration and Automation Team <goat@geant.org>

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.1}/lso/__init__.py

@@ -13,7 +13,7 @@
 
 """LSO, an API for remotely running Ansible playbooks."""
 
-__version__ = "2.3.1"
+__version__ = "2.4.1"
 
 import logging
 

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.1}/lso/execute.py

@@ -14,8 +14,8 @@
 """Module for handling the execution of arbitrary executables."""
 
 import subprocess  # noqa: S404
-import uuid
 from pathlib import Path
+from uuid import UUID, uuid4
 
 from pydantic import HttpUrl
 
@@ -30,12 +30,12 @@ def get_executable_path(executable_name: Path) -> Path:
     return Path(settings.EXECUTABLES_ROOT_DIR) / executable_name
 
 
-def run_executable_async(executable_path: Path, args: list[str], callback: HttpUrl | None) -> uuid.UUID:
+def run_executable_async(executable_path: Path, args: list[str], callback: HttpUrl | None) -> UUID:
     """Dispatch the task for executing an arbitrary executable remotely.
 
     Uses a ThreadPoolExecutor (for local execution) or a Celery worker (for distributed tasks).
     """
-    job_id = uuid.uuid4()
+    job_id = uuid4()
     callback_url = str(callback) if callback else None
     if settings.EXECUTOR == ExecutorType.THREADPOOL:
         executor = get_thread_pool()

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.1}/lso/playbook.py

@@ -13,9 +13,9 @@
 
 """Module that gathers common API responses and data models."""
 
-import uuid
 from pathlib import Path
 from typing import Any
+from uuid import UUID, uuid4
 
 from pydantic import HttpUrl
 
@@ -33,8 +33,11 @@ def run_playbook(
     playbook_path: Path,
     extra_vars: dict[str, Any],
     inventory: dict[str, Any] | str,
-    callback: HttpUrl,
-) -> uuid.UUID:
+    callback: HttpUrl | None,
+    progress: HttpUrl | None,
+    *,
+    progress_is_incremental: bool,
+) -> UUID:
     """Run an Ansible playbook against a specified inventory.
 
     :param Path playbook_path: Playbook to be executed.
@@ -42,19 +45,40 @@ def run_playbook(
     :param dict[str, Any] | str inventory: The inventory that the playbook is executed against.
     :param HttpUrl callback: Callback URL where the playbook should send a status update when execution is completed.
                              This is used for workflow-orchestrator to continue with the next step in a workflow.
-    :return: Result of playbook launch, this could either be successful or unsuccessful.
-    :rtype: :class:`fastapi.responses.JSONResponse`
+    :return UUID: Job ID of the launched playbook.
     """
-    job_id = uuid.uuid4()
+    job_id = uuid4()
+    callback_str = None
+    progress_str = None
+    if callback:
+        callback_str = str(callback)
+    if progress:
+        progress_str = str(progress)
+
     if settings.EXECUTOR == ExecutorType.THREADPOOL:
         executor = get_thread_pool()
         executor_handle = executor.submit(
-            run_playbook_proc_task, str(job_id), str(playbook_path), extra_vars, inventory, str(callback)
+            run_playbook_proc_task,
+            str(job_id),
+            str(playbook_path),
+            extra_vars,
+            inventory,
+            callback_str,
+            progress_str,
+            progress_is_incremental=progress_is_incremental,
         )
         if settings.TESTING:
             executor_handle.result()
 
     elif settings.EXECUTOR == ExecutorType.WORKER:
-        run_playbook_proc_task.delay(str(job_id), str(playbook_path), extra_vars, inventory, str(callback))
+        run_playbook_proc_task.delay(
+            str(job_id),
+            str(playbook_path),
+            extra_vars,
+            inventory,
+            callback_str,
+            progress_str,
+            progress_is_incremental=progress_is_incremental,
+        )
 
     return job_id

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.1}/lso/routes/execute.py

@@ -14,9 +14,9 @@
 """FastAPI route for running arbitrary executables."""
 
 import asyncio
-import uuid
 from pathlib import Path
 from typing import Annotated
+from uuid import uuid4
 
 from fastapi import APIRouter, HTTPException, status
 from pydantic import AfterValidator, BaseModel, HttpUrl
@@ -61,6 +61,6 @@ async def run_executable_endpoint(params: ExecutableRunParams) -> ExecutableRunR
         job_id = run_executable_async(params.executable_name, params.args, params.callback)
         return ExecutableRunResponse(job_id=job_id)
 
-    job_id = uuid.uuid4()
+    job_id = uuid4()
     result = await asyncio.to_thread(run_executable_sync, str(params.executable_name), params.args)
     return ExecutableRunResponse(job_id=job_id, result=result)

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.1}/lso/routes/playbook.py

@@ -15,11 +15,11 @@
 
 import json
 import tempfile
-import uuid
 from contextlib import redirect_stderr
 from io import StringIO
 from pathlib import Path
 from typing import Annotated, Any
+from uuid import UUID
 
 import ansible_runner
 from ansible.inventory.manager import InventoryManager
@@ -44,7 +44,7 @@ def _inventory_validator(inventory: dict[str, Any] | str) -> dict[str, Any] | st
     """
     if not ansible_runner.utils.isinventory(inventory):
         detail = "Invalid inventory provided. Should be a string, or JSON object."
-        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=detail)
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=detail)
 
     loader = DataLoader()
     output = StringIO()
@@ -58,7 +58,7 @@ def _inventory_validator(inventory: dict[str, Any] | str) -> dict[str, Any] | st
     output.seek(0)
     error_messages = output.readlines()
     if error_messages:
-        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=error_messages)
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=error_messages)
 
     return inventory
 
@@ -79,7 +79,7 @@ PlaybookName = Annotated[Path, AfterValidator(_playbook_path_validator)]
 class PlaybookRunResponse(BaseModel):
     """PlaybookRunResponse domain model schema."""
 
-    job_id: uuid.UUID
+    job_id: UUID
 
 
 class PlaybookRunParams(BaseModel):
@@ -89,7 +89,11 @@ class PlaybookRunParams(BaseModel):
     #: configuration option ``ANSIBLE_PLAYBOOKS_ROOT_DIR``.
     playbook_name: PlaybookName
     #: The address where LSO should call back to upon completion.
-    callback: HttpUrl
+    callback: HttpUrl | None = None
+    #: Optionally, the address where LSO should send progress updates as the playbook executes.
+    progress: HttpUrl | None = None
+    #: Optionally, whether progress updates should be incremental or not.
+    progress_is_incremental: bool = True
     #: The inventory to run the playbook against. This inventory can also include any host vars, if needed. When
     #: including host vars, it should be a dictionary. Can be a simple string containing hostnames when no host vars are
     #: needed. In the latter case, multiple hosts should be separated with a ``\n`` newline character only.
@@ -114,6 +118,8 @@ def run_playbook_endpoint(params: PlaybookRunParams) -> PlaybookRunResponse:
         extra_vars=params.extra_vars,
         inventory=params.inventory,
         callback=params.callback,
+        progress=params.progress,
+        progress_is_incremental=params.progress_is_incremental,
     )
 
     return PlaybookRunResponse(job_id=job_id)

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.1}/lso/schema.py

@@ -13,8 +13,8 @@
 
 """Module for defining the schema for running arbitrary executables."""
 
-import uuid
 from enum import StrEnum
+from uuid import UUID
 
 from pydantic import BaseModel, model_validator
 
@@ -46,5 +46,5 @@ class ExecutionResult(BaseModel):
 class ExecutableRunResponse(BaseModel):
     """Response for running an arbitrary executable."""
 
-    job_id: uuid.UUID
+    job_id: UUID
     result: ExecutionResult | None = None

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.1}/lso/tasks.py

@@ -18,11 +18,12 @@ the results to a specified callback URL.
 """
 
 import logging
+from collections.abc import Callable
 from typing import Any
 from uuid import UUID
 
-import ansible_runner
 import requests
+from ansible_runner import Runner, run
 from starlette import status
 
 from lso.config import settings
@@ -36,9 +37,72 @@ class CallbackFailedError(Exception):
     """Exception raised when a callback url can't be reached."""
 
 
+def playbook_event_handler_factory(
+    progress: str | None, *, progress_is_incremental: bool
+) -> Callable[[dict], bool] | None:
+    """Create an event handler for Ansible playbook runs.
+
+    This is used to send incremental progress updates to the external system that called for this playbook to be run.
+
+    :param str progress: The progress URL where the external system expects to receive updates.
+    :param bool progress_is_incremental: Whether progress updates are sent incrementally, or only contain the latest
+                                         event data.
+    """
+    events_stdout = []
+
+    def _playbook_event_handler(event: dict) -> bool:
+        if progress_is_incremental:
+            emit_body = event["stdout"].strip()
+        else:
+            events_stdout.append(event["stdout"].strip())
+            emit_body = events_stdout
+
+        requests.post(str(progress), json={"progress": emit_body}, timeout=settings.REQUEST_TIMEOUT_SEC)
+        return True
+
+    if progress:
+        return _playbook_event_handler
+    return None
+
+
+def playbook_finished_handler_factory(callback: str | None, job_id: str) -> Callable[[Runner], None] | None:
+    """Create an event handler for finished Ansible playbook runs.
+
+    Once Ansible runner is finished, it will call the handler method created by this factory before teardown.
+
+    :param str callback: The callback URL that ansible runner should report to.
+    :param str job_id: The job ID of this playbook run, used for reporting.
+    :return Callable: A handler method that sends one request to the callback URL.
+    """
+
+    def _playbook_finished_handler(runner: Runner) -> None:
+        payload = {
+            "status": runner.status,
+            "job_id": job_id,
+            "output": runner.stdout.readlines(),
+            "return_code": int(runner.rc),
+        }
+
+        response = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
+        if not (status.HTTP_200_OK <= response.status_code < status.HTTP_300_MULTIPLE_CHOICES):
+            msg = f"Callback failed: {response.text}, url: {callback}"
+            raise CallbackFailedError(msg)
+
+    if callback:
+        return _playbook_finished_handler
+    return None
+
+
 @celery.task(name=RUN_PLAYBOOK)  # type: ignore[misc]
 def run_playbook_proc_task(
-    job_id: str, playbook_path: str, extra_vars: dict[str, Any], inventory: dict[str, Any] | str, callback: str
+    job_id: str,
+    playbook_path: str,
+    extra_vars: dict[str, Any],
+    inventory: dict[str, Any] | str,
+    callback: str | None,
+    progress: str | None,
+    *,
+    progress_is_incremental: bool,
 ) -> None:
     """Celery task to run a playbook.
 
@@ -46,24 +110,20 @@ def run_playbook_proc_task(
     :param str playbook_path: Path to the playbook to be executed.
     :param dict[str, Any] extra_vars: Extra variables to pass to the playbook.
     :param dict[str, Any] | str inventory: Inventory to run the playbook against.
-    :param HttpUrl callback: Callback URL for status updates.
+    :param str callback: Callback URL for status updates.
+    :param str progress: URL for sending progress updates.
+    :param bool progress_is_incremental: Whether progress updates include all past progress.
     :return: None
     """
     msg = f"playbook_path: {playbook_path}, callback: {callback}"
     logger.info(msg)
-    ansible_playbook_run = ansible_runner.run(playbook=playbook_path, inventory=inventory, extravars=extra_vars)
-
-    payload = {
-        "status": ansible_playbook_run.status,
-        "job_id": job_id,
-        "output": ansible_playbook_run.stdout.readlines(),
-        "return_code": int(ansible_playbook_run.rc),
-    }
-
-    response = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
-    if not (status.HTTP_200_OK <= response.status_code < status.HTTP_300_MULTIPLE_CHOICES):
-        msg = f"Callback failed: {response.text}, url: {callback}"
-        raise CallbackFailedError(msg)
+    run(
+        playbook=playbook_path,
+        inventory=inventory,
+        extravars=extra_vars,
+        event_handler=playbook_event_handler_factory(progress, progress_is_incremental=progress_is_incremental),
+        finished_callback=playbook_finished_handler_factory(callback, job_id),
+    )
 
 
 @celery.task(name=RUN_EXECUTABLE)  # type: ignore[misc]

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "orchestrator-lso"
-version = "2.3.1"
+version = "2.4.1"
 description = "LSO, an API for remotely running Ansible playbooks."
 readme = "README.md"
 license = "Apache-2.0"
@@ -120,8 +120,9 @@ ignore = [
     "D203",
     "D213",
     "N805",
-    "PLR0913",
     "PLR0904",
+    "PLR0913",
+    "PLR0917",
     "PLW1514",
     "S104"
 ]