orchestrator-lso 2.3.1__tar.gz → 2.4.0__tar.gz

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orchestrator-lso
-Version: 2.3.1
+Version: 2.4.0
 Summary: LSO, an API for remotely running Ansible playbooks.
 Author: GÉANT Orchestration and Automation Team
 Author-email: GÉANT Orchestration and Automation Team <goat@geant.org>

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.0}/lso/__init__.py

@@ -13,7 +13,7 @@
 
 """LSO, an API for remotely running Ansible playbooks."""
 
-__version__ = "2.3.1"
+__version__ = "2.4.0"
 
 import logging
 

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.0}/lso/execute.py

@@ -14,8 +14,8 @@
 """Module for handling the execution of arbitrary executables."""
 
 import subprocess  # noqa: S404
-import uuid
 from pathlib import Path
+from uuid import UUID, uuid4
 
 from pydantic import HttpUrl
 
@@ -30,12 +30,12 @@ def get_executable_path(executable_name: Path) -> Path:
     return Path(settings.EXECUTABLES_ROOT_DIR) / executable_name
 
 
-def run_executable_async(executable_path: Path, args: list[str], callback: HttpUrl | None) -> uuid.UUID:
+def run_executable_async(executable_path: Path, args: list[str], callback: HttpUrl | None) -> UUID:
     """Dispatch the task for executing an arbitrary executable remotely.
 
     Uses a ThreadPoolExecutor (for local execution) or a Celery worker (for distributed tasks).
     """
-    job_id = uuid.uuid4()
+    job_id = uuid4()
     callback_url = str(callback) if callback else None
     if settings.EXECUTOR == ExecutorType.THREADPOOL:
         executor = get_thread_pool()

orchestrator_lso-2.4.0/lso/playbook.py

@@ -0,0 +1,143 @@
+# Copyright 2023-2025 GÉANT Vereniging.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module that gathers common API responses and data models."""
+
+import logging
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+from uuid import UUID, uuid4
+
+import requests
+from ansible_runner import Runner
+from pydantic import HttpUrl
+from starlette import status
+
+from lso.config import ExecutorType, settings
+from lso.tasks import run_playbook_proc_task
+from lso.utils import CallbackFailedError, get_thread_pool
+
+logger = logging.getLogger(__name__)
+
+
+def get_playbook_path(playbook_name: Path) -> Path:
+    """Get the path of a playbook on the local filesystem."""
+    return Path(settings.ANSIBLE_PLAYBOOKS_ROOT_DIR) / playbook_name
+
+
+def playbook_event_handler_factory(progress: str, *, progress_is_incremental: bool) -> Callable[[dict], bool]:
+    """Create an event handler for Ansible playbook runs.
+
+    This is used to send incremental progress updates to the external system that called for this playbook to be run.
+
+    :param str progress: The progress URL where the external system expects to receive updates.
+    :param bool progress_is_incremental: Whether progress updates are sent incrementally, or only contain the latest
+                                         event data.
+    :return Callable[[dict], bool]]: A handler method that processes every Ansible playbook event.
+    """
+    events_stdout = []
+
+    def _playbook_event_handler(event: dict) -> bool:
+        if progress_is_incremental:
+            emit_body = event["stdout"].strip()
+        else:
+            events_stdout.append(event["stdout"].strip())
+            emit_body = events_stdout
+
+        requests.post(str(progress), json={"progress": emit_body}, timeout=settings.REQUEST_TIMEOUT_SEC)
+        return True
+
+    return _playbook_event_handler
+
+
+def playbook_finished_handler_factory(callback: str, job_id: UUID) -> Callable[[Runner], None]:
+    """Create an event handler for finished Ansible playbook runs.
+
+    Once Ansible runner is finished, it will call the handler method created by this factory before teardown.
+
+    :param str callback: The callback URL that ansible runner should report to.
+    :param UUID job_id: The job ID of this playbook run, used for reporting.
+    :return Callable[[Runner], None]: A handler method that sends one request to the callback URL.
+    """
+
+    def _playbook_finished_handler(runner: Runner) -> None:
+        payload = {
+            "status": runner.status,
+            "job_id": str(job_id),
+            "output": runner.stdout.readlines(),
+            "return_code": int(runner.rc),
+        }
+
+        response = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
+        if not (status.HTTP_200_OK <= response.status_code < status.HTTP_300_MULTIPLE_CHOICES):
+            msg = f"Callback failed: {response.text}, url: {callback}"
+            raise CallbackFailedError(msg)
+
+    return _playbook_finished_handler
+
+
+def run_playbook(
+    playbook_path: Path,
+    extra_vars: dict[str, Any],
+    inventory: dict[str, Any] | str,
+    callback: HttpUrl | None,
+    progress: HttpUrl | None,
+    *,
+    progress_is_incremental: bool,
+) -> UUID:
+    """Run an Ansible playbook against a specified inventory.
+
+    :param Path playbook_path: Playbook to be executed.
+    :param dict[str, Any] extra_vars: Any extra vars needed for the playbook to run.
+    :param dict[str, Any] | str inventory: The inventory that the playbook is executed against.
+    :param HttpUrl callback: Callback URL where the playbook should send a status update when execution is completed.
+                             This is used for workflow-orchestrator to continue with the next step in a workflow.
+    :return UUID: Job ID of the launched playbook.
+    """
+    msg = f"playbook_path: {playbook_path}"
+    job_id = uuid4()
+    callback_str = None
+    progress_str = None
+    event_handler = None
+    finished_callback = None
+
+    if callback:
+        callback_str = str(callback)
+        msg += f", callback URL: {callback_str}"
+        finished_callback = playbook_finished_handler_factory(callback_str, job_id)
+    if progress:
+        progress_str = str(progress)
+        msg += f", progress URL: {progress_str}"
+        event_handler = playbook_event_handler_factory(progress_str, progress_is_incremental=progress_is_incremental)
+
+    logger.info(msg)
+
+    if settings.EXECUTOR == ExecutorType.THREADPOOL:
+        executor = get_thread_pool()
+        executor_handle = executor.submit(
+            run_playbook_proc_task, str(playbook_path), extra_vars, inventory, event_handler, finished_callback
+        )
+        if settings.TESTING:
+            executor_handle.result()
+
+    elif settings.EXECUTOR == ExecutorType.WORKER:
+        run_playbook_proc_task.delay(
+            str(playbook_path),
+            extra_vars,
+            inventory,
+            event_handler,
+            finished_callback,
+        )
+
+    return job_id

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.0}/lso/routes/execute.py

@@ -14,9 +14,9 @@
 """FastAPI route for running arbitrary executables."""
 
 import asyncio
-import uuid
 from pathlib import Path
 from typing import Annotated
+from uuid import uuid4
 
 from fastapi import APIRouter, HTTPException, status
 from pydantic import AfterValidator, BaseModel, HttpUrl
@@ -61,6 +61,6 @@ async def run_executable_endpoint(params: ExecutableRunParams) -> ExecutableRunR
         job_id = run_executable_async(params.executable_name, params.args, params.callback)
         return ExecutableRunResponse(job_id=job_id)
 
-    job_id = uuid.uuid4()
+    job_id = uuid4()
     result = await asyncio.to_thread(run_executable_sync, str(params.executable_name), params.args)
     return ExecutableRunResponse(job_id=job_id, result=result)

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.0}/lso/routes/playbook.py

@@ -15,11 +15,11 @@
 
 import json
 import tempfile
-import uuid
 from contextlib import redirect_stderr
 from io import StringIO
 from pathlib import Path
 from typing import Annotated, Any
+from uuid import UUID
 
 import ansible_runner
 from ansible.inventory.manager import InventoryManager
@@ -44,7 +44,7 @@ def _inventory_validator(inventory: dict[str, Any] | str) -> dict[str, Any] | st
     """
     if not ansible_runner.utils.isinventory(inventory):
         detail = "Invalid inventory provided. Should be a string, or JSON object."
-        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=detail)
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=detail)
 
     loader = DataLoader()
     output = StringIO()
@@ -58,7 +58,7 @@ def _inventory_validator(inventory: dict[str, Any] | str) -> dict[str, Any] | st
     output.seek(0)
     error_messages = output.readlines()
     if error_messages:
-        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=error_messages)
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=error_messages)
 
     return inventory
 
@@ -79,7 +79,7 @@ PlaybookName = Annotated[Path, AfterValidator(_playbook_path_validator)]
 class PlaybookRunResponse(BaseModel):
     """PlaybookRunResponse domain model schema."""
 
-    job_id: uuid.UUID
+    job_id: UUID
 
 
 class PlaybookRunParams(BaseModel):
@@ -89,7 +89,11 @@ class PlaybookRunParams(BaseModel):
     #: configuration option ``ANSIBLE_PLAYBOOKS_ROOT_DIR``.
     playbook_name: PlaybookName
     #: The address where LSO should call back to upon completion.
-    callback: HttpUrl
+    callback: HttpUrl | None = None
+    #: Optionally, the address where LSO should send progress updates as the playbook executes.
+    progress: HttpUrl | None = None
+    #: Optionally, whether progress updates should be incremental or not.
+    progress_is_incremental: bool = True
     #: The inventory to run the playbook against. This inventory can also include any host vars, if needed. When
     #: including host vars, it should be a dictionary. Can be a simple string containing hostnames when no host vars are
     #: needed. In the latter case, multiple hosts should be separated with a ``\n`` newline character only.
@@ -114,6 +118,8 @@ def run_playbook_endpoint(params: PlaybookRunParams) -> PlaybookRunResponse:
         extra_vars=params.extra_vars,
         inventory=params.inventory,
         callback=params.callback,
+        progress=params.progress,
+        progress_is_incremental=params.progress_is_incremental,
     )
 
     return PlaybookRunResponse(job_id=job_id)

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.0}/lso/schema.py

@@ -13,8 +13,8 @@
 
 """Module for defining the schema for running arbitrary executables."""
 
-import uuid
 from enum import StrEnum
+from uuid import UUID
 
 from pydantic import BaseModel, model_validator
 
@@ -46,5 +46,5 @@ class ExecutionResult(BaseModel):
 class ExecutableRunResponse(BaseModel):
     """Response for running an arbitrary executable."""
 
-    job_id: uuid.UUID
+    job_id: UUID
     result: ExecutionResult | None = None

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.0}/lso/tasks.py

@@ -18,52 +18,48 @@ the results to a specified callback URL.
 """
 
 import logging
+from collections.abc import Callable
 from typing import Any
 from uuid import UUID
 
-import ansible_runner
 import requests
+from ansible_runner import Runner, run
 from starlette import status
 
 from lso.config import settings
 from lso.schema import ExecutableRunResponse
+from lso.utils import CallbackFailedError
 from lso.worker import RUN_EXECUTABLE, RUN_PLAYBOOK, celery
 
 logger = logging.getLogger(__name__)
 
 
-class CallbackFailedError(Exception):
-    """Exception raised when a callback url can't be reached."""
-
-
 @celery.task(name=RUN_PLAYBOOK)  # type: ignore[misc]
 def run_playbook_proc_task(
-    job_id: str, playbook_path: str, extra_vars: dict[str, Any], inventory: dict[str, Any] | str, callback: str
+    playbook_path: str,
+    extra_vars: dict[str, Any],
+    inventory: dict[str, Any] | str,
+    event_handler: Callable[[dict], bool] | None = None,
+    finished_callback: Callable[[Runner], None] | None = None,
 ) -> None:
     """Celery task to run a playbook.
 
-    :param str job_id: Identifier of the job being executed.
     :param str playbook_path: Path to the playbook to be executed.
     :param dict[str, Any] extra_vars: Extra variables to pass to the playbook.
     :param dict[str, Any] | str inventory: Inventory to run the playbook against.
-    :param HttpUrl callback: Callback URL for status updates.
+    :param Callable[[dict], bool] event_handler: Event handler method that is executed on every event while the playbook
+                                                 runs.
+    :param Callable[[Runner], None] finished_callback: Callback handler method that is executed once the playbook run is
+                                                       completed.
     :return: None
     """
-    msg = f"playbook_path: {playbook_path}, callback: {callback}"
-    logger.info(msg)
-    ansible_playbook_run = ansible_runner.run(playbook=playbook_path, inventory=inventory, extravars=extra_vars)
-
-    payload = {
-        "status": ansible_playbook_run.status,
-        "job_id": job_id,
-        "output": ansible_playbook_run.stdout.readlines(),
-        "return_code": int(ansible_playbook_run.rc),
-    }
-
-    response = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
-    if not (status.HTTP_200_OK <= response.status_code < status.HTTP_300_MULTIPLE_CHOICES):
-        msg = f"Callback failed: {response.text}, url: {callback}"
-        raise CallbackFailedError(msg)
+    run(
+        playbook=playbook_path,
+        inventory=inventory,
+        extravars=extra_vars,
+        event_handler=event_handler,
+        finished_callback=finished_callback,
+    )
 
 
 @celery.task(name=RUN_EXECUTABLE)  # type: ignore[misc]

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.0}/lso/utils.py

@@ -20,6 +20,10 @@ from lso.config import settings
 _executor = None
 
 
+class CallbackFailedError(Exception):
+    """Exception raised when a callback url can't be reached."""
+
+
 def get_thread_pool() -> ThreadPoolExecutor:
     """Initialize or return a cached ThreadPoolExecutor for local asynchronous execution."""
     global _executor  # noqa: PLW0603

{orchestrator_lso-2.3.1 → orchestrator_lso-2.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "orchestrator-lso"
-version = "2.3.1"
+version = "2.4.0"
 description = "LSO, an API for remotely running Ansible playbooks."
 readme = "README.md"
 license = "Apache-2.0"
@@ -120,8 +120,9 @@ ignore = [
     "D203",
     "D213",
     "N805",
-    "PLR0913",
     "PLR0904",
+    "PLR0913",
+    "PLR0917",
     "PLW1514",
     "S104"
 ]

orchestrator_lso-2.3.1/lso/playbook.py

@@ -1,60 +0,0 @@
-# Copyright 2023-2025 GÉANT Vereniging.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Module that gathers common API responses and data models."""
-
-import uuid
-from pathlib import Path
-from typing import Any
-
-from pydantic import HttpUrl
-
-from lso.config import ExecutorType, settings
-from lso.tasks import run_playbook_proc_task
-from lso.utils import get_thread_pool
-
-
-def get_playbook_path(playbook_name: Path) -> Path:
-    """Get the path of a playbook on the local filesystem."""
-    return Path(settings.ANSIBLE_PLAYBOOKS_ROOT_DIR) / playbook_name
-
-
-def run_playbook(
-    playbook_path: Path,
-    extra_vars: dict[str, Any],
-    inventory: dict[str, Any] | str,
-    callback: HttpUrl,
-) -> uuid.UUID:
-    """Run an Ansible playbook against a specified inventory.
-
-    :param Path playbook_path: Playbook to be executed.
-    :param dict[str, Any] extra_vars: Any extra vars needed for the playbook to run.
-    :param dict[str, Any] | str inventory: The inventory that the playbook is executed against.
-    :param HttpUrl callback: Callback URL where the playbook should send a status update when execution is completed.
-                             This is used for workflow-orchestrator to continue with the next step in a workflow.
-    :return: Result of playbook launch, this could either be successful or unsuccessful.
-    :rtype: :class:`fastapi.responses.JSONResponse`
-    """
-    job_id = uuid.uuid4()
-    if settings.EXECUTOR == ExecutorType.THREADPOOL:
-        executor = get_thread_pool()
-        executor_handle = executor.submit(
-            run_playbook_proc_task, str(job_id), str(playbook_path), extra_vars, inventory, str(callback)
-        )
-        if settings.TESTING:
-            executor_handle.result()
-
-    elif settings.EXECUTOR == ExecutorType.WORKER:
-        run_playbook_proc_task.delay(str(job_id), str(playbook_path), extra_vars, inventory, str(callback))
-
-    return job_id