orchestrator-lso 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

lso/__init__.py

@@ -13,7 +13,7 @@
 
 """LSO, an API for remotely running Ansible playbooks."""
 
-__version__ = "2.0.1"
+__version__ = "2.1.0"
 
 import logging
 
@@ -22,6 +22,7 @@ from fastapi.middleware.cors import CORSMiddleware
 
 from lso import environment
 from lso.routes.default import router as default_router
+from lso.routes.execute import router as executable_router
 from lso.routes.playbook import router as playbook_router
 
 logger = logging.getLogger(__name__)
@@ -37,6 +38,7 @@ def create_app() -> FastAPI:
 
     app.include_router(default_router, prefix="/api")
     app.include_router(playbook_router, prefix="/api/playbook")
+    app.include_router(executable_router, prefix="/api/execute")
 
     environment.setup_logging()
 

lso/config.py

@@ -34,15 +34,15 @@ class Config(BaseSettings):
 
     TESTING: bool = False
     ANSIBLE_PLAYBOOKS_ROOT_DIR: str = "/path/to/ansible/playbooks"
+    EXECUTABLES_ROOT_DIR: str = "/path/to/executables"
     EXECUTOR: ExecutorType = ExecutorType.THREADPOOL
     MAX_THREAD_POOL_WORKERS: int = min(32, (os.cpu_count() or 1) + 4)
     REQUEST_TIMEOUT_SEC: int = 10
     CELERY_BROKER_URL: str = "redis://localhost:6379/0"
     CELERY_RESULT_BACKEND: str = "redis://localhost:6379/0"
-    CELERY_TIMEZONE: str = "Europe/Amsterdam"
-    CELERY_ENABLE_UTC: bool = True
     CELERY_RESULT_EXPIRES: int = 3600
     WORKER_QUEUE_NAME: str | None = None
+    EXECUTABLE_TIMEOUT_SEC: int = 300
 
 
 settings = Config()

lso/execute.py

@@ -0,0 +1,31 @@
+"""Module for handling the execution of arbitrary executables."""
+
+import uuid
+from pathlib import Path
+
+from pydantic import HttpUrl
+
+from lso.config import ExecutorType, settings
+from lso.tasks import run_executable_proc_task
+from lso.utils import get_thread_pool
+
+
+def get_executable_path(executable_name: Path) -> Path:
+    """Return the full path of an executable, based on the configured EXECUTABLES_ROOT_DIR."""
+    return Path(settings.EXECUTABLES_ROOT_DIR) / executable_name
+
+
+def run_executable(executable_path: Path, args: list[str], callback: HttpUrl) -> uuid.UUID:
+    """Dispatch the task for executing an arbitrary executable remotely.
+
+    Uses a ThreadPoolExecutor (for local execution) or a Celery worker (for distributed tasks).
+    """
+    job_id = uuid.uuid4()
+    if settings.EXECUTOR == ExecutorType.THREADPOOL:
+        executor = get_thread_pool()
+        future = executor.submit(run_executable_proc_task, str(job_id), str(executable_path), args, str(callback))
+        if settings.TESTING:
+            future.result()
+    elif settings.EXECUTOR == ExecutorType.WORKER:
+        run_executable_proc_task.delay(str(job_id), str(executable_path), args, str(callback))
+    return job_id

lso/playbook.py

@@ -14,7 +14,6 @@
 """Module that gathers common API responses and data models."""
 
 import uuid
-from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Any
 
@@ -22,22 +21,7 @@ from pydantic import HttpUrl
 
 from lso.config import ExecutorType, settings
 from lso.tasks import run_playbook_proc_task
-
-_executor = None
-
-
-def get_thread_pool() -> ThreadPoolExecutor:
-    """Get and optionally initialise a ThreadPoolExecutor.
-
-    Returns:
-        ThreadPoolExecutor
-
-    """
-    global _executor  # noqa: PLW0603
-    if _executor is None:
-        _executor = ThreadPoolExecutor(max_workers=settings.MAX_THREAD_POOL_WORKERS)
-
-    return _executor
+from lso.utils import get_thread_pool
 
 
 def get_playbook_path(playbook_name: Path) -> Path:
@@ -53,7 +37,7 @@ def run_playbook(
 ) -> uuid.UUID:
     """Run an Ansible playbook against a specified inventory.
 
-    :param Path playbook_path: playbook to be executed.
+    :param Path playbook_path: Playbook to be executed.
     :param dict[str, Any] extra_vars: Any extra vars needed for the playbook to run.
     :param dict[str, Any] | str inventory: The inventory that the playbook is executed against.
     :param HttpUrl callback: Callback URL where the playbook should send a status update when execution is completed.

lso/routes/execute.py

@@ -0,0 +1,51 @@
+"""FastAPI route for running arbitrary executables."""
+
+import uuid
+from pathlib import Path
+from typing import Annotated
+
+from fastapi import APIRouter, HTTPException, status
+from pydantic import AfterValidator, BaseModel, HttpUrl
+
+from lso.execute import get_executable_path, run_executable
+
+router = APIRouter()
+
+
+def _executable_path_validator(executable_name: Path) -> Path:
+    """Validate that the executable exists, is a file, and is marked as executable."""
+    executable_path = get_executable_path(executable_name)
+    if not executable_path.exists():
+        msg = f"Executable '{executable_path}' does not exist."
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=msg)
+    if not executable_path.is_file():
+        msg = f"Executable '{executable_name}' is not a valid file."
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=msg)
+    if not executable_path.stat().st_mode & 0o111:
+        msg = f"Executable '{executable_name}' is not marked as executable."
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=msg)
+    return executable_path
+
+
+ExecutableName = Annotated[Path, AfterValidator(_executable_path_validator)]
+
+
+class ExecutableRunParams(BaseModel):
+    """Request parameters for running an arbitrary executable."""
+
+    executable_name: ExecutableName
+    args: list[str] = []
+    callback: HttpUrl
+
+
+class ExecutableRunResponse(BaseModel):
+    """Response for running an arbitrary executable."""
+
+    job_id: uuid.UUID
+
+
+@router.post("/", response_model=ExecutableRunResponse, status_code=status.HTTP_201_CREATED)
+def run_executable_endpoint(params: ExecutableRunParams) -> ExecutableRunResponse:
+    """Dispatch an asynchronous task to run an arbitrary executable."""
+    job_id = run_executable(params.executable_name, params.args, params.callback)
+    return ExecutableRunResponse(job_id=job_id)

lso/routes/playbook.py

@@ -36,7 +36,7 @@ def _inventory_validator(inventory: dict[str, Any] | str) -> dict[str, Any] | st
     """Validate the provided inventory format.
 
     Attempts to parse the inventory to verify its validity. If the inventory cannot be parsed or the inventory
-    format is incorrect an HTTP 422 error is raised.
+    format is incorrect, an HTTP 422 error is raised.
 
     :param inventory: The inventory to validate, can be a dictionary or a string.
     :return: The validated inventory if no errors are found.

lso/tasks.py

@@ -18,6 +18,8 @@ the results to a specified callback URL.
 """
 
 import logging
+import subprocess  # noqa: S404
+from enum import StrEnum
 from typing import Any
 
 import ansible_runner
@@ -25,7 +27,7 @@ import requests
 from starlette import status
 
 from lso.config import settings
-from lso.worker import RUN_PLAYBOOK, celery
+from lso.worker import RUN_EXECUTABLE, RUN_PLAYBOOK, celery
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +36,13 @@ class CallbackFailedError(Exception):
     """Exception raised when a callback url can't be reached."""
 
 
+class JobStatus(StrEnum):
+    """Enumeration of possible job statuses."""
+
+    SUCCESSFUL = "successful"
+    FAILED = "failed"
+
+
 @celery.task(name=RUN_PLAYBOOK)  # type: ignore[misc]
 def run_playbook_proc_task(
     job_id: str, playbook_path: str, extra_vars: dict[str, Any], inventory: dict[str, Any] | str, callback: str
@@ -58,7 +67,55 @@ def run_playbook_proc_task(
         "return_code": int(ansible_playbook_run.rc),
     }
 
-    request_result = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
-    if not status.HTTP_200_OK <= request_result.status_code < status.HTTP_300_MULTIPLE_CHOICES:
-        msg = f"Callback failed: {request_result.text}, url: {callback}"
+    response = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
+    if not (status.HTTP_200_OK <= response.status_code < status.HTTP_300_MULTIPLE_CHOICES):
+        msg = f"Callback failed: {response.text}, url: {callback}"
         raise CallbackFailedError(msg)
+
+
+@celery.task(name=RUN_EXECUTABLE)  # type: ignore[misc]
+def run_executable_proc_task(job_id: str, executable_path: str, args: list[str], callback: str) -> None:
+    """Celery task to run an arbitrary executable and notify via callback.
+
+    Executes the executable with the provided arguments and posts back the output and status.
+    """
+    msg = f"Executing executable: {executable_path} with args: {args}, callback: {callback}"
+    logger.info(msg)
+    try:
+        result = subprocess.run(  # noqa: S603
+            [executable_path, *args],
+            text=True,
+            capture_output=True,
+            timeout=settings.EXECUTABLE_TIMEOUT_SEC,
+            check=False,
+        )
+        output = result.stdout + result.stderr
+        return_code = result.returncode
+    except subprocess.TimeoutExpired:
+        output = "Execution timed out."
+        return_code = -1
+    except Exception as e:  # noqa: BLE001
+        output = str(e)
+        return_code = -1
+
+    payload = {
+        "job_id": job_id,
+        "output": output,
+        "return_code": return_code,
+        "status": JobStatus.SUCCESSFUL if return_code == 0 else JobStatus.FAILED,
+    }
+
+    def _raise_callback_error(message: str, error: Exception | None = None) -> None:
+        if error:
+            raise CallbackFailedError(message) from error
+        raise CallbackFailedError(message)
+
+    try:
+        response = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
+        if not (status.HTTP_200_OK <= response.status_code < status.HTTP_300_MULTIPLE_CHOICES):
+            msg = f"Callback failed: {response.text}, url: {callback}"
+            _raise_callback_error(msg)
+    except Exception as e:
+        error_msg = f"Callback error: {e}"
+        logger.exception(error_msg)
+        _raise_callback_error(error_msg, e)

lso/utils.py

@@ -0,0 +1,16 @@
+"""Utility functions for the LSO package."""
+
+from concurrent.futures import ThreadPoolExecutor
+
+from lso.config import settings
+
+_executor = None
+
+
+def get_thread_pool() -> ThreadPoolExecutor:
+    """Initialize or return a cached ThreadPoolExecutor for local asynchronous execution."""
+    global _executor  # noqa: PLW0603
+    if _executor is None:
+        _executor = ThreadPoolExecutor(max_workers=settings.MAX_THREAD_POOL_WORKERS)
+
+    return _executor

lso/worker.py

@@ -19,6 +19,7 @@ from celery.signals import worker_shutting_down
 from lso.config import settings
 
 RUN_PLAYBOOK = "lso.tasks.run_playbook_proc_task"
+RUN_EXECUTABLE = "lso.tasks.run_executable_proc_task"
 
 celery = Celery(
     "lso-worker",
@@ -26,11 +27,6 @@ celery = Celery(
     backend=settings.CELERY_RESULT_BACKEND,
 )
 
-if settings.TESTING:
-    celery.conf.update(backend=settings.CELERY_RESULT_BACKEND, task_ignore_result=False)
-else:
-    celery.conf.update(task_ignore_result=True)
-
 celery.conf.update(
     result_expires=settings.CELERY_RESULT_EXPIRES,
     worker_prefetch_multiplier=1,
@@ -38,11 +34,13 @@ celery.conf.update(
     task_send_sent_event=True,
     redbeat_redis_url=settings.CELERY_BROKER_URL,
     broker_connection_retry_on_startup=True,
+    task_ignore_result=not settings.TESTING,
 )
 
 if settings.WORKER_QUEUE_NAME:
     celery.conf.task_routes = {
         RUN_PLAYBOOK: {"queue": settings.WORKER_QUEUE_NAME},
+        RUN_EXECUTABLE: {"queue": settings.WORKER_QUEUE_NAME},
     }
 
 

{orchestrator_lso-2.0.1.dist-info → orchestrator_lso-2.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: orchestrator-lso
-Version: 2.0.1
+Version: 2.1.0
 Summary: LSO, an API for remotely running Ansible playbooks.
 Author-email: GÉANT Orchestration and Automation Team <goat@geant.org>
 Requires-Python: >=3.11,<3.13
@@ -25,15 +25,16 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+License-File: LICENSE
 Requires-Dist: ansible-runner==2.4.0
-Requires-Dist: ansible==10.6.0
-Requires-Dist: fastapi==0.115.5
-Requires-Dist: httpx==0.28.0
-Requires-Dist: uvicorn[standard]==0.32.1
+Requires-Dist: ansible==10.7.0
+Requires-Dist: fastapi==0.115.8
+Requires-Dist: httpx==0.28.1
+Requires-Dist: uvicorn[standard]==0.34.0
 Requires-Dist: requests==2.32.3
-Requires-Dist: pydantic-settings==2.6.1
+Requires-Dist: pydantic-settings==2.7.1
 Requires-Dist: celery==5.4.0
-Requires-Dist: redis==5.2.0
+Requires-Dist: redis==5.2.1
 Requires-Dist: types-setuptools ; extra == "dev"
 Requires-Dist: types-requests ; extra == "dev"
 Requires-Dist: toml ; extra == "dev"

orchestrator_lso-2.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+lso/__init__.py,sha256=rvLpPFoeQhGsSfP4vjcdFBmHwJJhPVOzjwaMt-iNUfQ,1589
+lso/app.py,sha256=WDtlmjELIeFA437j-WPfqBQf6QT_l35LEABbEosHlqY,775
+lso/config.py,sha256=fceYs85UGN76GRyGs4x6TiKhktv-ykjc6-WocPRmFGY,1639
+lso/environment.py,sha256=iZ3DmsSKAC5a7VNL-HfJOJZ0sQwUMf7ZzNGC34B2CG0,1771
+lso/execute.py,sha256=BZeR0NYLZaLE28K7IcQ6JA7opdJOZrVK7rsSZyArsBo,1208
+lso/playbook.py,sha256=NHCeVttY5u1xGAdty954SZEYLcZEGBbBfjPuvBCgUwI,2409
+lso/tasks.py,sha256=ToHEDEqGipMUYk7sY8qZzrYV5dWwM8nekl2OGiEVVko,4509
+lso/utils.py,sha256=8B3_p5nliD0KQcU2dLRdbEa7LvDqtNH0ZViMJ4yB4gE,458
+lso/worker.py,sha256=61TXUefv8mGMq9LsD419r0O9Qxpa-WAtgSfu7SPEg44,1727
+lso/routes/__init__.py,sha256=1kRrth9zkFgmj6LChujieYJq5cjIETeTGXa1G70pduk,639
+lso/routes/default.py,sha256=a7STN1BJyFVizXUzmqKuADO0fpE1SHun-PzaZ-jx1wU,1438
+lso/routes/execute.py,sha256=8WACuI8QN8piI4pkHNtxARB8l5QcBKWSZ3qE_LWqBfU,1896
+lso/routes/playbook.py,sha256=YNPZGuJXh5JkDWJ-sVRVE-cTg9eEm6BPjNThFLDqxeM,4904
+orchestrator_lso-2.1.0.dist-info/licenses/LICENSE,sha256=CgFXf7XbZXJADozQIw2uUmmvU-zwAwXo4u7cgDfx3rE,10744
+orchestrator_lso-2.1.0.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
+orchestrator_lso-2.1.0.dist-info/METADATA,sha256=TlUqzxd8VwnU0I7RgS4xFxi2XfqAM2yabT3a0Zo2Hcs,6351
+orchestrator_lso-2.1.0.dist-info/RECORD,,

{orchestrator_lso-2.0.1.dist-info → orchestrator_lso-2.1.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.10.1
+Generator: flit 3.11.0
 Root-Is-Purelib: true
 Tag: py3-none-any

orchestrator_lso-2.0.1.dist-info/RECORD

@@ -1,14 +0,0 @@
-lso/__init__.py,sha256=Q6PrVmj3JOVZ4MJ9MYMVEI8L8w0GZy0EQZb4baSbzwA,1465
-lso/app.py,sha256=WDtlmjELIeFA437j-WPfqBQf6QT_l35LEABbEosHlqY,775
-lso/config.py,sha256=BXUl8SgYdaXbrmmqBRgxobGboCn02qfp_wdvZHZr_Gk,1627
-lso/environment.py,sha256=iZ3DmsSKAC5a7VNL-HfJOJZ0sQwUMf7ZzNGC34B2CG0,1771
-lso/playbook.py,sha256=PRnZMa93FZ-3s1pn2B4p6PBii-VN-Ti8DyhuMd8AI0I,2766
-lso/tasks.py,sha256=d_JRESlfs2dw-KZMLu2F21VHOz7l-5FhrhTDX58Pn38,2484
-lso/worker.py,sha256=3Y32F2DknsVK3JY4M6hDk35lvn7xLHqJhAIIbvxYIFE,1730
-lso/routes/__init__.py,sha256=1kRrth9zkFgmj6LChujieYJq5cjIETeTGXa1G70pduk,639
-lso/routes/default.py,sha256=a7STN1BJyFVizXUzmqKuADO0fpE1SHun-PzaZ-jx1wU,1438
-lso/routes/playbook.py,sha256=Q0Q-9fLOYpahgC7WJ5SEFt9P2NJnAKdQg3onjVnd6js,4903
-orchestrator_lso-2.0.1.dist-info/LICENSE,sha256=CgFXf7XbZXJADozQIw2uUmmvU-zwAwXo4u7cgDfx3rE,10744
-orchestrator_lso-2.0.1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-orchestrator_lso-2.0.1.dist-info/METADATA,sha256=Ex0bEBcEdeOPYt-mu1BbbkF2_Rqct3iww2u4nlQEGJk,6329
-orchestrator_lso-2.0.1.dist-info/RECORD,,