orchestrator-lso 2.1.1__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.bumpversion.cfg +1 -1
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/Dockerfile.example +1 -1
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/PKG-INFO +4 -4
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/__init__.py +1 -1
- orchestrator_lso-2.2.0/lso/execute.py +59 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/routes/execute.py +14 -12
- orchestrator_lso-2.2.0/lso/schema.py +50 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/tasks.py +29 -50
- orchestrator_lso-2.2.0/lso/utils.py +29 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/pyproject.toml +3 -3
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/test/routes/test_execute.py +64 -0
- orchestrator_lso-2.2.0/test/test_execute.py +126 -0
- orchestrator_lso-2.1.1/lso/execute.py +0 -31
- orchestrator_lso-2.1.1/lso/utils.py +0 -16
- orchestrator_lso-2.1.1/test/test_execute.py +0 -27
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.github/dependabot.yml +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.github/styles/config/vocabularies/Sphinx/accept.txt +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.github/styles/config/vocabularies/jargon/accept.txt +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.github/workflows/publish-package.yaml +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.github/workflows/run-linting-tests.yaml +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.github/workflows/run-unit-tests.yaml +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.github/workflows/sphinx.yaml +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.gitignore +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/.vale.ini +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/LICENSE +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/README.md +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/LSO_banner.jpg +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/Makefile +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/_static/custom.css +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/_static/lso_logo.png +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/conf.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/index.rst +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/module/config.rst +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/module/playbook.rst +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/module/routes/default.rst +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/module/routes/index.rst +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/module/routes/playbook.rst +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/docs/source/modules.rst +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/env.example +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/app.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/config.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/environment.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/playbook.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/routes/__init__.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/routes/default.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/routes/playbook.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/lso/worker.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/setup.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/test/__init__.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/test/conftest.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/test/routes/__init__.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/test/routes/test_default.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/test/routes/test_playbook.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/test/test-playbook.yaml +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/test/test_playbook.py +0 -0
- {orchestrator_lso-2.1.1 → orchestrator_lso-2.2.0}/test/utils.py +0 -0
|
@@ -11,7 +11,7 @@ COPY ./ansible-galaxy-requirements.yaml ./ansible-galaxy-requirements.yaml
|
|
|
11
11
|
RUN apk add --update --no-cache gcc libc-dev libffi-dev openssh
|
|
12
12
|
|
|
13
13
|
# Install the LSO python package, and additional requirements
|
|
14
|
-
RUN pip install orchestrator-lso=="2.
|
|
14
|
+
RUN pip install orchestrator-lso=="2.2.0"
|
|
15
15
|
RUN pip install -r requirements.txt
|
|
16
16
|
|
|
17
17
|
# Install required Ansible Galaxy roles and collections
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: orchestrator-lso
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: LSO, an API for remotely running Ansible playbooks.
|
|
5
5
|
Author-email: GÉANT Orchestration and Automation Team <goat@geant.org>
|
|
6
6
|
Requires-Python: >=3.11,<3.13
|
|
@@ -30,10 +30,10 @@ Requires-Dist: ansible-runner==2.4.1
|
|
|
30
30
|
Requires-Dist: ansible==10.7.0
|
|
31
31
|
Requires-Dist: fastapi==0.115.12
|
|
32
32
|
Requires-Dist: httpx==0.28.1
|
|
33
|
-
Requires-Dist: uvicorn[standard]==0.34.
|
|
34
|
-
Requires-Dist: requests==2.32.
|
|
33
|
+
Requires-Dist: uvicorn[standard]==0.34.3
|
|
34
|
+
Requires-Dist: requests==2.32.4
|
|
35
35
|
Requires-Dist: pydantic-settings==2.9.1
|
|
36
|
-
Requires-Dist: celery==5.5.
|
|
36
|
+
Requires-Dist: celery==5.5.3
|
|
37
37
|
Requires-Dist: redis==5.2.1
|
|
38
38
|
Requires-Dist: types-setuptools ; extra == "dev"
|
|
39
39
|
Requires-Dist: types-requests ; extra == "dev"
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Module for handling the execution of arbitrary executables."""
|
|
2
|
+
|
|
3
|
+
import subprocess # noqa: S404
|
|
4
|
+
import uuid
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from pydantic import HttpUrl
|
|
8
|
+
|
|
9
|
+
from lso.config import ExecutorType, settings
|
|
10
|
+
from lso.schema import ExecutionResult
|
|
11
|
+
from lso.tasks import run_executable_proc_task
|
|
12
|
+
from lso.utils import get_thread_pool
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_executable_path(executable_name: Path) -> Path:
|
|
16
|
+
"""Return the full path of an executable, based on the configured EXECUTABLES_ROOT_DIR."""
|
|
17
|
+
return Path(settings.EXECUTABLES_ROOT_DIR) / executable_name
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def run_executable_async(executable_path: Path, args: list[str], callback: HttpUrl | None) -> uuid.UUID:
|
|
21
|
+
"""Dispatch the task for executing an arbitrary executable remotely.
|
|
22
|
+
|
|
23
|
+
Uses a ThreadPoolExecutor (for local execution) or a Celery worker (for distributed tasks).
|
|
24
|
+
"""
|
|
25
|
+
job_id = uuid.uuid4()
|
|
26
|
+
callback_url = str(callback) if callback else None
|
|
27
|
+
if settings.EXECUTOR == ExecutorType.THREADPOOL:
|
|
28
|
+
executor = get_thread_pool()
|
|
29
|
+
future = executor.submit(run_executable_proc_task, str(job_id), str(executable_path), args, callback_url)
|
|
30
|
+
if settings.TESTING:
|
|
31
|
+
future.result()
|
|
32
|
+
elif settings.EXECUTOR == ExecutorType.WORKER:
|
|
33
|
+
run_executable_proc_task.delay(str(job_id), str(executable_path), args, callback_url)
|
|
34
|
+
return job_id
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def run_executable_sync(executable_path: str, args: list[str]) -> ExecutionResult:
|
|
38
|
+
"""Run the given executable synchronously and return the result."""
|
|
39
|
+
try:
|
|
40
|
+
result = subprocess.run( # noqa: S603
|
|
41
|
+
[executable_path, *args],
|
|
42
|
+
text=True,
|
|
43
|
+
capture_output=True,
|
|
44
|
+
timeout=settings.EXECUTABLE_TIMEOUT_SEC,
|
|
45
|
+
check=False,
|
|
46
|
+
)
|
|
47
|
+
output = result.stdout + result.stderr
|
|
48
|
+
return_code = result.returncode
|
|
49
|
+
except subprocess.TimeoutExpired:
|
|
50
|
+
output = "Execution timed out."
|
|
51
|
+
return_code = -1
|
|
52
|
+
except Exception as e: # noqa: BLE001
|
|
53
|
+
output = str(e)
|
|
54
|
+
return_code = -1
|
|
55
|
+
|
|
56
|
+
return ExecutionResult( # type: ignore[call-arg]
|
|
57
|
+
output=output,
|
|
58
|
+
return_code=return_code,
|
|
59
|
+
)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""FastAPI route for running arbitrary executables."""
|
|
2
2
|
|
|
3
|
+
import asyncio
|
|
3
4
|
import uuid
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
from typing import Annotated
|
|
@@ -7,7 +8,8 @@ from typing import Annotated
|
|
|
7
8
|
from fastapi import APIRouter, HTTPException, status
|
|
8
9
|
from pydantic import AfterValidator, BaseModel, HttpUrl
|
|
9
10
|
|
|
10
|
-
from lso.execute import get_executable_path,
|
|
11
|
+
from lso.execute import get_executable_path, run_executable_async, run_executable_sync
|
|
12
|
+
from lso.schema import ExecutableRunResponse
|
|
11
13
|
|
|
12
14
|
router = APIRouter()
|
|
13
15
|
|
|
@@ -35,17 +37,17 @@ class ExecutableRunParams(BaseModel):
|
|
|
35
37
|
|
|
36
38
|
executable_name: ExecutableName
|
|
37
39
|
args: list[str] = []
|
|
38
|
-
callback: HttpUrl
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
class ExecutableRunResponse(BaseModel):
|
|
42
|
-
"""Response for running an arbitrary executable."""
|
|
43
|
-
|
|
44
|
-
job_id: uuid.UUID
|
|
40
|
+
callback: HttpUrl | None = None
|
|
41
|
+
is_async: bool = True
|
|
45
42
|
|
|
46
43
|
|
|
47
44
|
@router.post("/", response_model=ExecutableRunResponse, status_code=status.HTTP_201_CREATED)
|
|
48
|
-
def run_executable_endpoint(params: ExecutableRunParams) -> ExecutableRunResponse:
|
|
49
|
-
"""Dispatch
|
|
50
|
-
|
|
51
|
-
|
|
45
|
+
async def run_executable_endpoint(params: ExecutableRunParams) -> ExecutableRunResponse:
|
|
46
|
+
"""Dispatch a task to run an arbitrary executable."""
|
|
47
|
+
if params.is_async:
|
|
48
|
+
job_id = run_executable_async(params.executable_name, params.args, params.callback)
|
|
49
|
+
return ExecutableRunResponse(job_id=job_id)
|
|
50
|
+
|
|
51
|
+
job_id = uuid.uuid4()
|
|
52
|
+
result = await asyncio.to_thread(run_executable_sync, str(params.executable_name), params.args)
|
|
53
|
+
return ExecutableRunResponse(job_id=job_id, result=result)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Copyright 2024-2025 GÉANT Vereniging.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
|
|
14
|
+
"""Module for defining the schema for running arbitrary executables."""
|
|
15
|
+
|
|
16
|
+
import uuid
|
|
17
|
+
from enum import StrEnum
|
|
18
|
+
|
|
19
|
+
from pydantic import BaseModel, model_validator
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class JobStatus(StrEnum):
|
|
23
|
+
"""Enumeration of possible job statuses."""
|
|
24
|
+
|
|
25
|
+
SUCCESSFUL = "successful"
|
|
26
|
+
FAILED = "failed"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ExecutionResult(BaseModel):
|
|
30
|
+
"""Model for capturing the result of an executable run."""
|
|
31
|
+
|
|
32
|
+
output: str
|
|
33
|
+
return_code: int
|
|
34
|
+
status: JobStatus
|
|
35
|
+
|
|
36
|
+
@model_validator(mode="before")
|
|
37
|
+
def populate_status(cls, values: dict) -> dict:
|
|
38
|
+
"""Set the status based on the return code."""
|
|
39
|
+
rc = values.get("return_code")
|
|
40
|
+
if rc is not None:
|
|
41
|
+
values["status"] = JobStatus.SUCCESSFUL if rc == 0 else JobStatus.FAILED
|
|
42
|
+
|
|
43
|
+
return values
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class ExecutableRunResponse(BaseModel):
|
|
47
|
+
"""Response for running an arbitrary executable."""
|
|
48
|
+
|
|
49
|
+
job_id: uuid.UUID
|
|
50
|
+
result: ExecutionResult | None = None
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2024-2025 GÉANT Vereniging.
|
|
2
2
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
3
|
# you may not use this file except in compliance with the License.
|
|
4
4
|
# You may obtain a copy of the License at
|
|
@@ -18,15 +18,15 @@ the results to a specified callback URL.
|
|
|
18
18
|
"""
|
|
19
19
|
|
|
20
20
|
import logging
|
|
21
|
-
import subprocess # noqa: S404
|
|
22
|
-
from enum import StrEnum
|
|
23
21
|
from typing import Any
|
|
22
|
+
from uuid import UUID
|
|
24
23
|
|
|
25
24
|
import ansible_runner
|
|
26
25
|
import requests
|
|
27
26
|
from starlette import status
|
|
28
27
|
|
|
29
28
|
from lso.config import settings
|
|
29
|
+
from lso.schema import ExecutableRunResponse
|
|
30
30
|
from lso.worker import RUN_EXECUTABLE, RUN_PLAYBOOK, celery
|
|
31
31
|
|
|
32
32
|
logger = logging.getLogger(__name__)
|
|
@@ -36,13 +36,6 @@ class CallbackFailedError(Exception):
|
|
|
36
36
|
"""Exception raised when a callback url can't be reached."""
|
|
37
37
|
|
|
38
38
|
|
|
39
|
-
class JobStatus(StrEnum):
|
|
40
|
-
"""Enumeration of possible job statuses."""
|
|
41
|
-
|
|
42
|
-
SUCCESSFUL = "successful"
|
|
43
|
-
FAILED = "failed"
|
|
44
|
-
|
|
45
|
-
|
|
46
39
|
@celery.task(name=RUN_PLAYBOOK) # type: ignore[misc]
|
|
47
40
|
def run_playbook_proc_task(
|
|
48
41
|
job_id: str, playbook_path: str, extra_vars: dict[str, Any], inventory: dict[str, Any] | str, callback: str
|
|
@@ -74,48 +67,34 @@ def run_playbook_proc_task(
|
|
|
74
67
|
|
|
75
68
|
|
|
76
69
|
@celery.task(name=RUN_EXECUTABLE) # type: ignore[misc]
|
|
77
|
-
def run_executable_proc_task(job_id: str, executable_path: str, args: list[str], callback: str) -> None:
|
|
70
|
+
def run_executable_proc_task(job_id: str, executable_path: str, args: list[str], callback: str | None) -> None:
|
|
78
71
|
"""Celery task to run an arbitrary executable and notify via callback.
|
|
79
72
|
|
|
80
|
-
Executes the executable with the provided arguments and posts back the
|
|
73
|
+
Executes the executable with the provided arguments and posts back the result if a callback URL is provided.
|
|
81
74
|
"""
|
|
75
|
+
from lso.execute import run_executable_sync # noqa: PLC0415
|
|
76
|
+
|
|
82
77
|
msg = f"Executing executable: {executable_path} with args: {args}, callback: {callback}"
|
|
83
78
|
logger.info(msg)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
def _raise_callback_error(message: str, error: Exception | None = None) -> None:
|
|
109
|
-
if error:
|
|
110
|
-
raise CallbackFailedError(message) from error
|
|
111
|
-
raise CallbackFailedError(message)
|
|
112
|
-
|
|
113
|
-
try:
|
|
114
|
-
response = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
|
|
115
|
-
if not (status.HTTP_200_OK <= response.status_code < status.HTTP_300_MULTIPLE_CHOICES):
|
|
116
|
-
msg = f"Callback failed: {response.text}, url: {callback}"
|
|
117
|
-
_raise_callback_error(msg)
|
|
118
|
-
except Exception as e:
|
|
119
|
-
error_msg = f"Callback error: {e}"
|
|
120
|
-
logger.exception(error_msg)
|
|
121
|
-
_raise_callback_error(error_msg, e)
|
|
79
|
+
result = run_executable_sync(executable_path, args)
|
|
80
|
+
|
|
81
|
+
if callback:
|
|
82
|
+
payload = ExecutableRunResponse(
|
|
83
|
+
job_id=UUID(job_id),
|
|
84
|
+
result=result,
|
|
85
|
+
).model_dump(mode="json")
|
|
86
|
+
|
|
87
|
+
def _raise_callback_error(message: str, error: Exception | None = None) -> None:
|
|
88
|
+
if error:
|
|
89
|
+
raise CallbackFailedError(message) from error
|
|
90
|
+
raise CallbackFailedError(message)
|
|
91
|
+
|
|
92
|
+
try:
|
|
93
|
+
response = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
|
|
94
|
+
if not (status.HTTP_200_OK <= response.status_code < status.HTTP_300_MULTIPLE_CHOICES):
|
|
95
|
+
msg = f"Callback failed: {response.text}, url: {callback}"
|
|
96
|
+
_raise_callback_error(msg)
|
|
97
|
+
except Exception as e:
|
|
98
|
+
error_msg = f"Callback error: {e}"
|
|
99
|
+
logger.exception(error_msg)
|
|
100
|
+
_raise_callback_error(error_msg, e)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Copyright 2024-2025 GÉANT Vereniging.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
|
|
14
|
+
"""Utility functions for the LSO package."""
|
|
15
|
+
|
|
16
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
17
|
+
|
|
18
|
+
from lso.config import settings
|
|
19
|
+
|
|
20
|
+
_executor = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def get_thread_pool() -> ThreadPoolExecutor:
|
|
24
|
+
"""Initialize or return a cached ThreadPoolExecutor for local asynchronous execution."""
|
|
25
|
+
global _executor # noqa: PLW0603
|
|
26
|
+
if _executor is None:
|
|
27
|
+
_executor = ThreadPoolExecutor(max_workers=settings.MAX_THREAD_POOL_WORKERS)
|
|
28
|
+
|
|
29
|
+
return _executor
|
|
@@ -33,10 +33,10 @@ dependencies = [
|
|
|
33
33
|
"ansible==10.7.0",
|
|
34
34
|
"fastapi==0.115.12",
|
|
35
35
|
"httpx==0.28.1",
|
|
36
|
-
"uvicorn[standard]==0.34.
|
|
37
|
-
"requests==2.32.
|
|
36
|
+
"uvicorn[standard]==0.34.3",
|
|
37
|
+
"requests==2.32.4",
|
|
38
38
|
"pydantic-settings==2.9.1",
|
|
39
|
-
"celery==5.5.
|
|
39
|
+
"celery==5.5.3",
|
|
40
40
|
"redis==5.2.1",
|
|
41
41
|
]
|
|
42
42
|
|
|
@@ -7,6 +7,7 @@ from fastapi import status
|
|
|
7
7
|
from fastapi.testclient import TestClient
|
|
8
8
|
|
|
9
9
|
from lso.config import ExecutorType
|
|
10
|
+
from lso.schema import JobStatus
|
|
10
11
|
from test.utils import temp_executable_env
|
|
11
12
|
|
|
12
13
|
TEST_CALLBACK_URL = "http://localhost/callback"
|
|
@@ -121,3 +122,66 @@ def test_execute_endpoint_not_a_file(client: TestClient):
|
|
|
121
122
|
response = client.post("/api/execute/", json=params)
|
|
122
123
|
assert response.status_code == status.HTTP_404_NOT_FOUND
|
|
123
124
|
assert response.json() == {"detail": "Executable 'not_a_file' is not a valid file."}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@responses.activate
|
|
128
|
+
def test_execute_endpoint_async_and_sync(client: TestClient, temp_executable: Path):
|
|
129
|
+
"""Endpoint: async missing callback →422; async with callback →201+job_id; sync →201+job_id+result."""
|
|
130
|
+
with temp_executable_env(ExecutorType.THREADPOOL) as exec_dir:
|
|
131
|
+
target_exe = exec_dir / temp_executable.name
|
|
132
|
+
target_exe.write_text(temp_executable.read_text())
|
|
133
|
+
target_exe.chmod(0o755)
|
|
134
|
+
|
|
135
|
+
responses.add(responses.POST, TEST_CALLBACK_URL, status=200)
|
|
136
|
+
|
|
137
|
+
# async without callback
|
|
138
|
+
r1 = client.post("/api/execute/", json={"executable_name": temp_executable.name})
|
|
139
|
+
assert r1.status_code == status.HTTP_201_CREATED
|
|
140
|
+
responses.assert_call_count(TEST_CALLBACK_URL, 0)
|
|
141
|
+
|
|
142
|
+
# async with callback
|
|
143
|
+
r2 = client.post("/api/execute/", json={"executable_name": temp_executable.name, "callback": TEST_CALLBACK_URL})
|
|
144
|
+
assert r2.status_code == status.HTTP_201_CREATED
|
|
145
|
+
d2 = r2.json()
|
|
146
|
+
assert "job_id" in d2
|
|
147
|
+
assert d2.get("result") is None
|
|
148
|
+
responses.assert_call_count(TEST_CALLBACK_URL, 1)
|
|
149
|
+
|
|
150
|
+
# sync path
|
|
151
|
+
r3 = client.post("/api/execute/", json={"executable_name": temp_executable.name, "args": [], "is_async": False})
|
|
152
|
+
assert r3.status_code == status.HTTP_201_CREATED
|
|
153
|
+
d3 = r3.json()
|
|
154
|
+
assert "job_id" in d3
|
|
155
|
+
assert "result" in d3
|
|
156
|
+
res = d3["result"]
|
|
157
|
+
assert res["return_code"] == 0
|
|
158
|
+
assert res["status"] == JobStatus.SUCCESSFUL
|
|
159
|
+
assert "Executable Test" in res["output"]
|
|
160
|
+
responses.assert_call_count(TEST_CALLBACK_URL, 1)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@responses.activate
|
|
164
|
+
def test_execute_endpoint_sync_ignores_callback(client: TestClient, temp_executable: Path):
|
|
165
|
+
"""When is_async=False, callback is accepted but not invoked."""
|
|
166
|
+
with temp_executable_env(ExecutorType.THREADPOOL) as exec_dir:
|
|
167
|
+
target_exe = exec_dir / temp_executable.name
|
|
168
|
+
target_exe.write_text(temp_executable.read_text())
|
|
169
|
+
target_exe.chmod(0o755)
|
|
170
|
+
|
|
171
|
+
# prepare a bogus callback just to see if it's ever called
|
|
172
|
+
responses.add(responses.POST, TEST_CALLBACK_URL, status=200)
|
|
173
|
+
|
|
174
|
+
rv = client.post(
|
|
175
|
+
"/api/execute/",
|
|
176
|
+
json={
|
|
177
|
+
"executable_name": temp_executable.name,
|
|
178
|
+
"is_async": False,
|
|
179
|
+
"callback": TEST_CALLBACK_URL,
|
|
180
|
+
},
|
|
181
|
+
)
|
|
182
|
+
assert rv.status_code == status.HTTP_201_CREATED
|
|
183
|
+
data = rv.json()
|
|
184
|
+
# result must be present
|
|
185
|
+
assert data["result"] is not None
|
|
186
|
+
# and no callback should have been invoked:
|
|
187
|
+
responses.assert_call_count(TEST_CALLBACK_URL, 0)
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import subprocess # noqa: S404
|
|
2
|
+
import uuid
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
import responses
|
|
7
|
+
|
|
8
|
+
from lso.config import ExecutorType
|
|
9
|
+
from lso.execute import get_executable_path, run_executable_async, run_executable_sync
|
|
10
|
+
from lso.schema import JobStatus
|
|
11
|
+
from lso.tasks import CallbackFailedError
|
|
12
|
+
from test.utils import temp_executable_env
|
|
13
|
+
|
|
14
|
+
TEST_CALLBACK_URL = "http://localhost/callback"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@responses.activate
|
|
18
|
+
def test_run_executable_async_threadpool_success(temp_executable: Path):
|
|
19
|
+
"""ThreadPool mode: successful async run invokes callback once."""
|
|
20
|
+
with temp_executable_env(ExecutorType.THREADPOOL) as exec_dir:
|
|
21
|
+
target_exe = exec_dir / temp_executable.name
|
|
22
|
+
target_exe.write_text(temp_executable.read_text())
|
|
23
|
+
target_exe.chmod(0o755)
|
|
24
|
+
|
|
25
|
+
responses.add(responses.POST, TEST_CALLBACK_URL, status=200)
|
|
26
|
+
|
|
27
|
+
job_id = run_executable_async(target_exe, ["--version"], TEST_CALLBACK_URL)
|
|
28
|
+
assert isinstance(job_id, uuid.UUID)
|
|
29
|
+
responses.assert_call_count(TEST_CALLBACK_URL, 1)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@responses.activate
|
|
33
|
+
def test_run_executable_async_threadpool_callback_failure(temp_executable: Path):
|
|
34
|
+
"""ThreadPool mode: non-2xx callback raises CallbackFailedError."""
|
|
35
|
+
with temp_executable_env(ExecutorType.THREADPOOL) as exec_dir:
|
|
36
|
+
target_exe = exec_dir / temp_executable.name
|
|
37
|
+
target_exe.write_text(temp_executable.read_text())
|
|
38
|
+
target_exe.chmod(0o755)
|
|
39
|
+
|
|
40
|
+
responses.add(responses.POST, TEST_CALLBACK_URL, status=500)
|
|
41
|
+
|
|
42
|
+
with pytest.raises(CallbackFailedError) as exc:
|
|
43
|
+
run_executable_async(target_exe, [], TEST_CALLBACK_URL)
|
|
44
|
+
|
|
45
|
+
assert f"Callback failed: , url: {TEST_CALLBACK_URL}" in str(exc.value)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_run_executable_async_worker_delay(monkeypatch, temp_executable: Path):
|
|
49
|
+
"""Worker mode: schedules Celery .delay without HTTP calls."""
|
|
50
|
+
with temp_executable_env(ExecutorType.WORKER) as exec_dir:
|
|
51
|
+
target_exe = exec_dir / temp_executable.name
|
|
52
|
+
target_exe.write_text(temp_executable.read_text())
|
|
53
|
+
target_exe.chmod(0o755)
|
|
54
|
+
|
|
55
|
+
calls = []
|
|
56
|
+
import lso.execute as exec_mod # noqa: PLC0415
|
|
57
|
+
|
|
58
|
+
monkeypatch.setattr(
|
|
59
|
+
exec_mod.run_executable_proc_task,
|
|
60
|
+
"delay",
|
|
61
|
+
lambda job_id, path, args, callback: calls.append((job_id, path, args, callback)),
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
job_id = run_executable_async(target_exe, ["a", "b"], TEST_CALLBACK_URL)
|
|
65
|
+
assert isinstance(job_id, uuid.UUID)
|
|
66
|
+
assert len(calls) == 1
|
|
67
|
+
called_job_id, called_path, called_args, called_callback = calls[0]
|
|
68
|
+
assert called_job_id == str(job_id)
|
|
69
|
+
assert called_path == str(target_exe)
|
|
70
|
+
assert called_args == ["a", "b"]
|
|
71
|
+
assert called_callback == TEST_CALLBACK_URL
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@pytest.mark.parametrize("args", [[], ["x"], ["1", "2", "3"]])
|
|
75
|
+
def test_run_executable_sync_various_args(temp_executable: Path, args):
|
|
76
|
+
"""Sync helper accepts different arg lists and returns correct status."""
|
|
77
|
+
# success exit
|
|
78
|
+
result = run_executable_sync(str(temp_executable), args)
|
|
79
|
+
assert result.return_code == 0
|
|
80
|
+
assert "Executable Test" in result.output
|
|
81
|
+
assert result.status == JobStatus.SUCCESSFUL
|
|
82
|
+
|
|
83
|
+
# failure exit
|
|
84
|
+
fail = temp_executable.with_name("fail.sh")
|
|
85
|
+
fail.write_text("#!/bin/sh\nexit 1\n")
|
|
86
|
+
fail.chmod(0o755)
|
|
87
|
+
result2 = run_executable_sync(str(fail), [])
|
|
88
|
+
assert result2.return_code == 1
|
|
89
|
+
assert result2.status == JobStatus.FAILED
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_get_executable_path_under_root():
|
|
93
|
+
"""get_executable_path returns a path under EXECUTABLES_ROOT_DIR."""
|
|
94
|
+
with temp_executable_env(ExecutorType.THREADPOOL):
|
|
95
|
+
p = get_executable_path(Path("foo.sh"))
|
|
96
|
+
assert Path(p).parent.exists()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_run_executable_sync_timeout(monkeypatch):
|
|
100
|
+
"""If subprocess.run raises TimeoutExpired, we get return_code=-1 and the timeout message."""
|
|
101
|
+
|
|
102
|
+
# simulate subprocess.run raising TimeoutExpired
|
|
103
|
+
def fake_run(*args, **kwargs): # noqa: ARG001
|
|
104
|
+
raise subprocess.TimeoutExpired(cmd=kwargs.get("args", []), timeout=0)
|
|
105
|
+
|
|
106
|
+
monkeypatch.setattr(subprocess, "run", fake_run)
|
|
107
|
+
|
|
108
|
+
result = run_executable_sync("/does/not/matter", [])
|
|
109
|
+
assert result.return_code == -1
|
|
110
|
+
assert result.output == "Execution timed out."
|
|
111
|
+
assert result.status == JobStatus.FAILED
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_run_executable_sync_exception(monkeypatch):
|
|
115
|
+
"""If subprocess.run raises a generic Exception, it's captured as output and rc=-1."""
|
|
116
|
+
|
|
117
|
+
def fake_run(*args, **kwargs): # noqa: ARG001
|
|
118
|
+
msg = "boom!"
|
|
119
|
+
raise RuntimeError(msg)
|
|
120
|
+
|
|
121
|
+
monkeypatch.setattr(subprocess, "run", fake_run)
|
|
122
|
+
|
|
123
|
+
result = run_executable_sync("/does/not/matter", ["x"])
|
|
124
|
+
assert result.return_code == -1
|
|
125
|
+
assert "boom!" in result.output
|
|
126
|
+
assert result.status == JobStatus.FAILED
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
"""Module for handling the execution of arbitrary executables."""
|
|
2
|
-
|
|
3
|
-
import uuid
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
6
|
-
from pydantic import HttpUrl
|
|
7
|
-
|
|
8
|
-
from lso.config import ExecutorType, settings
|
|
9
|
-
from lso.tasks import run_executable_proc_task
|
|
10
|
-
from lso.utils import get_thread_pool
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def get_executable_path(executable_name: Path) -> Path:
|
|
14
|
-
"""Return the full path of an executable, based on the configured EXECUTABLES_ROOT_DIR."""
|
|
15
|
-
return Path(settings.EXECUTABLES_ROOT_DIR) / executable_name
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def run_executable(executable_path: Path, args: list[str], callback: HttpUrl) -> uuid.UUID:
|
|
19
|
-
"""Dispatch the task for executing an arbitrary executable remotely.
|
|
20
|
-
|
|
21
|
-
Uses a ThreadPoolExecutor (for local execution) or a Celery worker (for distributed tasks).
|
|
22
|
-
"""
|
|
23
|
-
job_id = uuid.uuid4()
|
|
24
|
-
if settings.EXECUTOR == ExecutorType.THREADPOOL:
|
|
25
|
-
executor = get_thread_pool()
|
|
26
|
-
future = executor.submit(run_executable_proc_task, str(job_id), str(executable_path), args, str(callback))
|
|
27
|
-
if settings.TESTING:
|
|
28
|
-
future.result()
|
|
29
|
-
elif settings.EXECUTOR == ExecutorType.WORKER:
|
|
30
|
-
run_executable_proc_task.delay(str(job_id), str(executable_path), args, str(callback))
|
|
31
|
-
return job_id
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
"""Utility functions for the LSO package."""
|
|
2
|
-
|
|
3
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
4
|
-
|
|
5
|
-
from lso.config import settings
|
|
6
|
-
|
|
7
|
-
_executor = None
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def get_thread_pool() -> ThreadPoolExecutor:
|
|
11
|
-
"""Initialize or return a cached ThreadPoolExecutor for local asynchronous execution."""
|
|
12
|
-
global _executor # noqa: PLW0603
|
|
13
|
-
if _executor is None:
|
|
14
|
-
_executor = ThreadPoolExecutor(max_workers=settings.MAX_THREAD_POOL_WORKERS)
|
|
15
|
-
|
|
16
|
-
return _executor
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import uuid
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
|
-
import responses
|
|
5
|
-
|
|
6
|
-
from lso.config import ExecutorType
|
|
7
|
-
from lso.execute import run_executable
|
|
8
|
-
from test.utils import temp_executable_env
|
|
9
|
-
|
|
10
|
-
TEST_CALLBACK_URL = "http://localhost/callback"
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
@responses.activate
|
|
14
|
-
def test_run_executable_threadpool(temp_executable: Path):
|
|
15
|
-
"""
|
|
16
|
-
Test direct invocation of run_executable using the ThreadPool executor.
|
|
17
|
-
"""
|
|
18
|
-
with temp_executable_env(ExecutorType.THREADPOOL) as exec_dir:
|
|
19
|
-
target_exe = exec_dir / temp_executable.name
|
|
20
|
-
target_exe.write_text(temp_executable.read_text())
|
|
21
|
-
target_exe.chmod(0o755)
|
|
22
|
-
|
|
23
|
-
# Simulate a successful callback.
|
|
24
|
-
responses.add(responses.POST, TEST_CALLBACK_URL, status=200)
|
|
25
|
-
|
|
26
|
-
job_id = run_executable(target_exe, ["--version"], TEST_CALLBACK_URL)
|
|
27
|
-
uuid.UUID(str(job_id)) # Validate job_id format.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|