orchestrator-lso 1.0.3__py3-none-any.whl → 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lso/__init__.py +7 -11
- lso/config.py +22 -44
- lso/playbook.py +29 -75
- lso/routes/playbook.py +38 -10
- lso/tasks.py +64 -0
- lso/worker.py +52 -0
- {orchestrator_lso-1.0.3.dist-info → orchestrator_lso-2.0.1.dist-info}/METADATA +36 -16
- orchestrator_lso-2.0.1.dist-info/RECORD +14 -0
- {orchestrator_lso-1.0.3.dist-info → orchestrator_lso-2.0.1.dist-info}/WHEEL +1 -1
- orchestrator_lso-1.0.3.dist-info/RECORD +0 -12
- {orchestrator_lso-1.0.3.dist-info → orchestrator_lso-2.0.1.dist-info}/LICENSE +0 -0
lso/__init__.py
CHANGED
|
@@ -13,24 +13,23 @@
|
|
|
13
13
|
|
|
14
14
|
"""LSO, an API for remotely running Ansible playbooks."""
|
|
15
15
|
|
|
16
|
-
__version__ = "
|
|
16
|
+
__version__ = "2.0.1"
|
|
17
17
|
|
|
18
18
|
import logging
|
|
19
19
|
|
|
20
20
|
from fastapi import FastAPI
|
|
21
21
|
from fastapi.middleware.cors import CORSMiddleware
|
|
22
22
|
|
|
23
|
-
from lso import
|
|
23
|
+
from lso import environment
|
|
24
24
|
from lso.routes.default import router as default_router
|
|
25
25
|
from lso.routes.playbook import router as playbook_router
|
|
26
26
|
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
27
28
|
|
|
28
|
-
def create_app() -> FastAPI:
|
|
29
|
-
"""Override default settings with those found in the file read from environment variable `SETTINGS_FILENAME`.
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
"""
|
|
33
|
-
app = FastAPI()
|
|
30
|
+
def create_app() -> FastAPI:
|
|
31
|
+
"""Initialise the :term:`LSO` app."""
|
|
32
|
+
app = FastAPI(docs_url="/api/doc", redoc_url="/api/redoc", openapi_url="/api/openapi.json")
|
|
34
33
|
|
|
35
34
|
app.add_middleware(
|
|
36
35
|
CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"]
|
|
@@ -39,11 +38,8 @@ def create_app() -> FastAPI:
|
|
|
39
38
|
app.include_router(default_router, prefix="/api")
|
|
40
39
|
app.include_router(playbook_router, prefix="/api/playbook")
|
|
41
40
|
|
|
42
|
-
# test that configuration parameters are loaded and available
|
|
43
|
-
config.load()
|
|
44
|
-
|
|
45
41
|
environment.setup_logging()
|
|
46
42
|
|
|
47
|
-
|
|
43
|
+
logger.info("FastAPI app initialized")
|
|
48
44
|
|
|
49
45
|
return app
|
lso/config.py
CHANGED
|
@@ -11,60 +11,38 @@
|
|
|
11
11
|
# See the License for the specific language governing permissions and
|
|
12
12
|
# limitations under the License.
|
|
13
13
|
|
|
14
|
-
"""
|
|
14
|
+
"""Module for loading and managing configuration settings for the LSO app.
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
Configuration file location can also be loaded from environment variable ``$SETTINGS_FILENAME``, which is default
|
|
18
|
-
behaviour in :func:`load`.
|
|
16
|
+
Uses `pydantic`'s `BaseSettings` to load settings from environment variables.
|
|
19
17
|
"""
|
|
20
18
|
|
|
21
|
-
import json
|
|
22
19
|
import os
|
|
23
|
-
from
|
|
20
|
+
from enum import Enum
|
|
24
21
|
|
|
25
|
-
import
|
|
26
|
-
from pydantic import BaseModel
|
|
22
|
+
from pydantic_settings import BaseSettings
|
|
27
23
|
|
|
28
|
-
CONFIG_SCHEMA = {
|
|
29
|
-
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
30
|
-
"type": "object",
|
|
31
|
-
"properties": {"ansible_playbooks_root_dir": {"type": "string"}},
|
|
32
|
-
"required": ["ansible_playbooks_root_dir"],
|
|
33
|
-
"additionalProperties": False,
|
|
34
|
-
}
|
|
35
|
-
DEFAULT_REQUEST_TIMEOUT = 10
|
|
36
24
|
|
|
25
|
+
class ExecutorType(Enum):
|
|
26
|
+
"""Enum representing the types of executors available for task execution."""
|
|
37
27
|
|
|
38
|
-
|
|
39
|
-
""
|
|
28
|
+
WORKER = "celery"
|
|
29
|
+
THREADPOOL = "threadpool"
|
|
40
30
|
|
|
41
|
-
Contains the root directory at which Ansible playbooks are present.
|
|
42
|
-
"""
|
|
43
31
|
|
|
44
|
-
|
|
32
|
+
class Config(BaseSettings):
|
|
33
|
+
"""The set of parameters required for running :term:`LSO`."""
|
|
45
34
|
|
|
35
|
+
TESTING: bool = False
|
|
36
|
+
ANSIBLE_PLAYBOOKS_ROOT_DIR: str = "/path/to/ansible/playbooks"
|
|
37
|
+
EXECUTOR: ExecutorType = ExecutorType.THREADPOOL
|
|
38
|
+
MAX_THREAD_POOL_WORKERS: int = min(32, (os.cpu_count() or 1) + 4)
|
|
39
|
+
REQUEST_TIMEOUT_SEC: int = 10
|
|
40
|
+
CELERY_BROKER_URL: str = "redis://localhost:6379/0"
|
|
41
|
+
CELERY_RESULT_BACKEND: str = "redis://localhost:6379/0"
|
|
42
|
+
CELERY_TIMEZONE: str = "Europe/Amsterdam"
|
|
43
|
+
CELERY_ENABLE_UTC: bool = True
|
|
44
|
+
CELERY_RESULT_EXPIRES: int = 3600
|
|
45
|
+
WORKER_QUEUE_NAME: str | None = None
|
|
46
46
|
|
|
47
|
-
def load_from_file(file: Path) -> Config:
|
|
48
|
-
"""Load, validate and return configuration parameters.
|
|
49
47
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
.. asjson:: lso.config.CONFIG_SCHEMA
|
|
53
|
-
|
|
54
|
-
:param file: :class:`Path` object that produces the configuration file.
|
|
55
|
-
:return: a dict containing the parsed configuration parameters.
|
|
56
|
-
"""
|
|
57
|
-
config = json.loads(file.read_text())
|
|
58
|
-
jsonschema.validate(config, CONFIG_SCHEMA)
|
|
59
|
-
return Config(**config)
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def load() -> Config:
|
|
63
|
-
"""Load a configuration file, located at the path specified in the environment variable ``$SETTINGS_FILENAME``.
|
|
64
|
-
|
|
65
|
-
Loading and validating the file is performed by :func:`load_from_file`.
|
|
66
|
-
|
|
67
|
-
:return: a dict containing the parsed configuration parameters
|
|
68
|
-
"""
|
|
69
|
-
assert "SETTINGS_FILENAME" in os.environ, "Environment variable SETTINGS_FILENAME not set" # noqa: S101
|
|
70
|
-
return load_from_file(Path(os.environ["SETTINGS_FILENAME"]))
|
|
48
|
+
settings = Config()
|
lso/playbook.py
CHANGED
|
@@ -13,73 +13,36 @@
|
|
|
13
13
|
|
|
14
14
|
"""Module that gathers common API responses and data models."""
|
|
15
15
|
|
|
16
|
-
import logging
|
|
17
|
-
import threading
|
|
18
16
|
import uuid
|
|
17
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
19
18
|
from pathlib import Path
|
|
20
19
|
from typing import Any
|
|
21
20
|
|
|
22
|
-
import ansible_runner
|
|
23
|
-
import requests
|
|
24
|
-
from fastapi import status
|
|
25
|
-
from fastapi.responses import JSONResponse
|
|
26
21
|
from pydantic import HttpUrl
|
|
27
22
|
|
|
28
|
-
from lso import
|
|
29
|
-
from lso.
|
|
23
|
+
from lso.config import ExecutorType, settings
|
|
24
|
+
from lso.tasks import run_playbook_proc_task
|
|
30
25
|
|
|
31
|
-
|
|
26
|
+
_executor = None
|
|
32
27
|
|
|
33
28
|
|
|
34
|
-
def
|
|
35
|
-
"""Get
|
|
36
|
-
config_params = config.load()
|
|
37
|
-
return Path(config_params.ansible_playbooks_root_dir) / playbook_name
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def playbook_launch_success(job_id: str) -> JSONResponse:
|
|
41
|
-
"""Return a :class:`PlaybookLaunchResponse` for the successful start of a playbook execution.
|
|
42
|
-
|
|
43
|
-
:return JSONResponse: A playbook launch response that's successful.
|
|
44
|
-
"""
|
|
45
|
-
return JSONResponse(content={"job_id": job_id}, status_code=status.HTTP_201_CREATED)
|
|
29
|
+
def get_thread_pool() -> ThreadPoolExecutor:
|
|
30
|
+
"""Get and optionally initialise a ThreadPoolExecutor.
|
|
46
31
|
|
|
32
|
+
Returns:
|
|
33
|
+
ThreadPoolExecutor
|
|
47
34
|
|
|
48
|
-
def playbook_launch_error(reason: str, status_code: int = status.HTTP_400_BAD_REQUEST) -> JSONResponse:
|
|
49
|
-
"""Return a :class:`PlaybookLaunchResponse` for the erroneous start of a playbook execution.
|
|
50
|
-
|
|
51
|
-
:param str reason: The reason why a request has failed.
|
|
52
|
-
:param status status_code: The HTTP status code that should be associated with this request. Defaults to HTTP 400:
|
|
53
|
-
Bad request.
|
|
54
|
-
:return JSONResponse: A playbook launch response that's unsuccessful.
|
|
55
35
|
"""
|
|
56
|
-
|
|
57
|
-
|
|
36
|
+
global _executor # noqa: PLW0603
|
|
37
|
+
if _executor is None:
|
|
38
|
+
_executor = ThreadPoolExecutor(max_workers=settings.MAX_THREAD_POOL_WORKERS)
|
|
58
39
|
|
|
59
|
-
|
|
60
|
-
job_id: str, playbook_path: str, extra_vars: dict, inventory: dict[str, Any] | str, callback: str
|
|
61
|
-
) -> None:
|
|
62
|
-
"""Run a playbook, internal function.
|
|
40
|
+
return _executor
|
|
63
41
|
|
|
64
|
-
:param str job_id: Identifier of the job that's executed.
|
|
65
|
-
:param str playbook_path: Ansible playbook to be executed.
|
|
66
|
-
:param dict extra_vars: Extra variables passed to the Ansible playbook.
|
|
67
|
-
:param str callback: Callback URL to return output to when execution is completed.
|
|
68
|
-
:param dict[str, Any] | str inventory: Ansible inventory to run the playbook against.
|
|
69
|
-
"""
|
|
70
|
-
ansible_playbook_run = ansible_runner.run(playbook=playbook_path, inventory=inventory, extravars=extra_vars)
|
|
71
42
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
"output": ansible_playbook_run.stdout.readlines(),
|
|
76
|
-
"return_code": int(ansible_playbook_run.rc),
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
request_result = requests.post(callback, json=payload, timeout=DEFAULT_REQUEST_TIMEOUT)
|
|
80
|
-
if not status.HTTP_200_OK <= request_result.status_code < status.HTTP_300_MULTIPLE_CHOICES:
|
|
81
|
-
msg = f"Callback failed: {request_result.text}"
|
|
82
|
-
logger.error(msg)
|
|
43
|
+
def get_playbook_path(playbook_name: Path) -> Path:
|
|
44
|
+
"""Get the path of a playbook on the local filesystem."""
|
|
45
|
+
return Path(settings.ANSIBLE_PLAYBOOKS_ROOT_DIR) / playbook_name
|
|
83
46
|
|
|
84
47
|
|
|
85
48
|
def run_playbook(
|
|
@@ -87,7 +50,7 @@ def run_playbook(
|
|
|
87
50
|
extra_vars: dict[str, Any],
|
|
88
51
|
inventory: dict[str, Any] | str,
|
|
89
52
|
callback: HttpUrl,
|
|
90
|
-
) ->
|
|
53
|
+
) -> uuid.UUID:
|
|
91
54
|
"""Run an Ansible playbook against a specified inventory.
|
|
92
55
|
|
|
93
56
|
:param Path playbook_path: playbook to be executed.
|
|
@@ -98,25 +61,16 @@ def run_playbook(
|
|
|
98
61
|
:return: Result of playbook launch, this could either be successful or unsuccessful.
|
|
99
62
|
:rtype: :class:`fastapi.responses.JSONResponse`
|
|
100
63
|
"""
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
"playbook_path": str(playbook_path),
|
|
115
|
-
"inventory": inventory,
|
|
116
|
-
"extra_vars": extra_vars,
|
|
117
|
-
"callback": callback,
|
|
118
|
-
},
|
|
119
|
-
)
|
|
120
|
-
thread.start()
|
|
121
|
-
|
|
122
|
-
return playbook_launch_success(job_id=job_id)
|
|
64
|
+
job_id = uuid.uuid4()
|
|
65
|
+
if settings.EXECUTOR == ExecutorType.THREADPOOL:
|
|
66
|
+
executor = get_thread_pool()
|
|
67
|
+
executor_handle = executor.submit(
|
|
68
|
+
run_playbook_proc_task, str(job_id), str(playbook_path), extra_vars, inventory, str(callback)
|
|
69
|
+
)
|
|
70
|
+
if settings.TESTING:
|
|
71
|
+
executor_handle.result()
|
|
72
|
+
|
|
73
|
+
elif settings.EXECUTOR == ExecutorType.WORKER:
|
|
74
|
+
run_playbook_proc_task.delay(str(job_id), str(playbook_path), extra_vars, inventory, str(callback))
|
|
75
|
+
|
|
76
|
+
return job_id
|
lso/routes/playbook.py
CHANGED
|
@@ -15,14 +15,16 @@
|
|
|
15
15
|
|
|
16
16
|
import json
|
|
17
17
|
import tempfile
|
|
18
|
+
import uuid
|
|
18
19
|
from contextlib import redirect_stderr
|
|
19
20
|
from io import StringIO
|
|
21
|
+
from pathlib import Path
|
|
20
22
|
from typing import Annotated, Any
|
|
21
23
|
|
|
24
|
+
import ansible_runner
|
|
22
25
|
from ansible.inventory.manager import InventoryManager
|
|
23
26
|
from ansible.parsing.dataloader import DataLoader
|
|
24
27
|
from fastapi import APIRouter, HTTPException, status
|
|
25
|
-
from fastapi.responses import JSONResponse
|
|
26
28
|
from pydantic import AfterValidator, BaseModel, HttpUrl
|
|
27
29
|
|
|
28
30
|
from lso.playbook import get_playbook_path, run_playbook
|
|
@@ -31,11 +33,19 @@ router = APIRouter()
|
|
|
31
33
|
|
|
32
34
|
|
|
33
35
|
def _inventory_validator(inventory: dict[str, Any] | str) -> dict[str, Any] | str:
|
|
34
|
-
"""Validate the
|
|
36
|
+
"""Validate the provided inventory format.
|
|
35
37
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
+
Attempts to parse the inventory to verify its validity. If the inventory cannot be parsed or the inventory
|
|
39
|
+
format is incorrect an HTTP 422 error is raised.
|
|
40
|
+
|
|
41
|
+
:param inventory: The inventory to validate, can be a dictionary or a string.
|
|
42
|
+
:return: The validated inventory if no errors are found.
|
|
43
|
+
:raises HTTPException: If parsing fails or the format is incorrect.
|
|
38
44
|
"""
|
|
45
|
+
if not ansible_runner.utils.isinventory(inventory):
|
|
46
|
+
detail = "Invalid inventory provided. Should be a string, or JSON object."
|
|
47
|
+
raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=detail)
|
|
48
|
+
|
|
39
49
|
loader = DataLoader()
|
|
40
50
|
output = StringIO()
|
|
41
51
|
with tempfile.NamedTemporaryFile(mode="w+") as temp_inv, redirect_stderr(output):
|
|
@@ -53,15 +63,31 @@ def _inventory_validator(inventory: dict[str, Any] | str) -> dict[str, Any] | st
|
|
|
53
63
|
return inventory
|
|
54
64
|
|
|
55
65
|
|
|
66
|
+
def _playbook_path_validator(playbook_name: Path) -> Path:
|
|
67
|
+
playbook_path = get_playbook_path(playbook_name)
|
|
68
|
+
if not Path.exists(playbook_path):
|
|
69
|
+
msg = f"Filename '{playbook_path}' does not exist."
|
|
70
|
+
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=msg)
|
|
71
|
+
|
|
72
|
+
return playbook_path
|
|
73
|
+
|
|
74
|
+
|
|
56
75
|
PlaybookInventory = Annotated[dict[str, Any] | str, AfterValidator(_inventory_validator)]
|
|
76
|
+
PlaybookName = Annotated[Path, AfterValidator(_playbook_path_validator)]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class PlaybookRunResponse(BaseModel):
|
|
80
|
+
"""PlaybookRunResponse domain model schema."""
|
|
81
|
+
|
|
82
|
+
job_id: uuid.UUID
|
|
57
83
|
|
|
58
84
|
|
|
59
85
|
class PlaybookRunParams(BaseModel):
|
|
60
86
|
"""Parameters for executing an Ansible playbook."""
|
|
61
87
|
|
|
62
88
|
#: The filename of a playbook that's executed. It should be present inside the directory defined in the
|
|
63
|
-
#: configuration option ``
|
|
64
|
-
playbook_name:
|
|
89
|
+
#: configuration option ``ANSIBLE_PLAYBOOKS_ROOT_DIR``.
|
|
90
|
+
playbook_name: PlaybookName
|
|
65
91
|
#: The address where LSO should call back to upon completion.
|
|
66
92
|
callback: HttpUrl
|
|
67
93
|
#: The inventory to run the playbook against. This inventory can also include any host vars, if needed. When
|
|
@@ -74,8 +100,8 @@ class PlaybookRunParams(BaseModel):
|
|
|
74
100
|
extra_vars: dict[str, Any] = {}
|
|
75
101
|
|
|
76
102
|
|
|
77
|
-
@router.post("/")
|
|
78
|
-
def run_playbook_endpoint(params: PlaybookRunParams) ->
|
|
103
|
+
@router.post("/", response_model=PlaybookRunResponse, status_code=status.HTTP_201_CREATED)
|
|
104
|
+
def run_playbook_endpoint(params: PlaybookRunParams) -> PlaybookRunResponse:
|
|
79
105
|
"""Launch an Ansible playbook to modify or deploy a subscription instance.
|
|
80
106
|
|
|
81
107
|
The response will contain either a job ID, or error information.
|
|
@@ -83,9 +109,11 @@ def run_playbook_endpoint(params: PlaybookRunParams) -> JSONResponse:
|
|
|
83
109
|
:param PlaybookRunParams params: Parameters for executing a playbook.
|
|
84
110
|
:return JSONResponse: Response from the Ansible runner, including a run ID.
|
|
85
111
|
"""
|
|
86
|
-
|
|
87
|
-
playbook_path=
|
|
112
|
+
job_id = run_playbook(
|
|
113
|
+
playbook_path=params.playbook_name,
|
|
88
114
|
extra_vars=params.extra_vars,
|
|
89
115
|
inventory=params.inventory,
|
|
90
116
|
callback=params.callback,
|
|
91
117
|
)
|
|
118
|
+
|
|
119
|
+
return PlaybookRunResponse(job_id=job_id)
|
lso/tasks.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Copyright 2023-2024 GÉANT Vereniging.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
|
|
14
|
+
"""Module defines tasks for executing Ansible playbooks asynchronously using Celery.
|
|
15
|
+
|
|
16
|
+
The primary task, `run_playbook_proc_task`, runs an Ansible playbook and sends a POST request with
|
|
17
|
+
the results to a specified callback URL.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import logging
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import ansible_runner
|
|
24
|
+
import requests
|
|
25
|
+
from starlette import status
|
|
26
|
+
|
|
27
|
+
from lso.config import settings
|
|
28
|
+
from lso.worker import RUN_PLAYBOOK, celery
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger(__name__)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class CallbackFailedError(Exception):
|
|
34
|
+
"""Exception raised when a callback url can't be reached."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@celery.task(name=RUN_PLAYBOOK) # type: ignore[misc]
|
|
38
|
+
def run_playbook_proc_task(
|
|
39
|
+
job_id: str, playbook_path: str, extra_vars: dict[str, Any], inventory: dict[str, Any] | str, callback: str
|
|
40
|
+
) -> None:
|
|
41
|
+
"""Celery task to run a playbook.
|
|
42
|
+
|
|
43
|
+
:param str job_id: Identifier of the job being executed.
|
|
44
|
+
:param str playbook_path: Path to the playbook to be executed.
|
|
45
|
+
:param dict[str, Any] extra_vars: Extra variables to pass to the playbook.
|
|
46
|
+
:param dict[str, Any] | str inventory: Inventory to run the playbook against.
|
|
47
|
+
:param HttpUrl callback: Callback URL for status updates.
|
|
48
|
+
:return: None
|
|
49
|
+
"""
|
|
50
|
+
msg = f"playbook_path: {playbook_path}, callback: {callback}"
|
|
51
|
+
logger.info(msg)
|
|
52
|
+
ansible_playbook_run = ansible_runner.run(playbook=playbook_path, inventory=inventory, extravars=extra_vars)
|
|
53
|
+
|
|
54
|
+
payload = {
|
|
55
|
+
"status": ansible_playbook_run.status,
|
|
56
|
+
"job_id": job_id,
|
|
57
|
+
"output": ansible_playbook_run.stdout.readlines(),
|
|
58
|
+
"return_code": int(ansible_playbook_run.rc),
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
request_result = requests.post(str(callback), json=payload, timeout=settings.REQUEST_TIMEOUT_SEC)
|
|
62
|
+
if not status.HTTP_200_OK <= request_result.status_code < status.HTTP_300_MULTIPLE_CHOICES:
|
|
63
|
+
msg = f"Callback failed: {request_result.text}, url: {callback}"
|
|
64
|
+
raise CallbackFailedError(msg)
|
lso/worker.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Copyright 2023-2024 GÉANT Vereniging.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
|
|
14
|
+
"""Module that sets up :term:`LSO` as a Celery worker."""
|
|
15
|
+
|
|
16
|
+
from celery import Celery
|
|
17
|
+
from celery.signals import worker_shutting_down
|
|
18
|
+
|
|
19
|
+
from lso.config import settings
|
|
20
|
+
|
|
21
|
+
RUN_PLAYBOOK = "lso.tasks.run_playbook_proc_task"
|
|
22
|
+
|
|
23
|
+
celery = Celery(
|
|
24
|
+
"lso-worker",
|
|
25
|
+
broker=settings.CELERY_BROKER_URL,
|
|
26
|
+
backend=settings.CELERY_RESULT_BACKEND,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
if settings.TESTING:
|
|
30
|
+
celery.conf.update(backend=settings.CELERY_RESULT_BACKEND, task_ignore_result=False)
|
|
31
|
+
else:
|
|
32
|
+
celery.conf.update(task_ignore_result=True)
|
|
33
|
+
|
|
34
|
+
celery.conf.update(
|
|
35
|
+
result_expires=settings.CELERY_RESULT_EXPIRES,
|
|
36
|
+
worker_prefetch_multiplier=1,
|
|
37
|
+
worker_send_task_event=True,
|
|
38
|
+
task_send_sent_event=True,
|
|
39
|
+
redbeat_redis_url=settings.CELERY_BROKER_URL,
|
|
40
|
+
broker_connection_retry_on_startup=True,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
if settings.WORKER_QUEUE_NAME:
|
|
44
|
+
celery.conf.task_routes = {
|
|
45
|
+
RUN_PLAYBOOK: {"queue": settings.WORKER_QUEUE_NAME},
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@worker_shutting_down.connect # type: ignore[misc]
|
|
50
|
+
def worker_shutting_down_handler(sig, how, exitcode, **kwargs) -> None: # type: ignore[no-untyped-def] # noqa: ARG001
|
|
51
|
+
"""Handle the Celery worker shutdown event."""
|
|
52
|
+
celery.close()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: orchestrator-lso
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.1
|
|
4
4
|
Summary: LSO, an API for remotely running Ansible playbooks.
|
|
5
5
|
Author-email: GÉANT Orchestration and Automation Team <goat@geant.org>
|
|
6
6
|
Requires-Python: >=3.11,<3.13
|
|
@@ -25,13 +25,15 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
25
25
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
26
26
|
Classifier: Programming Language :: Python :: 3.11
|
|
27
27
|
Classifier: Programming Language :: Python :: 3.12
|
|
28
|
-
Requires-Dist: ansible-runner
|
|
29
|
-
Requires-Dist: ansible
|
|
30
|
-
Requires-Dist: fastapi
|
|
31
|
-
Requires-Dist: httpx
|
|
32
|
-
Requires-Dist:
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
Requires-Dist:
|
|
28
|
+
Requires-Dist: ansible-runner==2.4.0
|
|
29
|
+
Requires-Dist: ansible==10.6.0
|
|
30
|
+
Requires-Dist: fastapi==0.115.5
|
|
31
|
+
Requires-Dist: httpx==0.28.0
|
|
32
|
+
Requires-Dist: uvicorn[standard]==0.32.1
|
|
33
|
+
Requires-Dist: requests==2.32.3
|
|
34
|
+
Requires-Dist: pydantic-settings==2.6.1
|
|
35
|
+
Requires-Dist: celery==5.4.0
|
|
36
|
+
Requires-Dist: redis==5.2.0
|
|
35
37
|
Requires-Dist: types-setuptools ; extra == "dev"
|
|
36
38
|
Requires-Dist: types-requests ; extra == "dev"
|
|
37
39
|
Requires-Dist: toml ; extra == "dev"
|
|
@@ -82,11 +84,9 @@ Use the Docker image to then spin up an environment. An example Docker compose f
|
|
|
82
84
|
services:
|
|
83
85
|
lso:
|
|
84
86
|
image: my-lso:latest
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
ANSIBLE_ROLES_PATH: /app/lso/ansible_roles
|
|
87
|
+
env_file:
|
|
88
|
+
.env # Load default environment variables from the .env file
|
|
88
89
|
volumes:
|
|
89
|
-
- "/home/user/config.json:/app/config.json:ro"
|
|
90
90
|
- "/home/user/ansible_inventory:/opt/ansible_inventory:ro"
|
|
91
91
|
- "~/.ssh/id_ed25519.pub:/root/.ssh/id_ed25519.pub:ro"
|
|
92
92
|
- "~/.ssh/id_ed25519:/root/.ssh/id_ed25519:ro"
|
|
@@ -94,7 +94,8 @@ services:
|
|
|
94
94
|
|
|
95
95
|
This will expose the API on port 8000. The container requires some more files to be mounted:
|
|
96
96
|
|
|
97
|
-
*
|
|
97
|
+
* An .env file: Sets default environment variables, like ANSIBLE_PLAYBOOKS_ROOT_DIR for the location of Ansible playbooks **inside the container**.
|
|
98
|
+
* Environment variables: Specific configurations, such as ANSIBLE_ROLES_PATH, can be directly set in the environment section. This is ideal for values you may want to override without modifying the .env file.
|
|
98
99
|
* An Ansible inventory for all host and group variables that are used in the playbooks
|
|
99
100
|
* A public/private key pair for SSH authentication on external machines that are targeted by Ansible playbooks.
|
|
100
101
|
* Any Ansible-specific configuration (such as `collections_path`, `roles_path`, etc.) should be set using
|
|
@@ -132,11 +133,30 @@ As an alternative, below are a set of instructions for installing and running LS
|
|
|
132
133
|
|
|
133
134
|
### Running the app
|
|
134
135
|
|
|
135
|
-
*
|
|
136
|
+
* Set required environment variables; see `env.example` for reference.
|
|
136
137
|
* If necessary, set the environment variable `ANSIBLE_HOME` to a custom path.
|
|
137
138
|
* Run the app like this (`app.py` starts the server on port 44444):
|
|
138
139
|
|
|
139
140
|
```bash
|
|
140
|
-
|
|
141
|
+
source .env && python -m lso.app
|
|
141
142
|
```
|
|
142
143
|
|
|
144
|
+
### Task Execution Options
|
|
145
|
+
1. Celery (Distributed Execution)
|
|
146
|
+
|
|
147
|
+
- For distributed task execution, set `EXECUTOR=celery`.
|
|
148
|
+
- Add Celery config in your environment variables:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
CELERY_BROKER_URL=redis://localhost:6379/0
|
|
152
|
+
CELERY_RESULT_BACKEND=redis://localhost:6379/0
|
|
153
|
+
WORKER_QUEUE_NAME=lso-worker-queue # default value is None so you don't need this by default.
|
|
154
|
+
```
|
|
155
|
+
- Start a Celery worker:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
celery -A lso.worker worker --loglevel=info -Q lso-worker-queue
|
|
159
|
+
```
|
|
160
|
+
2. ThreadPoolExecutor (Local Execution)
|
|
161
|
+
|
|
162
|
+
For local concurrent tasks, set `EXECUTOR=threadpool` and configure `MAX_THREAD_POOL_WORKERS`.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
lso/__init__.py,sha256=Q6PrVmj3JOVZ4MJ9MYMVEI8L8w0GZy0EQZb4baSbzwA,1465
|
|
2
|
+
lso/app.py,sha256=WDtlmjELIeFA437j-WPfqBQf6QT_l35LEABbEosHlqY,775
|
|
3
|
+
lso/config.py,sha256=BXUl8SgYdaXbrmmqBRgxobGboCn02qfp_wdvZHZr_Gk,1627
|
|
4
|
+
lso/environment.py,sha256=iZ3DmsSKAC5a7VNL-HfJOJZ0sQwUMf7ZzNGC34B2CG0,1771
|
|
5
|
+
lso/playbook.py,sha256=PRnZMa93FZ-3s1pn2B4p6PBii-VN-Ti8DyhuMd8AI0I,2766
|
|
6
|
+
lso/tasks.py,sha256=d_JRESlfs2dw-KZMLu2F21VHOz7l-5FhrhTDX58Pn38,2484
|
|
7
|
+
lso/worker.py,sha256=3Y32F2DknsVK3JY4M6hDk35lvn7xLHqJhAIIbvxYIFE,1730
|
|
8
|
+
lso/routes/__init__.py,sha256=1kRrth9zkFgmj6LChujieYJq5cjIETeTGXa1G70pduk,639
|
|
9
|
+
lso/routes/default.py,sha256=a7STN1BJyFVizXUzmqKuADO0fpE1SHun-PzaZ-jx1wU,1438
|
|
10
|
+
lso/routes/playbook.py,sha256=Q0Q-9fLOYpahgC7WJ5SEFt9P2NJnAKdQg3onjVnd6js,4903
|
|
11
|
+
orchestrator_lso-2.0.1.dist-info/LICENSE,sha256=CgFXf7XbZXJADozQIw2uUmmvU-zwAwXo4u7cgDfx3rE,10744
|
|
12
|
+
orchestrator_lso-2.0.1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
|
|
13
|
+
orchestrator_lso-2.0.1.dist-info/METADATA,sha256=Ex0bEBcEdeOPYt-mu1BbbkF2_Rqct3iww2u4nlQEGJk,6329
|
|
14
|
+
orchestrator_lso-2.0.1.dist-info/RECORD,,
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
lso/__init__.py,sha256=vveEyutM6Gbg73OAo_oeAw418HED2t8du9PkXvRy4XI,1564
|
|
2
|
-
lso/app.py,sha256=WDtlmjELIeFA437j-WPfqBQf6QT_l35LEABbEosHlqY,775
|
|
3
|
-
lso/config.py,sha256=CB4Ydc8LxGaM24YrLMh59LxYFMOHGK03y4X9R-r_cAM,2436
|
|
4
|
-
lso/environment.py,sha256=iZ3DmsSKAC5a7VNL-HfJOJZ0sQwUMf7ZzNGC34B2CG0,1771
|
|
5
|
-
lso/playbook.py,sha256=EuMdKtXjkcD-D-q5ytrKB0fXTT5IhOjnyEXiln0bdk0,4925
|
|
6
|
-
lso/routes/__init__.py,sha256=1kRrth9zkFgmj6LChujieYJq5cjIETeTGXa1G70pduk,639
|
|
7
|
-
lso/routes/default.py,sha256=a7STN1BJyFVizXUzmqKuADO0fpE1SHun-PzaZ-jx1wU,1438
|
|
8
|
-
lso/routes/playbook.py,sha256=VwxCNRvIg-MTAXyHDHcFhAAY2vQQG77VVMJf_GjZsr4,3837
|
|
9
|
-
orchestrator_lso-1.0.3.dist-info/LICENSE,sha256=CgFXf7XbZXJADozQIw2uUmmvU-zwAwXo4u7cgDfx3rE,10744
|
|
10
|
-
orchestrator_lso-1.0.3.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
|
|
11
|
-
orchestrator_lso-1.0.3.dist-info/METADATA,sha256=2GebmTzglr0F-H-k5lJnYqBkUy0nYMCax-pUjctCVb0,5521
|
|
12
|
-
orchestrator_lso-1.0.3.dist-info/RECORD,,
|
|
File without changes
|