devservices 1.2.0__tar.gz → 1.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {devservices-1.2.0 → devservices-1.2.2}/PKG-INFO +1 -1
- {devservices-1.2.0 → devservices-1.2.2}/README.md +25 -2
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/down.py +24 -10
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/foreground.py +8 -7
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/logs.py +3 -5
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/serve.py +17 -9
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/status.py +10 -8
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/up.py +29 -14
- {devservices-1.2.0 → devservices-1.2.2}/devservices/configs/service_config.py +14 -10
- {devservices-1.2.0 → devservices-1.2.2}/devservices/constants.py +0 -1
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/docker.py +8 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/supervisor.py +73 -12
- {devservices-1.2.0 → devservices-1.2.2}/devservices.egg-info/PKG-INFO +1 -1
- {devservices-1.2.0 → devservices-1.2.2}/pyproject.toml +1 -1
- {devservices-1.2.0 → devservices-1.2.2}/testing/utils.py +0 -9
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_down.py +24 -59
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_foreground.py +59 -95
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_list_services.py +1 -1
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_logs.py +29 -45
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_serve.py +14 -19
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_status.py +39 -38
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_up.py +91 -47
- {devservices-1.2.0 → devservices-1.2.2}/tests/configs/test_service_config.py +54 -19
- {devservices-1.2.0 → devservices-1.2.2}/tests/utils/test_docker.py +21 -6
- {devservices-1.2.0 → devservices-1.2.2}/tests/utils/test_docker_compose.py +4 -1
- {devservices-1.2.0 → devservices-1.2.2}/tests/utils/test_supervisor.py +103 -10
- {devservices-1.2.0 → devservices-1.2.2}/LICENSE.md +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/__init__.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/__init__.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/list_dependencies.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/list_services.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/purge.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/reset.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/toggle.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/commands/update.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/exceptions.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/main.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/__init__.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/check_for_update.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/console.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/dependencies.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/devenv.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/docker_compose.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/file_lock.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/git.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/install_binary.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/services.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices/utils/state.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices.egg-info/SOURCES.txt +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices.egg-info/dependency_links.txt +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices.egg-info/entry_points.txt +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices.egg-info/requires.txt +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/devservices.egg-info/top_level.txt +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/setup.cfg +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/testing/__init__.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/__init__.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_list_dependencies.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_purge.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_reset.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_toggle.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/commands/test_update.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/conftest.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/utils/test_check_for_update.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/utils/test_dependencies.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/utils/test_git.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/utils/test_install_binary.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/utils/test_services.py +0 -0
- {devservices-1.2.0 → devservices-1.2.2}/tests/utils/test_state.py +0 -0
|
@@ -23,6 +23,8 @@ NOTE: service-name is an optional parameter. If not provided, devservices will a
|
|
|
23
23
|
- `devservices update` Update devservices to the latest version.
|
|
24
24
|
- `devservices purge`: Purge the local devservices cache.
|
|
25
25
|
- `devservices toggle <service-name>`: Toggle the runtime for a service between containerized and local.
|
|
26
|
+
- `devservices serve`: Run a service's devserver.
|
|
27
|
+
- `devservices foreground <process-name>`: Foreground a process that is currently running in the background. This enables interactive debugging.
|
|
26
28
|
|
|
27
29
|
## Installation
|
|
28
30
|
|
|
@@ -31,7 +33,7 @@ NOTE: service-name is an optional parameter. If not provided, devservices will a
|
|
|
31
33
|
The recommended way to install devservices is through a virtualenv in the requirements.txt. Once that is installed and a devservices config file is added, you should be able to run `devservices up` to begin local development.
|
|
32
34
|
|
|
33
35
|
```
|
|
34
|
-
devservices==1.2.
|
|
36
|
+
devservices==1.2.2
|
|
35
37
|
```
|
|
36
38
|
|
|
37
39
|
### 2. Add devservices config files
|
|
@@ -45,7 +47,7 @@ The configuration file is a yaml file that looks like this:
|
|
|
45
47
|
# - version: The version of the devservices config file. This is used to ensure compatibility between devservices and the config file.
|
|
46
48
|
# - service_name: The name of the service. This is used to identify the service in the config file.
|
|
47
49
|
# - dependencies: A list of dependencies for the service. Each dependency is a yaml block that holds the dependency configuration. There are two types of dependencies:
|
|
48
|
-
# - local: A dependency that is defined in the config file. These dependencies do not have a remote field
|
|
50
|
+
# - local: A dependency that is defined in the config file. These dependencies do not have a remote field and must correspond to either a service defined in the 'services' section or a program defined in the 'x-programs' section.
|
|
49
51
|
# - remote: A dependency that is defined in the devservices directory in a remote repository. These configs are automatically fetched from the remote repository and installed. Any dependency with a remote field will be treated as a remote dependency. Example: https://github.com/getsentry/snuba/blob/59a5258ccbb502827ebc1d3b1bf80c607a3301bf/devservices/config.yml#L8
|
|
50
52
|
# - modes: A list of modes for the service. Each mode includes a list of dependencies that are used in that mode.
|
|
51
53
|
x-sentry-service-config:
|
|
@@ -67,6 +69,27 @@ x-sentry-service-config:
|
|
|
67
69
|
default: [example-dependency-1, example-remote-dependency]
|
|
68
70
|
custom-mode: [example-dependency-1, example-dependency-2, example-remote-dependency]
|
|
69
71
|
|
|
72
|
+
# This block defines supervisor programs that can be managed by devservices.
|
|
73
|
+
# Programs defined here are managed using Python's supervisor package and can be:
|
|
74
|
+
# - Started and stopped along with docker services when running `devservices up/down`
|
|
75
|
+
# - Run in the foreground for interactive debugging using `devservices foreground <program-name>`
|
|
76
|
+
# - Used as development server with `devservices serve` (requires a program named "devserver")
|
|
77
|
+
#
|
|
78
|
+
# These are particularly useful for:
|
|
79
|
+
# - Defining a devserver to run locally
|
|
80
|
+
# - Background workers or task processors
|
|
81
|
+
# - Any process that needs to be managed alongside your docker services
|
|
82
|
+
x-programs:
|
|
83
|
+
# Example devserver
|
|
84
|
+
devserver:
|
|
85
|
+
# Required: The command to execute
|
|
86
|
+
command: "python manage.py run_server"
|
|
87
|
+
# Optional: You can also specify any of the supervisor program settings defined here: https://supervisord.org/configuration.html#program-x-section-settings
|
|
88
|
+
|
|
89
|
+
# Example background worker
|
|
90
|
+
example-worker:
|
|
91
|
+
command: "python manage.py worker"
|
|
92
|
+
|
|
70
93
|
# This will be a standard block used by docker compose to define dependencies.
|
|
71
94
|
#
|
|
72
95
|
# The following fields are important to all dependencies:
|
|
@@ -15,12 +15,12 @@ from devservices.constants import DependencyType
|
|
|
15
15
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
|
|
16
16
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
|
|
17
17
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
18
|
-
from devservices.constants import PROGRAMS_CONF_FILE_NAME
|
|
19
18
|
from devservices.exceptions import ConfigError
|
|
20
19
|
from devservices.exceptions import ConfigNotFoundError
|
|
21
20
|
from devservices.exceptions import DependencyError
|
|
22
21
|
from devservices.exceptions import DockerComposeError
|
|
23
22
|
from devservices.exceptions import ServiceNotFoundError
|
|
23
|
+
from devservices.exceptions import SupervisorConfigError
|
|
24
24
|
from devservices.exceptions import SupervisorError
|
|
25
25
|
from devservices.utils.console import Console
|
|
26
26
|
from devservices.utils.console import Status
|
|
@@ -305,22 +305,36 @@ def _bring_down_dependency(
|
|
|
305
305
|
return run_cmd(cmd.full_command, current_env)
|
|
306
306
|
|
|
307
307
|
|
|
308
|
+
def _stop_supervisor_program(
|
|
309
|
+
manager: SupervisorManager, program: str, status: Status
|
|
310
|
+
) -> None:
|
|
311
|
+
"""Stop a single supervisor program."""
|
|
312
|
+
status.info(f"Stopping {program}")
|
|
313
|
+
manager.stop_process(program)
|
|
314
|
+
|
|
315
|
+
|
|
308
316
|
def bring_down_supervisor_programs(
|
|
309
317
|
supervisor_programs: list[str], service: Service, status: Status
|
|
310
318
|
) -> None:
|
|
311
319
|
if len(supervisor_programs) == 0:
|
|
312
320
|
return
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
manager = SupervisorManager(
|
|
317
|
-
programs_config_path,
|
|
318
|
-
service_name=service.name,
|
|
321
|
+
|
|
322
|
+
config_file_path = os.path.join(
|
|
323
|
+
service.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
|
|
319
324
|
)
|
|
320
325
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
326
|
+
try:
|
|
327
|
+
manager = SupervisorManager(service.name, config_file_path)
|
|
328
|
+
except SupervisorConfigError:
|
|
329
|
+
raise
|
|
330
|
+
|
|
331
|
+
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
332
|
+
futures = [
|
|
333
|
+
executor.submit(_stop_supervisor_program, manager, program, status)
|
|
334
|
+
for program in supervisor_programs
|
|
335
|
+
]
|
|
336
|
+
for future in concurrent.futures.as_completed(futures):
|
|
337
|
+
_ = future.result()
|
|
324
338
|
|
|
325
339
|
status.info("Stopping supervisor daemon")
|
|
326
340
|
manager.stop_supervisor_daemon()
|
|
@@ -9,9 +9,9 @@ from argparse import Namespace
|
|
|
9
9
|
|
|
10
10
|
from sentry_sdk import capture_exception
|
|
11
11
|
|
|
12
|
+
from devservices.constants import CONFIG_FILE_NAME
|
|
12
13
|
from devservices.constants import DependencyType
|
|
13
14
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
14
|
-
from devservices.constants import PROGRAMS_CONF_FILE_NAME
|
|
15
15
|
from devservices.exceptions import ConfigError
|
|
16
16
|
from devservices.exceptions import ConfigNotFoundError
|
|
17
17
|
from devservices.exceptions import ServiceNotFoundError
|
|
@@ -92,14 +92,15 @@ def foreground(args: Namespace) -> None:
|
|
|
92
92
|
)
|
|
93
93
|
return
|
|
94
94
|
|
|
95
|
-
|
|
96
|
-
service.repo_path,
|
|
95
|
+
config_file_path = os.path.join(
|
|
96
|
+
service.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
|
|
97
97
|
)
|
|
98
98
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
99
|
+
try:
|
|
100
|
+
manager = SupervisorManager(service.name, config_file_path)
|
|
101
|
+
except SupervisorConfigError as e:
|
|
102
|
+
capture_exception(e, level="info")
|
|
103
|
+
return
|
|
103
104
|
|
|
104
105
|
try:
|
|
105
106
|
program_command = manager.get_program_command(program_name)
|
|
@@ -16,7 +16,6 @@ from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
|
|
|
16
16
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
|
|
17
17
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
18
18
|
from devservices.constants import MAX_LOG_LINES
|
|
19
|
-
from devservices.constants import PROGRAMS_CONF_FILE_NAME
|
|
20
19
|
from devservices.exceptions import ConfigError
|
|
21
20
|
from devservices.exceptions import ConfigNotFoundError
|
|
22
21
|
from devservices.exceptions import DependencyError
|
|
@@ -178,12 +177,11 @@ def _supervisor_logs(
|
|
|
178
177
|
|
|
179
178
|
supervisor_logs: dict[str, str] = {}
|
|
180
179
|
|
|
181
|
-
|
|
182
|
-
service.repo_path, DEVSERVICES_DIR_NAME,
|
|
180
|
+
config_file_path = os.path.join(
|
|
181
|
+
service.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
|
|
183
182
|
)
|
|
184
|
-
|
|
185
183
|
try:
|
|
186
|
-
manager = SupervisorManager(
|
|
184
|
+
manager = SupervisorManager(service.name, config_file_path)
|
|
187
185
|
except SupervisorConfigError as e:
|
|
188
186
|
capture_exception(e)
|
|
189
187
|
return supervisor_logs
|
|
@@ -9,8 +9,8 @@ from argparse import Namespace
|
|
|
9
9
|
|
|
10
10
|
from sentry_sdk import capture_exception
|
|
11
11
|
|
|
12
|
+
from devservices.constants import CONFIG_FILE_NAME
|
|
12
13
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
13
|
-
from devservices.constants import PROGRAMS_CONF_FILE_NAME
|
|
14
14
|
from devservices.exceptions import ConfigError
|
|
15
15
|
from devservices.exceptions import ConfigNotFoundError
|
|
16
16
|
from devservices.exceptions import SupervisorConfigError
|
|
@@ -46,16 +46,24 @@ def serve(args: Namespace) -> None:
|
|
|
46
46
|
console.failure(str(e))
|
|
47
47
|
exit(1)
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
service.repo_path,
|
|
49
|
+
config_file_path = os.path.join(
|
|
50
|
+
service.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
|
|
51
51
|
)
|
|
52
|
-
|
|
53
|
-
|
|
52
|
+
|
|
53
|
+
try:
|
|
54
|
+
manager = SupervisorManager(service.name, config_file_path)
|
|
55
|
+
except SupervisorConfigError as e:
|
|
56
|
+
capture_exception(e, level="info")
|
|
57
|
+
console.failure(
|
|
58
|
+
f"Unable to bring up devserver due to supervisor config error: {str(e)}"
|
|
59
|
+
)
|
|
60
|
+
return
|
|
61
|
+
|
|
62
|
+
if not manager.has_programs:
|
|
63
|
+
console.failure(
|
|
64
|
+
"No programs found in config. Please add the devserver in the `x-programs` block to your config.yml"
|
|
65
|
+
)
|
|
54
66
|
return
|
|
55
|
-
manager = SupervisorManager(
|
|
56
|
-
programs_config_path,
|
|
57
|
-
service_name=service.name,
|
|
58
|
-
)
|
|
59
67
|
|
|
60
68
|
try:
|
|
61
69
|
devserver_command = manager.get_program_command("devserver")
|
|
@@ -20,12 +20,12 @@ from devservices.constants import DependencyType
|
|
|
20
20
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
|
|
21
21
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
|
|
22
22
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
23
|
-
from devservices.constants import PROGRAMS_CONF_FILE_NAME
|
|
24
23
|
from devservices.exceptions import ConfigError
|
|
25
24
|
from devservices.exceptions import ConfigNotFoundError
|
|
26
25
|
from devservices.exceptions import DependencyError
|
|
27
26
|
from devservices.exceptions import DockerComposeError
|
|
28
27
|
from devservices.exceptions import ServiceNotFoundError
|
|
28
|
+
from devservices.exceptions import SupervisorConfigError
|
|
29
29
|
from devservices.utils.console import Console
|
|
30
30
|
from devservices.utils.dependencies import construct_dependency_graph
|
|
31
31
|
from devservices.utils.dependencies import DependencyGraph
|
|
@@ -103,16 +103,18 @@ def status(args: Namespace) -> None:
|
|
|
103
103
|
console.warning(f"Status unavailable. {service.name} is not running standalone")
|
|
104
104
|
return # Since exit(0) is captured as an internal_error by sentry
|
|
105
105
|
|
|
106
|
-
|
|
107
|
-
service.repo_path,
|
|
106
|
+
config_file_path = os.path.join(
|
|
107
|
+
service.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
|
|
108
108
|
)
|
|
109
109
|
process_statuses = {}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
service.name,
|
|
114
|
-
)
|
|
110
|
+
|
|
111
|
+
try:
|
|
112
|
+
supervisor_manager = SupervisorManager(service.name, config_file_path)
|
|
115
113
|
process_statuses = supervisor_manager.get_all_process_info()
|
|
114
|
+
except SupervisorConfigError as e:
|
|
115
|
+
capture_exception(e)
|
|
116
|
+
console.failure(str(e))
|
|
117
|
+
exit(1)
|
|
116
118
|
|
|
117
119
|
try:
|
|
118
120
|
status_tree = get_status_for_service(service, process_statuses)
|
|
@@ -17,7 +17,6 @@ from devservices.constants import DependencyType
|
|
|
17
17
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
|
|
18
18
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
|
|
19
19
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
20
|
-
from devservices.constants import PROGRAMS_CONF_FILE_NAME
|
|
21
20
|
from devservices.exceptions import ConfigError
|
|
22
21
|
from devservices.exceptions import ConfigNotFoundError
|
|
23
22
|
from devservices.exceptions import ContainerHealthcheckFailedError
|
|
@@ -25,6 +24,7 @@ from devservices.exceptions import DependencyError
|
|
|
25
24
|
from devservices.exceptions import DockerComposeError
|
|
26
25
|
from devservices.exceptions import ModeDoesNotExistError
|
|
27
26
|
from devservices.exceptions import ServiceNotFoundError
|
|
27
|
+
from devservices.exceptions import SupervisorConfigError
|
|
28
28
|
from devservices.exceptions import SupervisorError
|
|
29
29
|
from devservices.utils.console import Console
|
|
30
30
|
from devservices.utils.console import Status
|
|
@@ -98,14 +98,16 @@ def up(args: Namespace, existing_status: Status | None = None) -> None:
|
|
|
98
98
|
|
|
99
99
|
with Status(
|
|
100
100
|
lambda: (
|
|
101
|
-
console.warning(f"Starting '{service.name}' in mode: '{mode}'")
|
|
101
|
+
console.warning(f"Starting '{service.name}' dependencies in mode: '{mode}'")
|
|
102
102
|
if existing_status is None
|
|
103
|
-
else existing_status.warning(
|
|
103
|
+
else existing_status.warning(
|
|
104
|
+
f"Starting '{service.name}' dependencies in mode: '{mode}'"
|
|
105
|
+
)
|
|
104
106
|
),
|
|
105
107
|
lambda: (
|
|
106
|
-
console.success(f"{service.name} started")
|
|
108
|
+
console.success(f"{service.name} dependencies started")
|
|
107
109
|
if existing_status is None
|
|
108
|
-
else existing_status.success(f"{service.name} started")
|
|
110
|
+
else existing_status.success(f"{service.name} dependencies started")
|
|
109
111
|
),
|
|
110
112
|
) as status:
|
|
111
113
|
local_runtime_dependency_names = set()
|
|
@@ -381,6 +383,14 @@ def bring_up_docker_compose_services(
|
|
|
381
383
|
exit(1)
|
|
382
384
|
|
|
383
385
|
|
|
386
|
+
def _start_supervisor_program(
|
|
387
|
+
manager: SupervisorManager, program: str, status: Status
|
|
388
|
+
) -> None:
|
|
389
|
+
"""Start a single supervisor program."""
|
|
390
|
+
status.info(f"Starting {program}")
|
|
391
|
+
manager.start_process(program)
|
|
392
|
+
|
|
393
|
+
|
|
384
394
|
def bring_up_supervisor_programs(
|
|
385
395
|
service: Service, supervisor_programs: list[str], status: Status
|
|
386
396
|
) -> None:
|
|
@@ -391,18 +401,23 @@ def bring_up_supervisor_programs(
|
|
|
391
401
|
f"Cannot bring up supervisor programs from outside the service repository. Please run the command from the service repository ({service.repo_path})"
|
|
392
402
|
)
|
|
393
403
|
return
|
|
394
|
-
programs_config_path = os.path.join(
|
|
395
|
-
service.repo_path, f"{DEVSERVICES_DIR_NAME}/{PROGRAMS_CONF_FILE_NAME}"
|
|
396
|
-
)
|
|
397
404
|
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
service_name=service.name,
|
|
405
|
+
config_file_path = os.path.join(
|
|
406
|
+
service.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
|
|
401
407
|
)
|
|
402
408
|
|
|
409
|
+
try:
|
|
410
|
+
manager = SupervisorManager(service.name, config_file_path)
|
|
411
|
+
except SupervisorConfigError:
|
|
412
|
+
raise
|
|
413
|
+
|
|
403
414
|
status.info("Starting supervisor daemon")
|
|
404
415
|
manager.start_supervisor_daemon()
|
|
405
416
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
417
|
+
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
418
|
+
futures = [
|
|
419
|
+
executor.submit(_start_supervisor_program, manager, program, status)
|
|
420
|
+
for program in supervisor_programs
|
|
421
|
+
]
|
|
422
|
+
for future in concurrent.futures.as_completed(futures):
|
|
423
|
+
_ = future.result()
|
|
@@ -10,10 +10,10 @@ from supervisor.options import ServerOptions
|
|
|
10
10
|
from devservices.constants import CONFIG_FILE_NAME
|
|
11
11
|
from devservices.constants import DependencyType
|
|
12
12
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
13
|
-
from devservices.constants import PROGRAMS_CONF_FILE_NAME
|
|
14
13
|
from devservices.exceptions import ConfigNotFoundError
|
|
15
14
|
from devservices.exceptions import ConfigParseError
|
|
16
15
|
from devservices.exceptions import ConfigValidationError
|
|
16
|
+
from devservices.utils.supervisor import ProgramData
|
|
17
17
|
from devservices.utils.supervisor import SupervisorManager
|
|
18
18
|
|
|
19
19
|
VALID_VERSIONS = [0.1]
|
|
@@ -91,8 +91,10 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig:
|
|
|
91
91
|
|
|
92
92
|
docker_compose_services = config.get("services", {}).keys()
|
|
93
93
|
|
|
94
|
-
supervisor_programs =
|
|
95
|
-
|
|
94
|
+
supervisor_programs = load_supervisor_programs_from_programs_data(
|
|
95
|
+
config_path,
|
|
96
|
+
service_config_data.get("service_name"),
|
|
97
|
+
config.get("x-programs", {}),
|
|
96
98
|
)
|
|
97
99
|
|
|
98
100
|
valid_dependency_keys = {field.name for field in fields(Dependency)}
|
|
@@ -113,7 +115,7 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig:
|
|
|
113
115
|
dependency_type = DependencyType.COMPOSE
|
|
114
116
|
else:
|
|
115
117
|
raise ConfigValidationError(
|
|
116
|
-
f"Dependency '{key}' is not remote but is not defined in docker-compose services or programs
|
|
118
|
+
f"Dependency '{key}' is not remote but is not defined in docker-compose services or x-programs"
|
|
117
119
|
)
|
|
118
120
|
else:
|
|
119
121
|
dependency_type = DependencyType.SERVICE
|
|
@@ -142,13 +144,15 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig:
|
|
|
142
144
|
return service_config
|
|
143
145
|
|
|
144
146
|
|
|
145
|
-
def
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
if not os.path.exists(programs_config_path):
|
|
147
|
+
def load_supervisor_programs_from_programs_data(
|
|
148
|
+
service_config_path: str, service_name: str, programs_data: ProgramData
|
|
149
|
+
) -> set[str]:
|
|
150
|
+
if not programs_data:
|
|
150
151
|
return set()
|
|
151
|
-
|
|
152
|
+
|
|
153
|
+
manager = SupervisorManager(
|
|
154
|
+
service_name=service_name, service_config_path=service_config_path
|
|
155
|
+
)
|
|
152
156
|
opts = ServerOptions()
|
|
153
157
|
opts.configfile = manager.config_file_path
|
|
154
158
|
opts.process_config()
|
|
@@ -25,7 +25,6 @@ class DependencyType(StrEnum):
|
|
|
25
25
|
MINIMUM_DOCKER_COMPOSE_VERSION = "2.29.7"
|
|
26
26
|
DEVSERVICES_DIR_NAME = "devservices"
|
|
27
27
|
CONFIG_FILE_NAME = "config.yml"
|
|
28
|
-
PROGRAMS_CONF_FILE_NAME = "programs.conf"
|
|
29
28
|
DOCKER_CONFIG_DIR = os.environ.get("DOCKER_CONFIG", os.path.expanduser("~/.docker"))
|
|
30
29
|
DOCKER_USER_PLUGIN_DIR = os.path.join(DOCKER_CONFIG_DIR, "cli-plugins/")
|
|
31
30
|
|
|
@@ -10,6 +10,7 @@ from devservices.constants import HEALTHCHECK_TIMEOUT
|
|
|
10
10
|
from devservices.exceptions import ContainerHealthcheckFailedError
|
|
11
11
|
from devservices.exceptions import DockerDaemonNotRunningError
|
|
12
12
|
from devservices.exceptions import DockerError
|
|
13
|
+
from devservices.utils.console import Console
|
|
13
14
|
from devservices.utils.console import Status
|
|
14
15
|
|
|
15
16
|
|
|
@@ -20,6 +21,7 @@ class ContainerNames(NamedTuple):
|
|
|
20
21
|
|
|
21
22
|
def check_docker_daemon_running() -> None:
|
|
22
23
|
"""Checks if the Docker daemon is running. Raises DockerDaemonNotRunningError if not."""
|
|
24
|
+
console = Console()
|
|
23
25
|
try:
|
|
24
26
|
subprocess.run(
|
|
25
27
|
["docker", "info"],
|
|
@@ -27,7 +29,13 @@ def check_docker_daemon_running() -> None:
|
|
|
27
29
|
text=True,
|
|
28
30
|
check=True,
|
|
29
31
|
)
|
|
32
|
+
return
|
|
33
|
+
except subprocess.CalledProcessError:
|
|
34
|
+
console.info("Docker daemon is not running. Checking if colima is available")
|
|
35
|
+
try:
|
|
36
|
+
subprocess.run(["devenv", "colima", "start"], check=True)
|
|
30
37
|
except subprocess.CalledProcessError as e:
|
|
38
|
+
console.failure("Failed to start colima")
|
|
31
39
|
raise DockerDaemonNotRunningError from e
|
|
32
40
|
|
|
33
41
|
|
|
@@ -10,6 +10,7 @@ import xmlrpc.client
|
|
|
10
10
|
from enum import IntEnum
|
|
11
11
|
from typing import TypedDict
|
|
12
12
|
|
|
13
|
+
import yaml
|
|
13
14
|
from sentry_sdk import capture_exception
|
|
14
15
|
from supervisor.options import ServerOptions
|
|
15
16
|
|
|
@@ -91,24 +92,81 @@ class ProcessInfo(TypedDict):
|
|
|
91
92
|
group: str
|
|
92
93
|
|
|
93
94
|
|
|
95
|
+
class SupervisorProgramConfig(TypedDict, total=False):
|
|
96
|
+
"""Supervisor program configuration."""
|
|
97
|
+
|
|
98
|
+
command: str
|
|
99
|
+
autostart: str | bool
|
|
100
|
+
autorestart: str | bool
|
|
101
|
+
directory: str
|
|
102
|
+
environment: str
|
|
103
|
+
user: str
|
|
104
|
+
priority: str | int
|
|
105
|
+
startsecs: str | int
|
|
106
|
+
startretries: str | int
|
|
107
|
+
stdout_logfile: str
|
|
108
|
+
stderr_logfile: str
|
|
109
|
+
redirect_stderr: str | bool
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
ProgramData = dict[str, SupervisorProgramConfig]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# Default values for supervisor program configuration
|
|
116
|
+
SUPERVISOR_PROGRAM_DEFAULTS = {
|
|
117
|
+
"autostart": "false",
|
|
118
|
+
"autorestart": "true",
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
94
122
|
class SupervisorManager:
|
|
95
|
-
def __init__(
|
|
123
|
+
def __init__(
|
|
124
|
+
self,
|
|
125
|
+
service_name: str,
|
|
126
|
+
service_config_path: str,
|
|
127
|
+
) -> None:
|
|
96
128
|
self.service_name = service_name
|
|
97
|
-
if not os.path.exists(config_file_path):
|
|
98
|
-
raise SupervisorConfigError(
|
|
99
|
-
f"Config file {config_file_path} does not exist"
|
|
100
|
-
)
|
|
101
129
|
self.socket_path = os.path.join(
|
|
102
130
|
DEVSERVICES_SUPERVISOR_CONFIG_DIR, f"{service_name}.sock"
|
|
103
131
|
)
|
|
104
|
-
self.config_file_path = self._extend_config_file(config_file_path)
|
|
105
132
|
|
|
106
|
-
|
|
107
|
-
|
|
133
|
+
# Load service config and extract x-programs data
|
|
134
|
+
if os.path.exists(service_config_path):
|
|
135
|
+
with open(service_config_path, "r", encoding="utf-8") as stream:
|
|
136
|
+
config = yaml.safe_load(stream)
|
|
137
|
+
else:
|
|
138
|
+
raise SupervisorConfigError(f"Config file {service_config_path} not found")
|
|
139
|
+
|
|
140
|
+
if config is None:
|
|
141
|
+
raise SupervisorConfigError(f"Config file {service_config_path} is empty")
|
|
108
142
|
|
|
143
|
+
programs_data: ProgramData = config.get("x-programs", {})
|
|
144
|
+
|
|
145
|
+
self.has_programs = len(programs_data.keys()) > 0
|
|
146
|
+
|
|
147
|
+
# Generate supervisor config file from x-programs data
|
|
148
|
+
self.config_file_path = self._generate_config_from_programs_data(programs_data)
|
|
149
|
+
|
|
150
|
+
def _generate_config_from_programs_data(self, programs_data: ProgramData) -> str:
|
|
109
151
|
config = configparser.ConfigParser()
|
|
110
152
|
|
|
111
|
-
|
|
153
|
+
# Add program sections
|
|
154
|
+
for program_name, program_config in programs_data.items():
|
|
155
|
+
section_name = f"program:{program_name}"
|
|
156
|
+
config[section_name] = {}
|
|
157
|
+
|
|
158
|
+
# Apply defaults for any missing configuration values
|
|
159
|
+
program_config_with_defaults = {
|
|
160
|
+
**SUPERVISOR_PROGRAM_DEFAULTS,
|
|
161
|
+
**program_config,
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
for key, value in program_config_with_defaults.items():
|
|
165
|
+
if isinstance(value, bool):
|
|
166
|
+
config[section_name][key] = str(value).lower()
|
|
167
|
+
else:
|
|
168
|
+
config[section_name][key] = str(value)
|
|
169
|
+
|
|
112
170
|
os.makedirs(DEVSERVICES_SUPERVISOR_CONFIG_DIR, exist_ok=True)
|
|
113
171
|
|
|
114
172
|
# Set unix http server to use the socket path
|
|
@@ -129,13 +187,13 @@ class SupervisorManager:
|
|
|
129
187
|
"supervisor.rpcinterface_factory": "supervisor.rpcinterface:make_main_rpcinterface"
|
|
130
188
|
}
|
|
131
189
|
|
|
132
|
-
|
|
190
|
+
config_file_path = os.path.join(
|
|
133
191
|
DEVSERVICES_SUPERVISOR_CONFIG_DIR, f"{self.service_name}.processes.conf"
|
|
134
192
|
)
|
|
135
|
-
with open(
|
|
193
|
+
with open(config_file_path, "w") as f:
|
|
136
194
|
config.write(f)
|
|
137
195
|
|
|
138
|
-
return
|
|
196
|
+
return config_file_path
|
|
139
197
|
|
|
140
198
|
def _get_rpc_client(self) -> xmlrpc.client.ServerProxy:
|
|
141
199
|
"""Get or create an XML-RPC client that connects to the supervisor daemon."""
|
|
@@ -317,6 +375,9 @@ class SupervisorManager:
|
|
|
317
375
|
def get_all_process_info(self) -> dict[str, ProcessInfo]:
|
|
318
376
|
"""Get status information for all supervisor programs."""
|
|
319
377
|
# Check if supervisor client is up first, return empty list if down
|
|
378
|
+
if not self.has_programs:
|
|
379
|
+
return {}
|
|
380
|
+
|
|
320
381
|
try:
|
|
321
382
|
client = self._get_rpc_client()
|
|
322
383
|
client.supervisor.getState()
|
|
@@ -8,7 +8,6 @@ from pathlib import Path
|
|
|
8
8
|
import yaml
|
|
9
9
|
|
|
10
10
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
11
|
-
from devservices.constants import PROGRAMS_CONF_FILE_NAME
|
|
12
11
|
|
|
13
12
|
TESTING_DIR = os.path.abspath(os.path.dirname(__file__))
|
|
14
13
|
|
|
@@ -27,14 +26,6 @@ def create_config_file(
|
|
|
27
26
|
yaml.dump(config, f, sort_keys=False, default_flow_style=False)
|
|
28
27
|
|
|
29
28
|
|
|
30
|
-
def create_programs_conf_file(tmp_path: Path, config: str) -> None:
|
|
31
|
-
devservices_dir = Path(tmp_path, DEVSERVICES_DIR_NAME)
|
|
32
|
-
devservices_dir.mkdir(parents=True, exist_ok=True)
|
|
33
|
-
tmp_file = Path(devservices_dir, PROGRAMS_CONF_FILE_NAME)
|
|
34
|
-
with tmp_file.open("w") as f:
|
|
35
|
-
f.write(config)
|
|
36
|
-
|
|
37
|
-
|
|
38
29
|
def run_git_command(command: list[str], cwd: Path) -> None:
|
|
39
30
|
subprocess.run(
|
|
40
31
|
["git", *command], cwd=cwd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|