devservices 1.0.6__tar.gz → 1.0.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {devservices-1.0.6 → devservices-1.0.7}/PKG-INFO +1 -1
- {devservices-1.0.6 → devservices-1.0.7}/README.md +1 -1
- {devservices-1.0.6 → devservices-1.0.7}/devservices/commands/down.py +4 -4
- {devservices-1.0.6 → devservices-1.0.7}/devservices/commands/logs.py +1 -1
- devservices-1.0.7/devservices/commands/purge.py +94 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/commands/status.py +1 -1
- {devservices-1.0.6 → devservices-1.0.7}/devservices/commands/up.py +33 -6
- {devservices-1.0.6 → devservices-1.0.7}/devservices/constants.py +2 -1
- {devservices-1.0.6 → devservices-1.0.7}/devservices/exceptions.py +11 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/dependencies.py +49 -0
- devservices-1.0.7/devservices/utils/docker.py +219 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/docker_compose.py +44 -5
- {devservices-1.0.6 → devservices-1.0.7}/devservices.egg-info/PKG-INFO +1 -1
- {devservices-1.0.6 → devservices-1.0.7}/pyproject.toml +1 -1
- devservices-1.0.7/tests/commands/test_purge.py +517 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/commands/test_up.py +275 -2
- {devservices-1.0.6 → devservices-1.0.7}/tests/utils/test_dependencies.py +48 -0
- devservices-1.0.7/tests/utils/test_docker.py +493 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/utils/test_docker_compose.py +165 -99
- devservices-1.0.6/devservices/commands/purge.py +0 -78
- devservices-1.0.6/devservices/utils/docker.py +0 -88
- devservices-1.0.6/tests/commands/test_purge.py +0 -214
- devservices-1.0.6/tests/utils/test_docker.py +0 -182
- {devservices-1.0.6 → devservices-1.0.7}/LICENSE.md +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/__init__.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/commands/__init__.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/commands/list_dependencies.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/commands/list_services.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/commands/update.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/configs/service_config.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/main.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/__init__.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/check_for_update.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/console.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/devenv.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/file_lock.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/install_binary.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/services.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices/utils/state.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices.egg-info/SOURCES.txt +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices.egg-info/dependency_links.txt +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices.egg-info/entry_points.txt +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices.egg-info/requires.txt +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/devservices.egg-info/top_level.txt +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/setup.cfg +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/testing/__init__.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/testing/utils.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/__init__.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/commands/test_down.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/commands/test_list_dependencies.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/commands/test_list_services.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/commands/test_logs.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/commands/test_status.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/commands/test_update.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/configs/test_service_config.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/conftest.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/utils/test_check_for_update.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/utils/test_install_binary.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/utils/test_services.py +0 -0
- {devservices-1.0.6 → devservices-1.0.7}/tests/utils/test_state.py +0 -0
|
@@ -14,7 +14,6 @@ from devservices.constants import DEPENDENCY_CONFIG_VERSION
|
|
|
14
14
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
|
|
15
15
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
|
|
16
16
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
17
|
-
from devservices.constants import DOCKER_COMPOSE_COMMAND_LENGTH
|
|
18
17
|
from devservices.exceptions import ConfigError
|
|
19
18
|
from devservices.exceptions import DependencyError
|
|
20
19
|
from devservices.exceptions import DockerComposeError
|
|
@@ -24,6 +23,7 @@ from devservices.utils.console import Status
|
|
|
24
23
|
from devservices.utils.dependencies import get_non_shared_remote_dependencies
|
|
25
24
|
from devservices.utils.dependencies import install_and_verify_dependencies
|
|
26
25
|
from devservices.utils.dependencies import InstalledRemoteDependency
|
|
26
|
+
from devservices.utils.docker_compose import DockerComposeCommand
|
|
27
27
|
from devservices.utils.docker_compose import get_docker_compose_commands_to_run
|
|
28
28
|
from devservices.utils.docker_compose import run_cmd
|
|
29
29
|
from devservices.utils.services import find_matching_service
|
|
@@ -111,12 +111,12 @@ def down(args: Namespace) -> None:
|
|
|
111
111
|
|
|
112
112
|
|
|
113
113
|
def _bring_down_dependency(
|
|
114
|
-
cmd:
|
|
114
|
+
cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
|
|
115
115
|
) -> subprocess.CompletedProcess[str]:
|
|
116
116
|
# TODO: Get rid of these constants, we need a smarter way to determine the containers being brought down
|
|
117
|
-
for dependency in cmd
|
|
117
|
+
for dependency in cmd.services:
|
|
118
118
|
status.info(f"Stopping {dependency}")
|
|
119
|
-
return run_cmd(cmd, current_env)
|
|
119
|
+
return run_cmd(cmd.full_command, current_env)
|
|
120
120
|
|
|
121
121
|
|
|
122
122
|
def _down(
|
|
@@ -114,7 +114,7 @@ def _logs(
|
|
|
114
114
|
|
|
115
115
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
116
116
|
futures = [
|
|
117
|
-
executor.submit(run_cmd, cmd, current_env)
|
|
117
|
+
executor.submit(run_cmd, cmd.full_command, current_env)
|
|
118
118
|
for cmd in docker_compose_commands
|
|
119
119
|
]
|
|
120
120
|
for future in concurrent.futures.as_completed(futures):
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
from argparse import _SubParsersAction
|
|
6
|
+
from argparse import ArgumentParser
|
|
7
|
+
from argparse import Namespace
|
|
8
|
+
|
|
9
|
+
from devservices.constants import DEVSERVICES_CACHE_DIR
|
|
10
|
+
from devservices.constants import DEVSERVICES_ORCHESTRATOR_LABEL
|
|
11
|
+
from devservices.constants import DOCKER_NETWORK_NAME
|
|
12
|
+
from devservices.exceptions import DockerDaemonNotRunningError
|
|
13
|
+
from devservices.exceptions import DockerError
|
|
14
|
+
from devservices.utils.console import Console
|
|
15
|
+
from devservices.utils.console import Status
|
|
16
|
+
from devservices.utils.docker import get_matching_containers
|
|
17
|
+
from devservices.utils.docker import get_matching_networks
|
|
18
|
+
from devservices.utils.docker import get_volumes_for_containers
|
|
19
|
+
from devservices.utils.docker import remove_docker_resources
|
|
20
|
+
from devservices.utils.docker import stop_containers
|
|
21
|
+
from devservices.utils.state import State
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def add_parser(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
25
|
+
parser = subparsers.add_parser("purge", help="Purge the local devservices cache")
|
|
26
|
+
parser.set_defaults(func=purge)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def purge(_args: Namespace) -> None:
|
|
30
|
+
"""Purge the local devservices state and cache and remove all devservices containers and volumes."""
|
|
31
|
+
console = Console()
|
|
32
|
+
|
|
33
|
+
if os.path.exists(DEVSERVICES_CACHE_DIR):
|
|
34
|
+
try:
|
|
35
|
+
shutil.rmtree(DEVSERVICES_CACHE_DIR)
|
|
36
|
+
except PermissionError as e:
|
|
37
|
+
console.failure(f"Failed to purge cache: {e}")
|
|
38
|
+
exit(1)
|
|
39
|
+
state = State()
|
|
40
|
+
state.clear_state()
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
devservices_containers = get_matching_containers(DEVSERVICES_ORCHESTRATOR_LABEL)
|
|
44
|
+
except DockerDaemonNotRunningError as e:
|
|
45
|
+
console.warning(str(e))
|
|
46
|
+
return
|
|
47
|
+
except DockerError as de:
|
|
48
|
+
console.failure(f"Failed to get devservices containers {de.stderr}")
|
|
49
|
+
exit(1)
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
devservices_volumes = get_volumes_for_containers(devservices_containers)
|
|
53
|
+
except DockerError as e:
|
|
54
|
+
console.failure(f"Failed to get devservices volumes {e.stderr}")
|
|
55
|
+
exit(1)
|
|
56
|
+
|
|
57
|
+
with Status(
|
|
58
|
+
lambda: console.warning("Stopping all devservices containers"),
|
|
59
|
+
lambda: console.success("All devservices containers have been stopped"),
|
|
60
|
+
):
|
|
61
|
+
try:
|
|
62
|
+
stop_containers(devservices_containers, should_remove=True)
|
|
63
|
+
except DockerError as e:
|
|
64
|
+
console.failure(f"Failed to stop devservices containers {e.stderr}")
|
|
65
|
+
exit(1)
|
|
66
|
+
|
|
67
|
+
console.warning("Removing any devservices docker volumes")
|
|
68
|
+
if len(devservices_volumes) == 0:
|
|
69
|
+
console.success("No devservices volumes found to remove")
|
|
70
|
+
else:
|
|
71
|
+
try:
|
|
72
|
+
remove_docker_resources("volume", list(devservices_volumes))
|
|
73
|
+
console.success("All devservices volumes removed")
|
|
74
|
+
except DockerError as e:
|
|
75
|
+
# We don't want to exit here since we still want to try to remove the networks
|
|
76
|
+
console.failure(f"Failed to remove devservices volumes {e.stderr}")
|
|
77
|
+
|
|
78
|
+
console.warning("Removing any devservices networks")
|
|
79
|
+
try:
|
|
80
|
+
devservices_networks = get_matching_networks(DOCKER_NETWORK_NAME)
|
|
81
|
+
except DockerError as e:
|
|
82
|
+
console.failure(f"Failed to get devservices networks {e.stderr}")
|
|
83
|
+
exit(1)
|
|
84
|
+
if len(devservices_networks) == 0:
|
|
85
|
+
console.success("No devservices networks found to remove")
|
|
86
|
+
else:
|
|
87
|
+
try:
|
|
88
|
+
remove_docker_resources("network", devservices_networks)
|
|
89
|
+
console.success("All devservices networks removed")
|
|
90
|
+
except DockerError as e:
|
|
91
|
+
console.failure(f"Failed to remove devservices networks {e.stderr}")
|
|
92
|
+
exit(1)
|
|
93
|
+
|
|
94
|
+
console.success("The local devservices cache and state has been purged")
|
|
@@ -152,7 +152,7 @@ def _status(
|
|
|
152
152
|
|
|
153
153
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
154
154
|
futures = [
|
|
155
|
-
executor.submit(run_cmd, cmd, current_env)
|
|
155
|
+
executor.submit(run_cmd, cmd.full_command, current_env)
|
|
156
156
|
for cmd in docker_compose_commands
|
|
157
157
|
]
|
|
158
158
|
for future in concurrent.futures.as_completed(futures):
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import concurrent.futures
|
|
3
4
|
import os
|
|
4
5
|
import subprocess
|
|
5
6
|
from argparse import _SubParsersAction
|
|
@@ -13,8 +14,8 @@ from devservices.constants import DEPENDENCY_CONFIG_VERSION
|
|
|
13
14
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
|
|
14
15
|
from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
|
|
15
16
|
from devservices.constants import DEVSERVICES_DIR_NAME
|
|
16
|
-
from devservices.constants import DOCKER_COMPOSE_COMMAND_LENGTH
|
|
17
17
|
from devservices.exceptions import ConfigError
|
|
18
|
+
from devservices.exceptions import ContainerHealthcheckFailedError
|
|
18
19
|
from devservices.exceptions import DependencyError
|
|
19
20
|
from devservices.exceptions import DockerComposeError
|
|
20
21
|
from devservices.exceptions import ModeDoesNotExistError
|
|
@@ -24,6 +25,9 @@ from devservices.utils.console import Status
|
|
|
24
25
|
from devservices.utils.dependencies import construct_dependency_graph
|
|
25
26
|
from devservices.utils.dependencies import install_and_verify_dependencies
|
|
26
27
|
from devservices.utils.dependencies import InstalledRemoteDependency
|
|
28
|
+
from devservices.utils.docker import check_all_containers_healthy
|
|
29
|
+
from devservices.utils.docker_compose import DockerComposeCommand
|
|
30
|
+
from devservices.utils.docker_compose import get_container_names_for_project
|
|
27
31
|
from devservices.utils.docker_compose import get_docker_compose_commands_to_run
|
|
28
32
|
from devservices.utils.docker_compose import run_cmd
|
|
29
33
|
from devservices.utils.services import find_matching_service
|
|
@@ -101,12 +105,12 @@ def up(args: Namespace) -> None:
|
|
|
101
105
|
|
|
102
106
|
|
|
103
107
|
def _bring_up_dependency(
|
|
104
|
-
cmd:
|
|
108
|
+
cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
|
|
105
109
|
) -> subprocess.CompletedProcess[str]:
|
|
106
110
|
# TODO: Get rid of these constants, we need a smarter way to determine the containers being brought up
|
|
107
|
-
for dependency in cmd
|
|
111
|
+
for dependency in cmd.services:
|
|
108
112
|
status.info(f"Starting {dependency}")
|
|
109
|
-
return run_cmd(cmd, current_env)
|
|
113
|
+
return run_cmd(cmd.full_command, current_env)
|
|
110
114
|
|
|
111
115
|
|
|
112
116
|
def _up(
|
|
@@ -128,7 +132,7 @@ def _up(
|
|
|
128
132
|
current_env[
|
|
129
133
|
DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
|
|
130
134
|
] = relative_local_dependency_directory
|
|
131
|
-
options = ["-d"]
|
|
135
|
+
options = ["-d", "--pull", "always"]
|
|
132
136
|
dependency_graph = construct_dependency_graph(service, modes=modes)
|
|
133
137
|
starting_order = dependency_graph.get_starting_order()
|
|
134
138
|
sorted_remote_dependencies = sorted(
|
|
@@ -144,8 +148,31 @@ def _up(
|
|
|
144
148
|
mode_dependencies=mode_dependencies,
|
|
145
149
|
)
|
|
146
150
|
|
|
151
|
+
containers_to_check = []
|
|
152
|
+
with concurrent.futures.ThreadPoolExecutor() as dependency_executor:
|
|
153
|
+
futures = [
|
|
154
|
+
dependency_executor.submit(_bring_up_dependency, cmd, current_env, status)
|
|
155
|
+
for cmd in docker_compose_commands
|
|
156
|
+
]
|
|
157
|
+
for future in concurrent.futures.as_completed(futures):
|
|
158
|
+
_ = future.result()
|
|
159
|
+
|
|
147
160
|
for cmd in docker_compose_commands:
|
|
148
|
-
|
|
161
|
+
try:
|
|
162
|
+
container_names = get_container_names_for_project(
|
|
163
|
+
cmd.project_name, cmd.config_path
|
|
164
|
+
)
|
|
165
|
+
containers_to_check.extend(container_names)
|
|
166
|
+
except DockerComposeError as dce:
|
|
167
|
+
status.failure(
|
|
168
|
+
f"Failed to get containers to healthcheck for {cmd.project_name}: {dce.stderr}"
|
|
169
|
+
)
|
|
170
|
+
exit(1)
|
|
171
|
+
try:
|
|
172
|
+
check_all_containers_healthy(status, containers_to_check)
|
|
173
|
+
except ContainerHealthcheckFailedError as e:
|
|
174
|
+
status.failure(str(e))
|
|
175
|
+
exit(1)
|
|
149
176
|
|
|
150
177
|
|
|
151
178
|
def _create_devservices_network() -> None:
|
|
@@ -14,7 +14,6 @@ DEVSERVICES_DEPENDENCIES_CACHE_DIR = os.path.join(DEVSERVICES_CACHE_DIR, "depend
|
|
|
14
14
|
DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY = "DEVSERVICES_DEPENDENCIES_CACHE_DIR"
|
|
15
15
|
STATE_DB_FILE = os.path.join(DEVSERVICES_LOCAL_DIR, "state")
|
|
16
16
|
DEVSERVICES_ORCHESTRATOR_LABEL = "orchestrator=devservices"
|
|
17
|
-
DOCKER_COMPOSE_COMMAND_LENGTH = 7
|
|
18
17
|
|
|
19
18
|
DEPENDENCY_CONFIG_VERSION = "v1"
|
|
20
19
|
DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS = {
|
|
@@ -38,3 +37,5 @@ DEVSERVICES_LATEST_VERSION_CACHE_FILE = os.path.join(
|
|
|
38
37
|
DEVSERVICES_CACHE_DIR, "latest_version.txt"
|
|
39
38
|
)
|
|
40
39
|
DEVSERVICES_LATEST_VERSION_CACHE_TTL = timedelta(minutes=15)
|
|
40
|
+
HEALTHCHECK_TIMEOUT = 30
|
|
41
|
+
HEALTHCHECK_INTERVAL = 5
|
|
@@ -127,3 +127,14 @@ class FailedToSetGitConfigError(GitConfigError):
|
|
|
127
127
|
"""Raised when a git config cannot be set."""
|
|
128
128
|
|
|
129
129
|
pass
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class ContainerHealthcheckFailedError(Exception):
|
|
133
|
+
"""Raised when a container is not healthy."""
|
|
134
|
+
|
|
135
|
+
def __init__(self, container_name: str, timeout: int):
|
|
136
|
+
self.container_name = container_name
|
|
137
|
+
self.timeout = timeout
|
|
138
|
+
|
|
139
|
+
def __str__(self) -> str:
|
|
140
|
+
return f"Container {self.container_name} did not become healthy within {self.timeout} seconds."
|
|
@@ -12,6 +12,8 @@ from dataclasses import dataclass
|
|
|
12
12
|
from typing import TextIO
|
|
13
13
|
from typing import TypeGuard
|
|
14
14
|
|
|
15
|
+
from sentry_sdk import set_context
|
|
16
|
+
|
|
15
17
|
from devservices.configs.service_config import Dependency
|
|
16
18
|
from devservices.configs.service_config import load_service_config_from_file
|
|
17
19
|
from devservices.configs.service_config import RemoteConfig
|
|
@@ -36,6 +38,17 @@ from devservices.utils.services import find_matching_service
|
|
|
36
38
|
from devservices.utils.services import Service
|
|
37
39
|
from devservices.utils.state import State
|
|
38
40
|
|
|
41
|
+
RELEVANT_GIT_CONFIG_KEYS = [
|
|
42
|
+
"init.defaultbranch",
|
|
43
|
+
"core.sparsecheckout",
|
|
44
|
+
"remote.origin.url",
|
|
45
|
+
"remote.origin.fetch",
|
|
46
|
+
"remote.origin.promisor",
|
|
47
|
+
"remote.origin.partialclonefilter",
|
|
48
|
+
"protocol.version",
|
|
49
|
+
"extensions.partialclone",
|
|
50
|
+
]
|
|
51
|
+
|
|
39
52
|
|
|
40
53
|
class DependencyGraph:
|
|
41
54
|
def __init__(self) -> None:
|
|
@@ -149,6 +162,28 @@ class GitConfigManager:
|
|
|
149
162
|
if self.sparse_pattern:
|
|
150
163
|
self.sparse_checkout_manager.set_sparse_checkout(self.sparse_pattern)
|
|
151
164
|
|
|
165
|
+
def get_relevant_config(self) -> dict[str, str]:
|
|
166
|
+
"""
|
|
167
|
+
Get the relevant git config entries (to avoid logging sensitive information)
|
|
168
|
+
"""
|
|
169
|
+
git_config = (
|
|
170
|
+
subprocess.check_output(
|
|
171
|
+
["git", "config", "--list"],
|
|
172
|
+
cwd=self.repo_dir,
|
|
173
|
+
stderr=subprocess.PIPE,
|
|
174
|
+
)
|
|
175
|
+
.decode()
|
|
176
|
+
.strip()
|
|
177
|
+
)
|
|
178
|
+
git_config_dict = dict()
|
|
179
|
+
for line in git_config.split("\n"):
|
|
180
|
+
if not line:
|
|
181
|
+
continue
|
|
182
|
+
key, value = line.split("=")
|
|
183
|
+
if key in RELEVANT_GIT_CONFIG_KEYS:
|
|
184
|
+
git_config_dict[key] = value
|
|
185
|
+
return git_config_dict
|
|
186
|
+
|
|
152
187
|
def _set_config(self, key: str, value: str) -> None:
|
|
153
188
|
"""
|
|
154
189
|
Set a git config option for the repo
|
|
@@ -411,12 +446,15 @@ def _update_dependency(
|
|
|
411
446
|
repo_link=dependency.repo_link,
|
|
412
447
|
branch=dependency.branch,
|
|
413
448
|
) from e
|
|
449
|
+
|
|
414
450
|
try:
|
|
415
451
|
_run_command(
|
|
416
452
|
["git", "fetch", "origin", dependency.branch, "--filter=blob:none"],
|
|
417
453
|
cwd=dependency_repo_dir,
|
|
418
454
|
)
|
|
419
455
|
except subprocess.CalledProcessError as e:
|
|
456
|
+
# Try to set the git config context to help with debugging
|
|
457
|
+
_try_set_git_config_context(git_config_manager)
|
|
420
458
|
raise DependencyError(
|
|
421
459
|
repo_name=dependency.repo_name,
|
|
422
460
|
repo_link=dependency.repo_link,
|
|
@@ -536,6 +574,17 @@ def _run_command(
|
|
|
536
574
|
subprocess.run(cmd, cwd=cwd, check=True, stdout=stdout, stderr=subprocess.DEVNULL)
|
|
537
575
|
|
|
538
576
|
|
|
577
|
+
def _try_set_git_config_context(
|
|
578
|
+
git_config_manager: GitConfigManager,
|
|
579
|
+
) -> None:
|
|
580
|
+
try:
|
|
581
|
+
git_config = git_config_manager.get_relevant_config()
|
|
582
|
+
set_context("git_config", git_config)
|
|
583
|
+
except subprocess.CalledProcessError as e:
|
|
584
|
+
logger = logging.getLogger(LOGGER_NAME)
|
|
585
|
+
logger.exception(e)
|
|
586
|
+
|
|
587
|
+
|
|
539
588
|
def get_remote_dependency_config(remote_config: RemoteConfig) -> ServiceConfig:
|
|
540
589
|
dependency_repo_dir = os.path.join(
|
|
541
590
|
DEVSERVICES_DEPENDENCIES_CACHE_DIR,
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import concurrent.futures
|
|
4
|
+
import subprocess
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from devservices.constants import HEALTHCHECK_INTERVAL
|
|
8
|
+
from devservices.constants import HEALTHCHECK_TIMEOUT
|
|
9
|
+
from devservices.exceptions import ContainerHealthcheckFailedError
|
|
10
|
+
from devservices.exceptions import DockerDaemonNotRunningError
|
|
11
|
+
from devservices.exceptions import DockerError
|
|
12
|
+
from devservices.utils.console import Status
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def check_docker_daemon_running() -> None:
|
|
16
|
+
"""Checks if the Docker daemon is running. Raises DockerDaemonNotRunningError if not."""
|
|
17
|
+
try:
|
|
18
|
+
subprocess.run(
|
|
19
|
+
["docker", "info"],
|
|
20
|
+
capture_output=True,
|
|
21
|
+
text=True,
|
|
22
|
+
check=True,
|
|
23
|
+
)
|
|
24
|
+
except subprocess.CalledProcessError as e:
|
|
25
|
+
raise DockerDaemonNotRunningError from e
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def check_all_containers_healthy(status: Status, containers: list[str]) -> None:
|
|
29
|
+
"""Ensures all containers are healthy."""
|
|
30
|
+
with concurrent.futures.ThreadPoolExecutor() as healthcheck_executor:
|
|
31
|
+
futures = [
|
|
32
|
+
healthcheck_executor.submit(wait_for_healthy, container, status)
|
|
33
|
+
for container in containers
|
|
34
|
+
]
|
|
35
|
+
for future in concurrent.futures.as_completed(futures):
|
|
36
|
+
future.result()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def wait_for_healthy(container_name: str, status: Status) -> None:
|
|
40
|
+
"""
|
|
41
|
+
Polls a Docker container's health status until it becomes healthy or a timeout is reached.
|
|
42
|
+
"""
|
|
43
|
+
start = time.time()
|
|
44
|
+
while time.time() - start < HEALTHCHECK_TIMEOUT:
|
|
45
|
+
# Run docker inspect to get the container's health status
|
|
46
|
+
try:
|
|
47
|
+
# For containers with no healthchecks, the output will be "unknown"
|
|
48
|
+
result = subprocess.check_output(
|
|
49
|
+
[
|
|
50
|
+
"docker",
|
|
51
|
+
"inspect",
|
|
52
|
+
"-f",
|
|
53
|
+
"{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}",
|
|
54
|
+
container_name,
|
|
55
|
+
],
|
|
56
|
+
stderr=subprocess.DEVNULL,
|
|
57
|
+
text=True,
|
|
58
|
+
).strip()
|
|
59
|
+
except subprocess.CalledProcessError as e:
|
|
60
|
+
raise DockerError(
|
|
61
|
+
command=f"docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' {container_name}",
|
|
62
|
+
returncode=e.returncode,
|
|
63
|
+
stdout=e.stdout,
|
|
64
|
+
stderr=e.stderr,
|
|
65
|
+
) from e
|
|
66
|
+
|
|
67
|
+
if result == "healthy":
|
|
68
|
+
return
|
|
69
|
+
elif result == "unknown":
|
|
70
|
+
status.warning(
|
|
71
|
+
f"WARNING: Container {container_name} does not have a healthcheck"
|
|
72
|
+
)
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
# If not healthy, wait and try again
|
|
76
|
+
time.sleep(HEALTHCHECK_INTERVAL)
|
|
77
|
+
|
|
78
|
+
raise ContainerHealthcheckFailedError(container_name, HEALTHCHECK_TIMEOUT)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def get_matching_containers(label: str) -> list[str]:
|
|
82
|
+
"""
|
|
83
|
+
Returns a list of container names with the given label
|
|
84
|
+
"""
|
|
85
|
+
check_docker_daemon_running()
|
|
86
|
+
try:
|
|
87
|
+
return (
|
|
88
|
+
subprocess.check_output(
|
|
89
|
+
[
|
|
90
|
+
"docker",
|
|
91
|
+
"ps",
|
|
92
|
+
"-a",
|
|
93
|
+
"-q",
|
|
94
|
+
"--filter",
|
|
95
|
+
f"label={label}",
|
|
96
|
+
],
|
|
97
|
+
text=True,
|
|
98
|
+
stderr=subprocess.DEVNULL,
|
|
99
|
+
)
|
|
100
|
+
.strip()
|
|
101
|
+
.splitlines()
|
|
102
|
+
)
|
|
103
|
+
except subprocess.CalledProcessError as e:
|
|
104
|
+
raise DockerError(
|
|
105
|
+
command=f"docker ps -q --filter label={label}",
|
|
106
|
+
returncode=e.returncode,
|
|
107
|
+
stdout=e.stdout,
|
|
108
|
+
stderr=e.stderr,
|
|
109
|
+
) from e
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def get_matching_networks(name: str) -> list[str]:
|
|
113
|
+
"""
|
|
114
|
+
Returns a list of network IDs with the given name
|
|
115
|
+
"""
|
|
116
|
+
check_docker_daemon_running()
|
|
117
|
+
try:
|
|
118
|
+
return (
|
|
119
|
+
subprocess.check_output(
|
|
120
|
+
[
|
|
121
|
+
"docker",
|
|
122
|
+
"network",
|
|
123
|
+
"ls",
|
|
124
|
+
"--filter",
|
|
125
|
+
f"name={name}",
|
|
126
|
+
"--format",
|
|
127
|
+
"{{.ID}}",
|
|
128
|
+
],
|
|
129
|
+
text=True,
|
|
130
|
+
stderr=subprocess.DEVNULL,
|
|
131
|
+
)
|
|
132
|
+
.strip()
|
|
133
|
+
.splitlines()
|
|
134
|
+
)
|
|
135
|
+
except subprocess.CalledProcessError as e:
|
|
136
|
+
raise DockerError(
|
|
137
|
+
command=f"docker network ls --filter name={name} --format '{{.ID}}'",
|
|
138
|
+
returncode=e.returncode,
|
|
139
|
+
stdout=e.stdout,
|
|
140
|
+
stderr=e.stderr,
|
|
141
|
+
) from e
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def get_volumes_for_containers(containers: list[str]) -> set[str]:
|
|
145
|
+
"""
|
|
146
|
+
Returns a set of volume names for the given containers.
|
|
147
|
+
"""
|
|
148
|
+
if len(containers) == 0:
|
|
149
|
+
return set()
|
|
150
|
+
try:
|
|
151
|
+
return {
|
|
152
|
+
volume
|
|
153
|
+
for volume in subprocess.check_output(
|
|
154
|
+
[
|
|
155
|
+
"docker",
|
|
156
|
+
"inspect",
|
|
157
|
+
"--format",
|
|
158
|
+
"{{ range .Mounts }}{{ .Name }}\n{{ end }}",
|
|
159
|
+
*containers,
|
|
160
|
+
],
|
|
161
|
+
text=True,
|
|
162
|
+
stderr=subprocess.DEVNULL,
|
|
163
|
+
)
|
|
164
|
+
.strip()
|
|
165
|
+
.splitlines()
|
|
166
|
+
if volume
|
|
167
|
+
}
|
|
168
|
+
except subprocess.CalledProcessError as e:
|
|
169
|
+
raise DockerError(
|
|
170
|
+
command=f"docker inspect --format '{{ range .Mounts }}{{ .Name }}\n{{ end }}' {' '.join(containers)}",
|
|
171
|
+
returncode=e.returncode,
|
|
172
|
+
stdout=e.stdout,
|
|
173
|
+
stderr=e.stderr,
|
|
174
|
+
) from e
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def stop_containers(containers: list[str], should_remove: bool = False) -> None:
|
|
178
|
+
"""
|
|
179
|
+
Stops the given containers.
|
|
180
|
+
If should_remove is True, the containers will be removed.
|
|
181
|
+
"""
|
|
182
|
+
if len(containers) == 0:
|
|
183
|
+
return
|
|
184
|
+
try:
|
|
185
|
+
subprocess.run(
|
|
186
|
+
["docker", "stop"] + containers,
|
|
187
|
+
check=True,
|
|
188
|
+
stdout=subprocess.DEVNULL,
|
|
189
|
+
stderr=subprocess.DEVNULL,
|
|
190
|
+
)
|
|
191
|
+
except subprocess.CalledProcessError as e:
|
|
192
|
+
raise DockerError(
|
|
193
|
+
command=f"docker stop {' '.join(containers)}",
|
|
194
|
+
returncode=e.returncode,
|
|
195
|
+
stdout=e.stdout,
|
|
196
|
+
stderr=e.stderr,
|
|
197
|
+
) from e
|
|
198
|
+
if should_remove:
|
|
199
|
+
remove_docker_resources("container", containers)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def remove_docker_resources(resource_type: str, resources: list[str]) -> None:
|
|
203
|
+
"""
|
|
204
|
+
Removes the given Docker resources.
|
|
205
|
+
"""
|
|
206
|
+
try:
|
|
207
|
+
subprocess.run(
|
|
208
|
+
["docker", resource_type, "rm", *resources],
|
|
209
|
+
check=True,
|
|
210
|
+
stdout=subprocess.DEVNULL,
|
|
211
|
+
stderr=subprocess.DEVNULL,
|
|
212
|
+
)
|
|
213
|
+
except subprocess.CalledProcessError as e:
|
|
214
|
+
raise DockerError(
|
|
215
|
+
command=f"docker {resource_type} rm {' '.join(resources)}",
|
|
216
|
+
returncode=e.returncode,
|
|
217
|
+
stdout=e.stdout,
|
|
218
|
+
stderr=e.stderr,
|
|
219
|
+
) from e
|
|
@@ -7,6 +7,7 @@ import re
|
|
|
7
7
|
import subprocess
|
|
8
8
|
from collections.abc import Callable
|
|
9
9
|
from typing import cast
|
|
10
|
+
from typing import NamedTuple
|
|
10
11
|
|
|
11
12
|
from packaging import version
|
|
12
13
|
|
|
@@ -27,6 +28,13 @@ from devservices.utils.install_binary import install_binary
|
|
|
27
28
|
from devservices.utils.services import Service
|
|
28
29
|
|
|
29
30
|
|
|
31
|
+
class DockerComposeCommand(NamedTuple):
|
|
32
|
+
full_command: list[str]
|
|
33
|
+
project_name: str
|
|
34
|
+
config_path: str
|
|
35
|
+
services: list[str]
|
|
36
|
+
|
|
37
|
+
|
|
30
38
|
def install_docker_compose() -> None:
|
|
31
39
|
console = Console()
|
|
32
40
|
# Determine the platform
|
|
@@ -87,6 +95,32 @@ def install_docker_compose() -> None:
|
|
|
87
95
|
console.success(f"Verified Docker Compose installation: v{version}")
|
|
88
96
|
|
|
89
97
|
|
|
98
|
+
def get_container_names_for_project(project_name: str, config_path: str) -> list[str]:
|
|
99
|
+
try:
|
|
100
|
+
container_names = subprocess.check_output(
|
|
101
|
+
[
|
|
102
|
+
"docker",
|
|
103
|
+
"compose",
|
|
104
|
+
"-p",
|
|
105
|
+
project_name,
|
|
106
|
+
"-f",
|
|
107
|
+
config_path,
|
|
108
|
+
"ps",
|
|
109
|
+
"--format",
|
|
110
|
+
"{{.Name}}",
|
|
111
|
+
],
|
|
112
|
+
text=True,
|
|
113
|
+
).splitlines()
|
|
114
|
+
return container_names
|
|
115
|
+
except subprocess.CalledProcessError as e:
|
|
116
|
+
raise DockerComposeError(
|
|
117
|
+
command=f"docker compose -p {project_name} -f {config_path} ps --format {{.Name}}",
|
|
118
|
+
returncode=e.returncode,
|
|
119
|
+
stdout=e.stdout,
|
|
120
|
+
stderr=e.stderr,
|
|
121
|
+
) from e
|
|
122
|
+
|
|
123
|
+
|
|
90
124
|
def check_docker_compose_version() -> None:
|
|
91
125
|
console = Console()
|
|
92
126
|
# Throw an error if docker daemon isn't running
|
|
@@ -169,10 +203,12 @@ def get_docker_compose_commands_to_run(
|
|
|
169
203
|
options: list[str],
|
|
170
204
|
service_config_file_path: str,
|
|
171
205
|
mode_dependencies: list[str],
|
|
172
|
-
) -> list[
|
|
206
|
+
) -> list[DockerComposeCommand]:
|
|
173
207
|
docker_compose_commands = []
|
|
174
|
-
create_docker_compose_command: Callable[
|
|
175
|
-
|
|
208
|
+
create_docker_compose_command: Callable[
|
|
209
|
+
[str, str, set[str]], DockerComposeCommand
|
|
210
|
+
] = lambda name, config_path, services_to_use: DockerComposeCommand(
|
|
211
|
+
full_command=[
|
|
176
212
|
"docker",
|
|
177
213
|
"compose",
|
|
178
214
|
"-p",
|
|
@@ -181,8 +217,11 @@ def get_docker_compose_commands_to_run(
|
|
|
181
217
|
config_path,
|
|
182
218
|
command,
|
|
183
219
|
]
|
|
184
|
-
+ sorted(list(services_to_use))
|
|
185
|
-
+ options
|
|
220
|
+
+ sorted(list(services_to_use))
|
|
221
|
+
+ options,
|
|
222
|
+
project_name=name,
|
|
223
|
+
config_path=config_path,
|
|
224
|
+
services=sorted(list(services_to_use)),
|
|
186
225
|
)
|
|
187
226
|
for dependency in remote_dependencies:
|
|
188
227
|
# TODO: Consider passing in service config in InstalledRemoteDependency instead of loading it here
|