devservices 1.0.17__tar.gz → 1.0.18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {devservices-1.0.17 → devservices-1.0.18}/PKG-INFO +3 -2
- devservices-1.0.18/README.md +145 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/commands/down.py +47 -18
- {devservices-1.0.17 → devservices-1.0.18}/devservices/commands/up.py +55 -9
- {devservices-1.0.17 → devservices-1.0.18}/devservices/main.py +17 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/dependencies.py +2 -1
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/docker.py +16 -6
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/docker_compose.py +12 -4
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/state.py +71 -3
- {devservices-1.0.17 → devservices-1.0.18}/devservices.egg-info/PKG-INFO +3 -2
- {devservices-1.0.17 → devservices-1.0.18}/pyproject.toml +1 -1
- {devservices-1.0.17 → devservices-1.0.18}/tests/commands/test_down.py +355 -11
- {devservices-1.0.17 → devservices-1.0.18}/tests/commands/test_up.py +329 -118
- {devservices-1.0.17 → devservices-1.0.18}/tests/utils/test_dependencies.py +8 -8
- {devservices-1.0.17 → devservices-1.0.18}/tests/utils/test_docker.py +78 -18
- {devservices-1.0.17 → devservices-1.0.18}/tests/utils/test_docker_compose.py +21 -0
- devservices-1.0.18/tests/utils/test_state.py +160 -0
- devservices-1.0.17/README.md +0 -33
- devservices-1.0.17/tests/utils/test_state.py +0 -84
- {devservices-1.0.17 → devservices-1.0.18}/LICENSE.md +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/__init__.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/commands/__init__.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/commands/list_dependencies.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/commands/list_services.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/commands/logs.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/commands/purge.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/commands/status.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/commands/update.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/configs/service_config.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/constants.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/exceptions.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/__init__.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/check_for_update.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/console.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/devenv.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/file_lock.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/git.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/install_binary.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices/utils/services.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices.egg-info/SOURCES.txt +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices.egg-info/dependency_links.txt +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices.egg-info/entry_points.txt +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices.egg-info/requires.txt +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/devservices.egg-info/top_level.txt +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/setup.cfg +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/testing/__init__.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/testing/utils.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/__init__.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/commands/test_list_dependencies.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/commands/test_list_services.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/commands/test_logs.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/commands/test_purge.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/commands/test_status.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/commands/test_update.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/configs/test_service_config.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/conftest.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/utils/test_check_for_update.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/utils/test_git.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/utils/test_install_binary.py +0 -0
- {devservices-1.0.17 → devservices-1.0.18}/tests/utils/test_services.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: devservices
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.18
|
|
4
4
|
Requires-Python: >=3.10
|
|
5
5
|
License-File: LICENSE.md
|
|
6
6
|
Requires-Dist: pyyaml
|
|
@@ -13,3 +13,4 @@ Requires-Dist: mypy; extra == "dev"
|
|
|
13
13
|
Requires-Dist: pre-commit; extra == "dev"
|
|
14
14
|
Requires-Dist: pytest; extra == "dev"
|
|
15
15
|
Requires-Dist: types-PyYAML; extra == "dev"
|
|
16
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# devservices
|
|
2
|
+
|
|
3
|
+
A standalone cli tool used to manage dependencies for services. It simplifies the process of managing services for development purposes and bringing services up/down.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
`devservices` reads configuration files located in the `devservices` directory of your repository. These configurations define services, their dependencies, and various modes of operation.
|
|
8
|
+
|
|
9
|
+
## Usage
|
|
10
|
+
|
|
11
|
+
devservices provides several commands to manage your services:
|
|
12
|
+
|
|
13
|
+
### Commands
|
|
14
|
+
|
|
15
|
+
NOTE: service-name is an optional parameter. If not provided, devservices will attempt to automatically find a devservices configuration in the current directory in order to proceed.
|
|
16
|
+
|
|
17
|
+
- `devservices up <service-name>`: Bring up a service and its dependencies.
|
|
18
|
+
- `devservices down <service-name>`: Bring down service including its dependencies.
|
|
19
|
+
- `devservices status <service-name>`: Display the current status of all services, including their dependencies and ports.
|
|
20
|
+
- `devservices logs <service-name>`: View logs for a specific service.
|
|
21
|
+
- `devservices list-services`: List all available Sentry services.
|
|
22
|
+
- `devservices list-dependencies <service-name>`: List all dependencies for a service and whether they are enabled/disabled.
|
|
23
|
+
- `devservices update` Update devservices to the latest version.
|
|
24
|
+
- `devservices purge`: Purge the local devservices cache.
|
|
25
|
+
|
|
26
|
+
## Installation
|
|
27
|
+
|
|
28
|
+
### 1. Add devservices to your requirements.txt
|
|
29
|
+
|
|
30
|
+
The recommended way to install devservices is through a virtualenv in the requirements.txt. Once that is installed and a devservices config file is added, you should be able to run `devservices up` to begin local development.
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
devservices==1.0.18
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### 2. Add devservices config files
|
|
37
|
+
|
|
38
|
+
Each repo should have a `devservices` directory with a `config.yml` file. This file is used to define services, dependencies, and modes. Other files and subdirectories in the `devservices` directory are optional and can be most commonly used for volume mounts.
|
|
39
|
+
|
|
40
|
+
The configuration file is a yaml file that looks like this:
|
|
41
|
+
|
|
42
|
+
```yaml
|
|
43
|
+
# This is a yaml block that holds devservices specific configuration settings. This is comprised of a few main sections:
|
|
44
|
+
# - version: The version of the devservices config file. This is used to ensure compatibility between devservices and the config file.
|
|
45
|
+
# - service_name: The name of the service. This is used to identify the service in the config file.
|
|
46
|
+
# - dependencies: A list of dependencies for the service. Each dependency is a yaml block that holds the dependency configuration. There are two types of dependencies:
|
|
47
|
+
# - local: A dependency that is defined in the config file. These dependencies do not have a remote field. These dependency definitions are specific to the service and are not defined elsewhere.
|
|
48
|
+
# - remote: A dependency that is defined in the devservices directory in a remote repository. These configs are automatically fetched from the remote repository and installed. Any dependency with a remote field will be treated as a remote dependency. Example: https://github.com/getsentry/snuba/blob/59a5258ccbb502827ebc1d3b1bf80c607a3301bf/devservices/config.yml#L8
|
|
49
|
+
# - modes: A list of modes for the service. Each mode includes a list of dependencies that are used in that mode.
|
|
50
|
+
x-sentry-service-config:
|
|
51
|
+
version: 0.1
|
|
52
|
+
service_name: example-service
|
|
53
|
+
dependencies:
|
|
54
|
+
example-dependency-1:
|
|
55
|
+
description: Example dependency defined in the config file
|
|
56
|
+
example-dependency-2:
|
|
57
|
+
description: Example dependency defined in the config file
|
|
58
|
+
example-remote-dependency:
|
|
59
|
+
description: Remote dependency defined in the `devservices` directory in the example-repository repo
|
|
60
|
+
remote:
|
|
61
|
+
repo_name: example-repository
|
|
62
|
+
branch: main
|
|
63
|
+
repo_link: https://github.com/getsentry/example-repository.git
|
|
64
|
+
mode: default # Optional field, mode to run remote dependency in that defaults to `default`
|
|
65
|
+
modes:
|
|
66
|
+
default: [example-dependency-1, example-remote-dependency]
|
|
67
|
+
custom-mode: [example-dependency-1, example-dependency-2, example-remote-dependency]
|
|
68
|
+
|
|
69
|
+
# This will be a standard block used by docker compose to define dependencies.
|
|
70
|
+
#
|
|
71
|
+
# The following fields are important to all dependencies:
|
|
72
|
+
# - image: The docker image to use for the dependency.
|
|
73
|
+
# - ports: The ports to expose for the dependency. Please only expose ports to localhost(127.0.0.1)
|
|
74
|
+
# - healthcheck: The docker healthcheck to use for the dependency.
|
|
75
|
+
# - environment: The environment variables to set for the dependency.
|
|
76
|
+
# - extra_hosts: The extra hosts to add to the dependency.
|
|
77
|
+
# - networks: The networks to add to the dependency. In order for devservices to work properly, the dependency must be on the `devservices` network.
|
|
78
|
+
# - labels: The labels to add to the dependency. The `orchestrator=devservices` label is required for devservices to determine a container is managed by devservices.
|
|
79
|
+
# - restart: The restart policy to use for the dependency.
|
|
80
|
+
#
|
|
81
|
+
# These fields are optional:
|
|
82
|
+
# - ulimits: The ulimits to set for the dependency. This is useful for setting resource constraints for the dependency.
|
|
83
|
+
# - volumes: The volumes to mount for the dependency. This is useful for mounting data volumes for the dependency if data should be persisted between runs. It can also be useful to use a bind mount to mount a local directory into a container. Example of bind mounting clickhouse configs from a local directory https://github.com/getsentry/snuba/blob/59a5258ccbb502827ebc1d3b1bf80c607a3301bf/devservices/config.yml#L44
|
|
84
|
+
# - command: The command to run for the dependency. This can override the default command for the docker image.
|
|
85
|
+
# For more information on the docker compose file services block, see the docker compose file reference: https://docs.docker.com/reference/compose-file/services/
|
|
86
|
+
services:
|
|
87
|
+
example-dependency-1:
|
|
88
|
+
image: ghcr.io/getsentry/example-dependency-1:1.0.0
|
|
89
|
+
ulimits:
|
|
90
|
+
nofile:
|
|
91
|
+
soft: 262144
|
|
92
|
+
hard: 262144
|
|
93
|
+
healthcheck:
|
|
94
|
+
test: wget -q -O - http://localhost:1234/health
|
|
95
|
+
interval: 5s
|
|
96
|
+
timeout: 5s
|
|
97
|
+
retries: 3
|
|
98
|
+
environment:
|
|
99
|
+
EXAMPLE_ENV_VAR: example-value
|
|
100
|
+
volumes:
|
|
101
|
+
- example-dependency-1-data:/var/lib/example-dependency-1
|
|
102
|
+
restart: unless-stopped
|
|
103
|
+
# Everything below this line is required for devservices to work properly.
|
|
104
|
+
ports:
|
|
105
|
+
- 127.0.0.1:1234:1234
|
|
106
|
+
extra_hosts:
|
|
107
|
+
host.docker.internal: host-gateway
|
|
108
|
+
networks:
|
|
109
|
+
- devservices
|
|
110
|
+
labels:
|
|
111
|
+
- orchestrator=devservices
|
|
112
|
+
|
|
113
|
+
example-dependency-2:
|
|
114
|
+
image: ghcr.io/getsentry/example-dependency-2:1.0.0
|
|
115
|
+
command: ["devserver"]
|
|
116
|
+
healthcheck:
|
|
117
|
+
test: curl -f http://localhost:2345/health
|
|
118
|
+
interval: 5s
|
|
119
|
+
timeout: 5s
|
|
120
|
+
retries: 3
|
|
121
|
+
environment:
|
|
122
|
+
EXAMPLE_ENV_VAR: example-value
|
|
123
|
+
restart: unless-stopped
|
|
124
|
+
# Everything below this line is required for devservices to work properly.
|
|
125
|
+
ports:
|
|
126
|
+
- 127.0.0.1:2345:2345
|
|
127
|
+
extra_hosts:
|
|
128
|
+
host.docker.internal: host-gateway
|
|
129
|
+
networks:
|
|
130
|
+
- devservices
|
|
131
|
+
labels:
|
|
132
|
+
- orchestrator=devservices
|
|
133
|
+
|
|
134
|
+
# This is a standard block used by docker compose to define volumes.
|
|
135
|
+
# For more information, see the docker compose file reference: https://docs.docker.com/reference/compose-file/volumes/
|
|
136
|
+
volumes:
|
|
137
|
+
example-dependency-1-data:
|
|
138
|
+
|
|
139
|
+
# This is a standard block used by docker compose to define networks. Defining the devservices network is required for devservices to work properly. By default, devservices will create an external network called `devservices` that is used to connect all dependencies.
|
|
140
|
+
# For more information, see the docker compose file reference: https://docs.docker.com/reference/compose-file/networks/
|
|
141
|
+
networks:
|
|
142
|
+
devservices:
|
|
143
|
+
name: devservices
|
|
144
|
+
external: true
|
|
145
|
+
```
|
|
@@ -30,6 +30,7 @@ from devservices.utils.docker_compose import get_docker_compose_commands_to_run
|
|
|
30
30
|
from devservices.utils.docker_compose import run_cmd
|
|
31
31
|
from devservices.utils.services import find_matching_service
|
|
32
32
|
from devservices.utils.services import Service
|
|
33
|
+
from devservices.utils.state import ServiceRuntime
|
|
33
34
|
from devservices.utils.state import State
|
|
34
35
|
from devservices.utils.state import StateTables
|
|
35
36
|
|
|
@@ -122,26 +123,16 @@ def down(args: Namespace) -> None:
|
|
|
122
123
|
# Check if any service depends on the service we are trying to bring down
|
|
123
124
|
# TODO: We should also take into account the active modes of the other services (this is not trivial to do)
|
|
124
125
|
other_started_services = active_services.difference({service.name})
|
|
126
|
+
services_with_local_runtime = state.get_services_by_runtime(
|
|
127
|
+
ServiceRuntime.LOCAL
|
|
128
|
+
)
|
|
125
129
|
dependent_service_name = None
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
other_service_active_started_modes = state.get_active_modes_for_service(
|
|
132
|
-
other_service.name, StateTables.STARTED_SERVICES
|
|
133
|
-
)
|
|
134
|
-
other_service_active_modes = (
|
|
135
|
-
other_service_active_starting_modes
|
|
136
|
-
or other_service_active_started_modes
|
|
137
|
-
)
|
|
138
|
-
dependency_graph = construct_dependency_graph(
|
|
139
|
-
other_service, other_service_active_modes
|
|
130
|
+
# We can ignore checking if anything relies on the service
|
|
131
|
+
# if it is a locally running service
|
|
132
|
+
if service.name not in services_with_local_runtime:
|
|
133
|
+
dependent_service_name = _get_dependent_service(
|
|
134
|
+
service, other_started_services, state
|
|
140
135
|
)
|
|
141
|
-
# If the service we are trying to bring down is in the dependency graph of another service, we should not bring it down
|
|
142
|
-
if service.name in dependency_graph.graph:
|
|
143
|
-
dependent_service_name = other_started_service
|
|
144
|
-
break
|
|
145
136
|
|
|
146
137
|
# If no other service depends on the service we are trying to bring down, we can bring it down
|
|
147
138
|
if dependent_service_name is None:
|
|
@@ -190,6 +181,17 @@ def _down(
|
|
|
190
181
|
current_env[
|
|
191
182
|
DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
|
|
192
183
|
] = relative_local_dependency_directory
|
|
184
|
+
state = State()
|
|
185
|
+
# We want to ignore any dependencies that are set to run locally
|
|
186
|
+
locally_running_services = state.get_services_by_runtime(ServiceRuntime.LOCAL)
|
|
187
|
+
mode_dependencies = [
|
|
188
|
+
dep for dep in mode_dependencies if dep not in locally_running_services
|
|
189
|
+
]
|
|
190
|
+
remote_dependencies = {
|
|
191
|
+
dep
|
|
192
|
+
for dep in remote_dependencies
|
|
193
|
+
if dep.service_name not in locally_running_services
|
|
194
|
+
}
|
|
193
195
|
docker_compose_commands = get_docker_compose_commands_to_run(
|
|
194
196
|
service=service,
|
|
195
197
|
remote_dependencies=list(remote_dependencies),
|
|
@@ -209,3 +211,30 @@ def _down(
|
|
|
209
211
|
]
|
|
210
212
|
for future in concurrent.futures.as_completed(futures):
|
|
211
213
|
cmd_outputs.append(future.result())
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _get_dependent_service(
|
|
217
|
+
service: Service,
|
|
218
|
+
other_started_services: set[str],
|
|
219
|
+
state: State,
|
|
220
|
+
) -> str | None:
|
|
221
|
+
for other_started_service in other_started_services:
|
|
222
|
+
other_service = find_matching_service(other_started_service)
|
|
223
|
+
other_service_active_starting_modes = state.get_active_modes_for_service(
|
|
224
|
+
other_service.name, StateTables.STARTING_SERVICES
|
|
225
|
+
)
|
|
226
|
+
other_service_active_started_modes = state.get_active_modes_for_service(
|
|
227
|
+
other_service.name, StateTables.STARTED_SERVICES
|
|
228
|
+
)
|
|
229
|
+
other_service_active_modes = (
|
|
230
|
+
other_service_active_starting_modes or other_service_active_started_modes
|
|
231
|
+
)
|
|
232
|
+
dependency_graph = construct_dependency_graph(
|
|
233
|
+
other_service, other_service_active_modes
|
|
234
|
+
)
|
|
235
|
+
# If the service we are trying to bring down is in the dependency graph of another service,
|
|
236
|
+
# we should not bring it down
|
|
237
|
+
if service.name in dependency_graph.graph:
|
|
238
|
+
return other_started_service
|
|
239
|
+
|
|
240
|
+
return None
|
|
@@ -33,6 +33,7 @@ from devservices.utils.docker_compose import get_docker_compose_commands_to_run
|
|
|
33
33
|
from devservices.utils.docker_compose import run_cmd
|
|
34
34
|
from devservices.utils.services import find_matching_service
|
|
35
35
|
from devservices.utils.services import Service
|
|
36
|
+
from devservices.utils.state import ServiceRuntime
|
|
36
37
|
from devservices.utils.state import State
|
|
37
38
|
from devservices.utils.state import StateTables
|
|
38
39
|
|
|
@@ -106,8 +107,20 @@ def up(args: Namespace) -> None:
|
|
|
106
107
|
pass
|
|
107
108
|
# Add the service to the starting services table
|
|
108
109
|
state.update_service_entry(service.name, mode, StateTables.STARTING_SERVICES)
|
|
110
|
+
mode_dependencies = modes[mode]
|
|
111
|
+
# We want to ignore any dependencies that are set to run locally
|
|
112
|
+
services_with_local_runtime = state.get_services_by_runtime(
|
|
113
|
+
ServiceRuntime.LOCAL
|
|
114
|
+
)
|
|
115
|
+
mode_dependencies = [
|
|
116
|
+
dep for dep in mode_dependencies if dep not in services_with_local_runtime
|
|
117
|
+
]
|
|
118
|
+
remote_dependencies = {
|
|
119
|
+
dep
|
|
120
|
+
for dep in remote_dependencies
|
|
121
|
+
if dep.service_name not in services_with_local_runtime
|
|
122
|
+
}
|
|
109
123
|
try:
|
|
110
|
-
mode_dependencies = modes[mode]
|
|
111
124
|
_up(service, [mode], remote_dependencies, mode_dependencies, status)
|
|
112
125
|
except DockerComposeError as dce:
|
|
113
126
|
capture_exception(dce, level="info")
|
|
@@ -118,12 +131,20 @@ def up(args: Namespace) -> None:
|
|
|
118
131
|
state.update_service_entry(service.name, mode, StateTables.STARTED_SERVICES)
|
|
119
132
|
|
|
120
133
|
|
|
134
|
+
def _pull_dependency_images(
|
|
135
|
+
cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
|
|
136
|
+
) -> None:
|
|
137
|
+
run_cmd(cmd.full_command, current_env)
|
|
138
|
+
for dependency in cmd.services:
|
|
139
|
+
status.info(f"Pulled image for {dependency}")
|
|
140
|
+
|
|
141
|
+
|
|
121
142
|
def _bring_up_dependency(
|
|
122
143
|
cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
|
|
123
|
-
) ->
|
|
144
|
+
) -> None:
|
|
124
145
|
for dependency in cmd.services:
|
|
125
146
|
status.info(f"Starting {dependency}")
|
|
126
|
-
|
|
147
|
+
run_cmd(cmd.full_command, current_env)
|
|
127
148
|
|
|
128
149
|
|
|
129
150
|
def _up(
|
|
@@ -150,26 +171,51 @@ def _up(
|
|
|
150
171
|
sorted_remote_dependencies = sorted(
|
|
151
172
|
remote_dependencies, key=lambda dep: starting_order.index(dep.service_name)
|
|
152
173
|
)
|
|
153
|
-
|
|
174
|
+
# Pull all images in parallel
|
|
175
|
+
status.info("Pulling images")
|
|
176
|
+
pull_commands = get_docker_compose_commands_to_run(
|
|
177
|
+
service=service,
|
|
178
|
+
remote_dependencies=sorted_remote_dependencies,
|
|
179
|
+
current_env=current_env,
|
|
180
|
+
command="pull",
|
|
181
|
+
options=[],
|
|
182
|
+
service_config_file_path=service_config_file_path,
|
|
183
|
+
mode_dependencies=mode_dependencies,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
with concurrent.futures.ThreadPoolExecutor() as pull_dependency_executor:
|
|
187
|
+
futures = [
|
|
188
|
+
pull_dependency_executor.submit(
|
|
189
|
+
_pull_dependency_images, cmd, current_env, status
|
|
190
|
+
)
|
|
191
|
+
for cmd in pull_commands
|
|
192
|
+
]
|
|
193
|
+
for future in concurrent.futures.as_completed(futures):
|
|
194
|
+
_ = future.result()
|
|
195
|
+
|
|
196
|
+
# Bring up all necessary containers
|
|
197
|
+
up_commands = get_docker_compose_commands_to_run(
|
|
154
198
|
service=service,
|
|
155
199
|
remote_dependencies=sorted_remote_dependencies,
|
|
156
200
|
current_env=current_env,
|
|
157
201
|
command="up",
|
|
158
|
-
options=["-d"
|
|
202
|
+
options=["-d"],
|
|
159
203
|
service_config_file_path=service_config_file_path,
|
|
160
204
|
mode_dependencies=mode_dependencies,
|
|
161
205
|
)
|
|
162
206
|
|
|
163
207
|
containers_to_check = []
|
|
164
|
-
with concurrent.futures.ThreadPoolExecutor() as
|
|
208
|
+
with concurrent.futures.ThreadPoolExecutor() as up_dependency_executor:
|
|
165
209
|
futures = [
|
|
166
|
-
|
|
167
|
-
|
|
210
|
+
up_dependency_executor.submit(
|
|
211
|
+
_bring_up_dependency, cmd, current_env, status
|
|
212
|
+
)
|
|
213
|
+
for cmd in up_commands
|
|
168
214
|
]
|
|
169
215
|
for future in concurrent.futures.as_completed(futures):
|
|
170
216
|
_ = future.result()
|
|
171
217
|
|
|
172
|
-
for cmd in
|
|
218
|
+
for cmd in up_commands:
|
|
173
219
|
try:
|
|
174
220
|
container_names = get_container_names_for_project(
|
|
175
221
|
cmd.project_name, cmd.config_path
|
|
@@ -11,6 +11,7 @@ from importlib import metadata
|
|
|
11
11
|
from sentry_sdk import capture_exception
|
|
12
12
|
from sentry_sdk import flush
|
|
13
13
|
from sentry_sdk import init
|
|
14
|
+
from sentry_sdk import set_context
|
|
14
15
|
from sentry_sdk import set_tag
|
|
15
16
|
from sentry_sdk import set_user
|
|
16
17
|
from sentry_sdk import start_transaction
|
|
@@ -84,6 +85,22 @@ if not disable_sentry:
|
|
|
84
85
|
username = getpass.getuser()
|
|
85
86
|
set_user({"username": username})
|
|
86
87
|
set_tag("user_platform", platform.platform())
|
|
88
|
+
if sentry_environment == "CI":
|
|
89
|
+
set_context(
|
|
90
|
+
"github",
|
|
91
|
+
{
|
|
92
|
+
"github_action": os.environ.get("GITHUB_ACTION"),
|
|
93
|
+
"github_action_path": os.environ.get("GITHUB_ACTION_PATH"),
|
|
94
|
+
"github_repository": os.environ.get("GITHUB_REPOSITORY"),
|
|
95
|
+
"github_ref_name": os.environ.get("GITHUB_REF_NAME"),
|
|
96
|
+
"github_run_id": os.environ.get("GITHUB_RUN_ID"),
|
|
97
|
+
"github_url": f"{os.environ.get('GITHUB_SERVER_URL')}/{os.environ.get('GITHUB_REPOSITORY')}/actions/runs/{os.environ.get('GITHUB_RUN_ID')}",
|
|
98
|
+
"github_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
|
|
99
|
+
"github_workflow": os.environ.get("GITHUB_WORKFLOW"),
|
|
100
|
+
"github_workflow_run_id": os.environ.get("GITHUB_WORKFLOW_RUN_ID"),
|
|
101
|
+
"github_sha": os.environ.get("GITHUB_SHA"),
|
|
102
|
+
},
|
|
103
|
+
)
|
|
87
104
|
try:
|
|
88
105
|
git_version = get_git_version()
|
|
89
106
|
set_tag("git_version", git_version)
|
|
@@ -271,7 +271,8 @@ def get_non_shared_remote_dependencies(
|
|
|
271
271
|
started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
|
|
272
272
|
active_services = starting_services.union(started_services)
|
|
273
273
|
# We don't care about the remote dependencies of the service we are stopping
|
|
274
|
-
|
|
274
|
+
if service_to_stop.name in active_services:
|
|
275
|
+
active_services.remove(service_to_stop.name)
|
|
275
276
|
|
|
276
277
|
active_modes: dict[str, list[str]] = dict()
|
|
277
278
|
for active_service in active_services:
|
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import concurrent.futures
|
|
4
4
|
import subprocess
|
|
5
5
|
import time
|
|
6
|
+
from typing import NamedTuple
|
|
6
7
|
|
|
7
8
|
from devservices.constants import HEALTHCHECK_INTERVAL
|
|
8
9
|
from devservices.constants import HEALTHCHECK_TIMEOUT
|
|
@@ -12,6 +13,11 @@ from devservices.exceptions import DockerError
|
|
|
12
13
|
from devservices.utils.console import Status
|
|
13
14
|
|
|
14
15
|
|
|
16
|
+
class ContainerNames(NamedTuple):
|
|
17
|
+
name: str
|
|
18
|
+
short_name: str
|
|
19
|
+
|
|
20
|
+
|
|
15
21
|
def check_docker_daemon_running() -> None:
|
|
16
22
|
"""Checks if the Docker daemon is running. Raises DockerDaemonNotRunningError if not."""
|
|
17
23
|
try:
|
|
@@ -25,8 +31,11 @@ def check_docker_daemon_running() -> None:
|
|
|
25
31
|
raise DockerDaemonNotRunningError from e
|
|
26
32
|
|
|
27
33
|
|
|
28
|
-
def check_all_containers_healthy(
|
|
34
|
+
def check_all_containers_healthy(
|
|
35
|
+
status: Status, containers: list[ContainerNames]
|
|
36
|
+
) -> None:
|
|
29
37
|
"""Ensures all containers are healthy."""
|
|
38
|
+
status.info("Waiting for all containers to be healthy")
|
|
30
39
|
with concurrent.futures.ThreadPoolExecutor() as healthcheck_executor:
|
|
31
40
|
futures = [
|
|
32
41
|
healthcheck_executor.submit(wait_for_healthy, container, status)
|
|
@@ -36,7 +45,7 @@ def check_all_containers_healthy(status: Status, containers: list[str]) -> None:
|
|
|
36
45
|
future.result()
|
|
37
46
|
|
|
38
47
|
|
|
39
|
-
def wait_for_healthy(
|
|
48
|
+
def wait_for_healthy(container: ContainerNames, status: Status) -> None:
|
|
40
49
|
"""
|
|
41
50
|
Polls a Docker container's health status until it becomes healthy or a timeout is reached.
|
|
42
51
|
"""
|
|
@@ -51,31 +60,32 @@ def wait_for_healthy(container_name: str, status: Status) -> None:
|
|
|
51
60
|
"inspect",
|
|
52
61
|
"-f",
|
|
53
62
|
"{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}",
|
|
54
|
-
|
|
63
|
+
container.name,
|
|
55
64
|
],
|
|
56
65
|
stderr=subprocess.DEVNULL,
|
|
57
66
|
text=True,
|
|
58
67
|
).strip()
|
|
59
68
|
except subprocess.CalledProcessError as e:
|
|
60
69
|
raise DockerError(
|
|
61
|
-
command=f"docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' {
|
|
70
|
+
command=f"docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' {container.name}",
|
|
62
71
|
returncode=e.returncode,
|
|
63
72
|
stdout=e.stdout,
|
|
64
73
|
stderr=e.stderr,
|
|
65
74
|
) from e
|
|
66
75
|
|
|
67
76
|
if result == "healthy":
|
|
77
|
+
status.info(f"{container.short_name} is healthy")
|
|
68
78
|
return
|
|
69
79
|
if result == "unknown":
|
|
70
80
|
status.warning(
|
|
71
|
-
f"WARNING: Container {
|
|
81
|
+
f"WARNING: Container {container.short_name} does not have a healthcheck"
|
|
72
82
|
)
|
|
73
83
|
return
|
|
74
84
|
|
|
75
85
|
# If not healthy, wait and try again
|
|
76
86
|
time.sleep(HEALTHCHECK_INTERVAL)
|
|
77
87
|
|
|
78
|
-
raise ContainerHealthcheckFailedError(
|
|
88
|
+
raise ContainerHealthcheckFailedError(container.short_name, HEALTHCHECK_TIMEOUT)
|
|
79
89
|
|
|
80
90
|
|
|
81
91
|
def get_matching_containers(label: str) -> list[str]:
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import json
|
|
3
4
|
import logging
|
|
4
5
|
import os
|
|
5
6
|
import platform
|
|
@@ -23,6 +24,7 @@ from devservices.exceptions import DockerComposeInstallationError
|
|
|
23
24
|
from devservices.utils.console import Console
|
|
24
25
|
from devservices.utils.dependencies import InstalledRemoteDependency
|
|
25
26
|
from devservices.utils.docker import check_docker_daemon_running
|
|
27
|
+
from devservices.utils.docker import ContainerNames
|
|
26
28
|
from devservices.utils.install_binary import install_binary
|
|
27
29
|
from devservices.utils.services import Service
|
|
28
30
|
|
|
@@ -92,9 +94,11 @@ def install_docker_compose() -> None:
|
|
|
92
94
|
console.success(f"Verified Docker Compose installation: v{version}")
|
|
93
95
|
|
|
94
96
|
|
|
95
|
-
def get_container_names_for_project(
|
|
97
|
+
def get_container_names_for_project(
|
|
98
|
+
project_name: str, config_path: str
|
|
99
|
+
) -> list[ContainerNames]:
|
|
96
100
|
try:
|
|
97
|
-
|
|
101
|
+
output = subprocess.check_output(
|
|
98
102
|
[
|
|
99
103
|
"docker",
|
|
100
104
|
"compose",
|
|
@@ -104,11 +108,15 @@ def get_container_names_for_project(project_name: str, config_path: str) -> list
|
|
|
104
108
|
config_path,
|
|
105
109
|
"ps",
|
|
106
110
|
"--format",
|
|
107
|
-
"{{.
|
|
111
|
+
'{"name":"{{.Names}}", "short_name":"{{.Service}}"}',
|
|
108
112
|
],
|
|
109
113
|
text=True,
|
|
110
114
|
).splitlines()
|
|
111
|
-
return
|
|
115
|
+
return [
|
|
116
|
+
ContainerNames(name=json_data["name"], short_name=json_data["short_name"])
|
|
117
|
+
for line in output
|
|
118
|
+
if (json_data := json.loads(line))
|
|
119
|
+
]
|
|
112
120
|
except subprocess.CalledProcessError as e:
|
|
113
121
|
raise DockerComposeError(
|
|
114
122
|
command=f"docker compose -p {project_name} -f {config_path} ps --format {{.Name}}",
|
|
@@ -3,14 +3,21 @@ from __future__ import annotations
|
|
|
3
3
|
import os
|
|
4
4
|
import sqlite3
|
|
5
5
|
from enum import Enum
|
|
6
|
+
from typing import Literal
|
|
6
7
|
|
|
7
8
|
from devservices.constants import DEVSERVICES_LOCAL_DIR
|
|
8
9
|
from devservices.constants import STATE_DB_FILE
|
|
9
10
|
|
|
10
11
|
|
|
12
|
+
class ServiceRuntime(Enum):
|
|
13
|
+
LOCAL = "local"
|
|
14
|
+
CONTAINERIZED = "containerized"
|
|
15
|
+
|
|
16
|
+
|
|
11
17
|
class StateTables(Enum):
|
|
12
18
|
STARTED_SERVICES = "started_services"
|
|
13
19
|
STARTING_SERVICES = "starting_services"
|
|
20
|
+
SERVICE_RUNTIME = "service_runtime"
|
|
14
21
|
|
|
15
22
|
|
|
16
23
|
class State:
|
|
@@ -30,7 +37,7 @@ class State:
|
|
|
30
37
|
|
|
31
38
|
def initialize_database(self) -> None:
|
|
32
39
|
cursor = self.conn.cursor()
|
|
33
|
-
# Formatted strings here and throughout the
|
|
40
|
+
# Formatted strings here and throughout the file should be extremely low risk given these are constants
|
|
34
41
|
cursor.execute(
|
|
35
42
|
f"""
|
|
36
43
|
CREATE TABLE IF NOT EXISTS {StateTables.STARTED_SERVICES.value} (
|
|
@@ -50,10 +57,26 @@ class State:
|
|
|
50
57
|
)
|
|
51
58
|
"""
|
|
52
59
|
)
|
|
60
|
+
|
|
61
|
+
cursor.execute(
|
|
62
|
+
f"""
|
|
63
|
+
CREATE TABLE IF NOT EXISTS {StateTables.SERVICE_RUNTIME.value} (
|
|
64
|
+
service_name TEXT PRIMARY KEY,
|
|
65
|
+
runtime TEXT
|
|
66
|
+
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
67
|
+
)
|
|
68
|
+
"""
|
|
69
|
+
)
|
|
53
70
|
self.conn.commit()
|
|
54
71
|
|
|
55
72
|
def update_service_entry(
|
|
56
|
-
self,
|
|
73
|
+
self,
|
|
74
|
+
service_name: str,
|
|
75
|
+
mode: str,
|
|
76
|
+
table: (
|
|
77
|
+
Literal[StateTables.STARTED_SERVICES]
|
|
78
|
+
| Literal[StateTables.STARTING_SERVICES]
|
|
79
|
+
),
|
|
57
80
|
) -> None:
|
|
58
81
|
cursor = self.conn.cursor()
|
|
59
82
|
service_entries = self.get_service_entries(table)
|
|
@@ -96,7 +119,12 @@ class State:
|
|
|
96
119
|
return [row[0] for row in cursor.fetchall()]
|
|
97
120
|
|
|
98
121
|
def get_active_modes_for_service(
|
|
99
|
-
self,
|
|
122
|
+
self,
|
|
123
|
+
service_name: str,
|
|
124
|
+
table: (
|
|
125
|
+
Literal[StateTables.STARTED_SERVICES]
|
|
126
|
+
| Literal[StateTables.STARTING_SERVICES]
|
|
127
|
+
),
|
|
100
128
|
) -> list[str]:
|
|
101
129
|
cursor = self.conn.cursor()
|
|
102
130
|
cursor.execute(
|
|
@@ -110,6 +138,41 @@ class State:
|
|
|
110
138
|
return []
|
|
111
139
|
return str(result[0]).split(",")
|
|
112
140
|
|
|
141
|
+
def get_service_runtime(self, service_name: str) -> ServiceRuntime:
|
|
142
|
+
cursor = self.conn.cursor()
|
|
143
|
+
cursor.execute(
|
|
144
|
+
f"""
|
|
145
|
+
SELECT runtime FROM {StateTables.SERVICE_RUNTIME.value} WHERE service_name = ?
|
|
146
|
+
""",
|
|
147
|
+
(service_name,),
|
|
148
|
+
)
|
|
149
|
+
result = cursor.fetchone()
|
|
150
|
+
if result is None:
|
|
151
|
+
return ServiceRuntime.CONTAINERIZED
|
|
152
|
+
return ServiceRuntime(result[0])
|
|
153
|
+
|
|
154
|
+
def update_service_runtime(
|
|
155
|
+
self, service_name: str, runtime: ServiceRuntime
|
|
156
|
+
) -> None:
|
|
157
|
+
cursor = self.conn.cursor()
|
|
158
|
+
cursor.execute(
|
|
159
|
+
f"""
|
|
160
|
+
INSERT OR REPLACE INTO {StateTables.SERVICE_RUNTIME.value} (service_name, runtime) VALUES (?, ?)
|
|
161
|
+
""",
|
|
162
|
+
(service_name, runtime.value),
|
|
163
|
+
)
|
|
164
|
+
self.conn.commit()
|
|
165
|
+
|
|
166
|
+
def get_services_by_runtime(self, runtime: ServiceRuntime) -> list[str]:
|
|
167
|
+
cursor = self.conn.cursor()
|
|
168
|
+
cursor.execute(
|
|
169
|
+
f"""
|
|
170
|
+
SELECT service_name FROM {StateTables.SERVICE_RUNTIME.value} WHERE runtime = ?
|
|
171
|
+
""",
|
|
172
|
+
(runtime.value,),
|
|
173
|
+
)
|
|
174
|
+
return [row[0] for row in cursor.fetchall()]
|
|
175
|
+
|
|
113
176
|
def clear_state(self) -> None:
|
|
114
177
|
cursor = self.conn.cursor()
|
|
115
178
|
cursor.execute(
|
|
@@ -122,4 +185,9 @@ class State:
|
|
|
122
185
|
DELETE FROM {StateTables.STARTING_SERVICES.value}
|
|
123
186
|
"""
|
|
124
187
|
)
|
|
188
|
+
cursor.execute(
|
|
189
|
+
f"""
|
|
190
|
+
DELETE FROM {StateTables.SERVICE_RUNTIME.value}
|
|
191
|
+
"""
|
|
192
|
+
)
|
|
125
193
|
self.conn.commit()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: devservices
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.18
|
|
4
4
|
Requires-Python: >=3.10
|
|
5
5
|
License-File: LICENSE.md
|
|
6
6
|
Requires-Dist: pyyaml
|
|
@@ -13,3 +13,4 @@ Requires-Dist: mypy; extra == "dev"
|
|
|
13
13
|
Requires-Dist: pre-commit; extra == "dev"
|
|
14
14
|
Requires-Dist: pytest; extra == "dev"
|
|
15
15
|
Requires-Dist: types-PyYAML; extra == "dev"
|
|
16
|
+
Dynamic: license-file
|