devservices 1.0.16__tar.gz → 1.0.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {devservices-1.0.16 → devservices-1.0.18}/PKG-INFO +3 -2
  2. devservices-1.0.18/README.md +145 -0
  3. {devservices-1.0.16 → devservices-1.0.18}/devservices/commands/down.py +49 -20
  4. {devservices-1.0.16 → devservices-1.0.18}/devservices/commands/list_dependencies.py +1 -1
  5. {devservices-1.0.16 → devservices-1.0.18}/devservices/commands/logs.py +2 -2
  6. {devservices-1.0.16 → devservices-1.0.18}/devservices/commands/up.py +57 -11
  7. {devservices-1.0.16 → devservices-1.0.18}/devservices/main.py +49 -3
  8. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/dependencies.py +2 -1
  9. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/docker.py +16 -6
  10. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/docker_compose.py +12 -4
  11. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/state.py +71 -3
  12. {devservices-1.0.16 → devservices-1.0.18}/devservices.egg-info/PKG-INFO +3 -2
  13. {devservices-1.0.16 → devservices-1.0.18}/pyproject.toml +1 -1
  14. {devservices-1.0.16 → devservices-1.0.18}/tests/commands/test_down.py +355 -11
  15. {devservices-1.0.16 → devservices-1.0.18}/tests/commands/test_up.py +329 -118
  16. {devservices-1.0.16 → devservices-1.0.18}/tests/utils/test_dependencies.py +8 -8
  17. {devservices-1.0.16 → devservices-1.0.18}/tests/utils/test_docker.py +78 -18
  18. {devservices-1.0.16 → devservices-1.0.18}/tests/utils/test_docker_compose.py +21 -0
  19. devservices-1.0.18/tests/utils/test_state.py +160 -0
  20. devservices-1.0.16/README.md +0 -33
  21. devservices-1.0.16/tests/utils/test_state.py +0 -84
  22. {devservices-1.0.16 → devservices-1.0.18}/LICENSE.md +0 -0
  23. {devservices-1.0.16 → devservices-1.0.18}/devservices/__init__.py +0 -0
  24. {devservices-1.0.16 → devservices-1.0.18}/devservices/commands/__init__.py +0 -0
  25. {devservices-1.0.16 → devservices-1.0.18}/devservices/commands/list_services.py +0 -0
  26. {devservices-1.0.16 → devservices-1.0.18}/devservices/commands/purge.py +0 -0
  27. {devservices-1.0.16 → devservices-1.0.18}/devservices/commands/status.py +0 -0
  28. {devservices-1.0.16 → devservices-1.0.18}/devservices/commands/update.py +0 -0
  29. {devservices-1.0.16 → devservices-1.0.18}/devservices/configs/service_config.py +0 -0
  30. {devservices-1.0.16 → devservices-1.0.18}/devservices/constants.py +0 -0
  31. {devservices-1.0.16 → devservices-1.0.18}/devservices/exceptions.py +0 -0
  32. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/__init__.py +0 -0
  33. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/check_for_update.py +0 -0
  34. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/console.py +0 -0
  35. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/devenv.py +0 -0
  36. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/file_lock.py +0 -0
  37. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/git.py +0 -0
  38. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/install_binary.py +0 -0
  39. {devservices-1.0.16 → devservices-1.0.18}/devservices/utils/services.py +0 -0
  40. {devservices-1.0.16 → devservices-1.0.18}/devservices.egg-info/SOURCES.txt +0 -0
  41. {devservices-1.0.16 → devservices-1.0.18}/devservices.egg-info/dependency_links.txt +0 -0
  42. {devservices-1.0.16 → devservices-1.0.18}/devservices.egg-info/entry_points.txt +0 -0
  43. {devservices-1.0.16 → devservices-1.0.18}/devservices.egg-info/requires.txt +0 -0
  44. {devservices-1.0.16 → devservices-1.0.18}/devservices.egg-info/top_level.txt +0 -0
  45. {devservices-1.0.16 → devservices-1.0.18}/setup.cfg +0 -0
  46. {devservices-1.0.16 → devservices-1.0.18}/testing/__init__.py +0 -0
  47. {devservices-1.0.16 → devservices-1.0.18}/testing/utils.py +0 -0
  48. {devservices-1.0.16 → devservices-1.0.18}/tests/__init__.py +0 -0
  49. {devservices-1.0.16 → devservices-1.0.18}/tests/commands/test_list_dependencies.py +0 -0
  50. {devservices-1.0.16 → devservices-1.0.18}/tests/commands/test_list_services.py +0 -0
  51. {devservices-1.0.16 → devservices-1.0.18}/tests/commands/test_logs.py +0 -0
  52. {devservices-1.0.16 → devservices-1.0.18}/tests/commands/test_purge.py +0 -0
  53. {devservices-1.0.16 → devservices-1.0.18}/tests/commands/test_status.py +0 -0
  54. {devservices-1.0.16 → devservices-1.0.18}/tests/commands/test_update.py +0 -0
  55. {devservices-1.0.16 → devservices-1.0.18}/tests/configs/test_service_config.py +0 -0
  56. {devservices-1.0.16 → devservices-1.0.18}/tests/conftest.py +0 -0
  57. {devservices-1.0.16 → devservices-1.0.18}/tests/utils/test_check_for_update.py +0 -0
  58. {devservices-1.0.16 → devservices-1.0.18}/tests/utils/test_git.py +0 -0
  59. {devservices-1.0.16 → devservices-1.0.18}/tests/utils/test_install_binary.py +0 -0
  60. {devservices-1.0.16 → devservices-1.0.18}/tests/utils/test_services.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: devservices
3
- Version: 1.0.16
3
+ Version: 1.0.18
4
4
  Requires-Python: >=3.10
5
5
  License-File: LICENSE.md
6
6
  Requires-Dist: pyyaml
@@ -13,3 +13,4 @@ Requires-Dist: mypy; extra == "dev"
13
13
  Requires-Dist: pre-commit; extra == "dev"
14
14
  Requires-Dist: pytest; extra == "dev"
15
15
  Requires-Dist: types-PyYAML; extra == "dev"
16
+ Dynamic: license-file
@@ -0,0 +1,145 @@
1
+ # devservices
2
+
3
+ A standalone cli tool used to manage dependencies for services. It simplifies the process of managing services for development purposes and bringing services up/down.
4
+
5
+ ## Overview
6
+
7
+ `devservices` reads configuration files located in the `devservices` directory of your repository. These configurations define services, their dependencies, and various modes of operation.
8
+
9
+ ## Usage
10
+
11
+ devservices provides several commands to manage your services:
12
+
13
+ ### Commands
14
+
15
+ NOTE: service-name is an optional parameter. If not provided, devservices will attempt to automatically find a devservices configuration in the current directory in order to proceed.
16
+
17
+ - `devservices up <service-name>`: Bring up a service and its dependencies.
18
+ - `devservices down <service-name>`: Bring down service including its dependencies.
19
+ - `devservices status <service-name>`: Display the current status of all services, including their dependencies and ports.
20
+ - `devservices logs <service-name>`: View logs for a specific service.
21
+ - `devservices list-services`: List all available Sentry services.
22
+ - `devservices list-dependencies <service-name>`: List all dependencies for a service and whether they are enabled/disabled.
23
+ - `devservices update` Update devservices to the latest version.
24
+ - `devservices purge`: Purge the local devservices cache.
25
+
26
+ ## Installation
27
+
28
+ ### 1. Add devservices to your requirements.txt
29
+
30
+ The recommended way to install devservices is through a virtualenv in the requirements.txt. Once that is installed and a devservices config file is added, you should be able to run `devservices up` to begin local development.
31
+
32
+ ```
33
+ devservices==1.0.18
34
+ ```
35
+
36
+ ### 2. Add devservices config files
37
+
38
+ Each repo should have a `devservices` directory with a `config.yml` file. This file is used to define services, dependencies, and modes. Other files and subdirectories in the `devservices` directory are optional and can be most commonly used for volume mounts.
39
+
40
+ The configuration file is a yaml file that looks like this:
41
+
42
+ ```yaml
43
+ # This is a yaml block that holds devservices specific configuration settings. This is comprised of a few main sections:
44
+ # - version: The version of the devservices config file. This is used to ensure compatibility between devservices and the config file.
45
+ # - service_name: The name of the service. This is used to identify the service in the config file.
46
+ # - dependencies: A list of dependencies for the service. Each dependency is a yaml block that holds the dependency configuration. There are two types of dependencies:
47
+ # - local: A dependency that is defined in the config file. These dependencies do not have a remote field. These dependency definitions are specific to the service and are not defined elsewhere.
48
+ # - remote: A dependency that is defined in the devservices directory in a remote repository. These configs are automatically fetched from the remote repository and installed. Any dependency with a remote field will be treated as a remote dependency. Example: https://github.com/getsentry/snuba/blob/59a5258ccbb502827ebc1d3b1bf80c607a3301bf/devservices/config.yml#L8
49
+ # - modes: A list of modes for the service. Each mode includes a list of dependencies that are used in that mode.
50
+ x-sentry-service-config:
51
+ version: 0.1
52
+ service_name: example-service
53
+ dependencies:
54
+ example-dependency-1:
55
+ description: Example dependency defined in the config file
56
+ example-dependency-2:
57
+ description: Example dependency defined in the config file
58
+ example-remote-dependency:
59
+ description: Remote dependency defined in the `devservices` directory in the example-repository repo
60
+ remote:
61
+ repo_name: example-repository
62
+ branch: main
63
+ repo_link: https://github.com/getsentry/example-repository.git
64
+ mode: default # Optional field, mode to run remote dependency in that defaults to `default`
65
+ modes:
66
+ default: [example-dependency-1, example-remote-dependency]
67
+ custom-mode: [example-dependency-1, example-dependency-2, example-remote-dependency]
68
+
69
+ # This will be a standard block used by docker compose to define dependencies.
70
+ #
71
+ # The following fields are important to all dependencies:
72
+ # - image: The docker image to use for the dependency.
73
+ # - ports: The ports to expose for the dependency. Please only expose ports to localhost(127.0.0.1)
74
+ # - healthcheck: The docker healthcheck to use for the dependency.
75
+ # - environment: The environment variables to set for the dependency.
76
+ # - extra_hosts: The extra hosts to add to the dependency.
77
+ # - networks: The networks to add to the dependency. In order for devservices to work properly, the dependency must be on the `devservices` network.
78
+ # - labels: The labels to add to the dependency. The `orchestrator=devservices` label is required for devservices to determine a container is managed by devservices.
79
+ # - restart: The restart policy to use for the dependency.
80
+ #
81
+ # These fields are optional:
82
+ # - ulimits: The ulimits to set for the dependency. This is useful for setting resource constraints for the dependency.
83
+ # - volumes: The volumes to mount for the dependency. This is useful for mounting data volumes for the dependency if data should be persisted between runs. It can also be useful to use a bind mount to mount a local directory into a container. Example of bind mounting clickhouse configs from a local directory https://github.com/getsentry/snuba/blob/59a5258ccbb502827ebc1d3b1bf80c607a3301bf/devservices/config.yml#L44
84
+ # - command: The command to run for the dependency. This can override the default command for the docker image.
85
+ # For more information on the docker compose file services block, see the docker compose file reference: https://docs.docker.com/reference/compose-file/services/
86
+ services:
87
+ example-dependency-1:
88
+ image: ghcr.io/getsentry/example-dependency-1:1.0.0
89
+ ulimits:
90
+ nofile:
91
+ soft: 262144
92
+ hard: 262144
93
+ healthcheck:
94
+ test: wget -q -O - http://localhost:1234/health
95
+ interval: 5s
96
+ timeout: 5s
97
+ retries: 3
98
+ environment:
99
+ EXAMPLE_ENV_VAR: example-value
100
+ volumes:
101
+ - example-dependency-1-data:/var/lib/example-dependency-1
102
+ restart: unless-stopped
103
+ # Everything below this line is required for devservices to work properly.
104
+ ports:
105
+ - 127.0.0.1:1234:1234
106
+ extra_hosts:
107
+ host.docker.internal: host-gateway
108
+ networks:
109
+ - devservices
110
+ labels:
111
+ - orchestrator=devservices
112
+
113
+ example-dependency-2:
114
+ image: ghcr.io/getsentry/example-dependency-2:1.0.0
115
+ command: ["devserver"]
116
+ healthcheck:
117
+ test: curl -f http://localhost:2345/health
118
+ interval: 5s
119
+ timeout: 5s
120
+ retries: 3
121
+ environment:
122
+ EXAMPLE_ENV_VAR: example-value
123
+ restart: unless-stopped
124
+ # Everything below this line is required for devservices to work properly.
125
+ ports:
126
+ - 127.0.0.1:2345:2345
127
+ extra_hosts:
128
+ host.docker.internal: host-gateway
129
+ networks:
130
+ - devservices
131
+ labels:
132
+ - orchestrator=devservices
133
+
134
+ # This is a standard block used by docker compose to define volumes.
135
+ # For more information, see the docker compose file reference: https://docs.docker.com/reference/compose-file/volumes/
136
+ volumes:
137
+ example-dependency-1-data:
138
+
139
+ # This is a standard block used by docker compose to define networks. Defining the devservices network is required for devservices to work properly. By default, devservices will create an external network called `devservices` that is used to connect all dependencies.
140
+ # For more information, see the docker compose file reference: https://docs.docker.com/reference/compose-file/networks/
141
+ networks:
142
+ devservices:
143
+ name: devservices
144
+ external: true
145
+ ```
@@ -30,6 +30,7 @@ from devservices.utils.docker_compose import get_docker_compose_commands_to_run
30
30
  from devservices.utils.docker_compose import run_cmd
31
31
  from devservices.utils.services import find_matching_service
32
32
  from devservices.utils.services import Service
33
+ from devservices.utils.state import ServiceRuntime
33
34
  from devservices.utils.state import State
34
35
  from devservices.utils.state import StateTables
35
36
 
@@ -60,7 +61,7 @@ def down(args: Namespace) -> None:
60
61
  try:
61
62
  service = find_matching_service(service_name)
62
63
  except ConfigNotFoundError as e:
63
- capture_exception(e)
64
+ capture_exception(e, level="info")
64
65
  console.failure(
65
66
  f"{str(e)}. Please specify a service (i.e. `devservices down sentry`) or run the command from a directory with a devservices configuration."
66
67
  )
@@ -122,33 +123,23 @@ def down(args: Namespace) -> None:
122
123
  # Check if any service depends on the service we are trying to bring down
123
124
  # TODO: We should also take into account the active modes of the other services (this is not trivial to do)
124
125
  other_started_services = active_services.difference({service.name})
126
+ services_with_local_runtime = state.get_services_by_runtime(
127
+ ServiceRuntime.LOCAL
128
+ )
125
129
  dependent_service_name = None
126
- for other_started_service in other_started_services:
127
- other_service = find_matching_service(other_started_service)
128
- other_service_active_starting_modes = state.get_active_modes_for_service(
129
- other_service.name, StateTables.STARTING_SERVICES
130
- )
131
- other_service_active_started_modes = state.get_active_modes_for_service(
132
- other_service.name, StateTables.STARTED_SERVICES
133
- )
134
- other_service_active_modes = (
135
- other_service_active_starting_modes
136
- or other_service_active_started_modes
130
+ # We can ignore checking if anything relies on the service
131
+ # if it is a locally running service
132
+ if service.name not in services_with_local_runtime:
133
+ dependent_service_name = _get_dependent_service(
134
+ service, other_started_services, state
137
135
  )
138
- dependency_graph = construct_dependency_graph(
139
- other_service, other_service_active_modes
140
- )
141
- # If the service we are trying to bring down is in the dependency graph of another service, we should not bring it down
142
- if service.name in dependency_graph.graph:
143
- dependent_service_name = other_started_service
144
- break
145
136
 
146
137
  # If no other service depends on the service we are trying to bring down, we can bring it down
147
138
  if dependent_service_name is None:
148
139
  try:
149
140
  _down(service, remote_dependencies, list(mode_dependencies), status)
150
141
  except DockerComposeError as dce:
151
- capture_exception(dce)
142
+ capture_exception(dce, level="info")
152
143
  status.failure(f"Failed to stop {service.name}: {dce.stderr}")
153
144
  exit(1)
154
145
  else:
@@ -190,6 +181,17 @@ def _down(
190
181
  current_env[
191
182
  DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
192
183
  ] = relative_local_dependency_directory
184
+ state = State()
185
+ # We want to ignore any dependencies that are set to run locally
186
+ locally_running_services = state.get_services_by_runtime(ServiceRuntime.LOCAL)
187
+ mode_dependencies = [
188
+ dep for dep in mode_dependencies if dep not in locally_running_services
189
+ ]
190
+ remote_dependencies = {
191
+ dep
192
+ for dep in remote_dependencies
193
+ if dep.service_name not in locally_running_services
194
+ }
193
195
  docker_compose_commands = get_docker_compose_commands_to_run(
194
196
  service=service,
195
197
  remote_dependencies=list(remote_dependencies),
@@ -209,3 +211,30 @@ def _down(
209
211
  ]
210
212
  for future in concurrent.futures.as_completed(futures):
211
213
  cmd_outputs.append(future.result())
214
+
215
+
216
+ def _get_dependent_service(
217
+ service: Service,
218
+ other_started_services: set[str],
219
+ state: State,
220
+ ) -> str | None:
221
+ for other_started_service in other_started_services:
222
+ other_service = find_matching_service(other_started_service)
223
+ other_service_active_starting_modes = state.get_active_modes_for_service(
224
+ other_service.name, StateTables.STARTING_SERVICES
225
+ )
226
+ other_service_active_started_modes = state.get_active_modes_for_service(
227
+ other_service.name, StateTables.STARTED_SERVICES
228
+ )
229
+ other_service_active_modes = (
230
+ other_service_active_starting_modes or other_service_active_started_modes
231
+ )
232
+ dependency_graph = construct_dependency_graph(
233
+ other_service, other_service_active_modes
234
+ )
235
+ # If the service we are trying to bring down is in the dependency graph of another service,
236
+ # we should not bring it down
237
+ if service.name in dependency_graph.graph:
238
+ return other_started_service
239
+
240
+ return None
@@ -34,7 +34,7 @@ def list_dependencies(args: Namespace) -> None:
34
34
  try:
35
35
  service = find_matching_service(service_name)
36
36
  except ConfigNotFoundError as e:
37
- capture_exception(e)
37
+ capture_exception(e, level="info")
38
38
  console.failure(
39
39
  f"{str(e)}. Please specify a service (i.e. `devservices list-dependencies sentry`) or run the command from a directory with a devservices configuration."
40
40
  )
@@ -49,7 +49,7 @@ def logs(args: Namespace) -> None:
49
49
  try:
50
50
  service = find_matching_service(service_name)
51
51
  except ConfigNotFoundError as e:
52
- capture_exception(e)
52
+ capture_exception(e, level="info")
53
53
  console.failure(
54
54
  f"{str(e)}. Please specify a service (i.e. `devservices logs sentry`) or run the command from a directory with a devservices configuration."
55
55
  )
@@ -86,7 +86,7 @@ def logs(args: Namespace) -> None:
86
86
  try:
87
87
  logs_output = _logs(service, remote_dependencies, mode_dependencies)
88
88
  except DockerComposeError as dce:
89
- capture_exception(dce)
89
+ capture_exception(dce, level="info")
90
90
  console.failure(f"Failed to get logs for {service.name}: {dce.stderr}")
91
91
  exit(1)
92
92
  for log in logs_output:
@@ -33,6 +33,7 @@ from devservices.utils.docker_compose import get_docker_compose_commands_to_run
33
33
  from devservices.utils.docker_compose import run_cmd
34
34
  from devservices.utils.services import find_matching_service
35
35
  from devservices.utils.services import Service
36
+ from devservices.utils.state import ServiceRuntime
36
37
  from devservices.utils.state import State
37
38
  from devservices.utils.state import StateTables
38
39
 
@@ -63,7 +64,7 @@ def up(args: Namespace) -> None:
63
64
  try:
64
65
  service = find_matching_service(service_name)
65
66
  except ConfigNotFoundError as e:
66
- capture_exception(e)
67
+ capture_exception(e, level="info")
67
68
  console.failure(
68
69
  f"{str(e)}. Please specify a service (i.e. `devservices up sentry`) or run the command from a directory with a devservices configuration."
69
70
  )
@@ -106,11 +107,23 @@ def up(args: Namespace) -> None:
106
107
  pass
107
108
  # Add the service to the starting services table
108
109
  state.update_service_entry(service.name, mode, StateTables.STARTING_SERVICES)
110
+ mode_dependencies = modes[mode]
111
+ # We want to ignore any dependencies that are set to run locally
112
+ services_with_local_runtime = state.get_services_by_runtime(
113
+ ServiceRuntime.LOCAL
114
+ )
115
+ mode_dependencies = [
116
+ dep for dep in mode_dependencies if dep not in services_with_local_runtime
117
+ ]
118
+ remote_dependencies = {
119
+ dep
120
+ for dep in remote_dependencies
121
+ if dep.service_name not in services_with_local_runtime
122
+ }
109
123
  try:
110
- mode_dependencies = modes[mode]
111
124
  _up(service, [mode], remote_dependencies, mode_dependencies, status)
112
125
  except DockerComposeError as dce:
113
- capture_exception(dce)
126
+ capture_exception(dce, level="info")
114
127
  status.failure(f"Failed to start {service.name}: {dce.stderr}")
115
128
  exit(1)
116
129
  # TODO: We should factor in healthchecks here before marking service as running
@@ -118,12 +131,20 @@ def up(args: Namespace) -> None:
118
131
  state.update_service_entry(service.name, mode, StateTables.STARTED_SERVICES)
119
132
 
120
133
 
134
+ def _pull_dependency_images(
135
+ cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
136
+ ) -> None:
137
+ run_cmd(cmd.full_command, current_env)
138
+ for dependency in cmd.services:
139
+ status.info(f"Pulled image for {dependency}")
140
+
141
+
121
142
  def _bring_up_dependency(
122
143
  cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
123
- ) -> subprocess.CompletedProcess[str]:
144
+ ) -> None:
124
145
  for dependency in cmd.services:
125
146
  status.info(f"Starting {dependency}")
126
- return run_cmd(cmd.full_command, current_env)
147
+ run_cmd(cmd.full_command, current_env)
127
148
 
128
149
 
129
150
  def _up(
@@ -150,26 +171,51 @@ def _up(
150
171
  sorted_remote_dependencies = sorted(
151
172
  remote_dependencies, key=lambda dep: starting_order.index(dep.service_name)
152
173
  )
153
- docker_compose_commands = get_docker_compose_commands_to_run(
174
+ # Pull all images in parallel
175
+ status.info("Pulling images")
176
+ pull_commands = get_docker_compose_commands_to_run(
177
+ service=service,
178
+ remote_dependencies=sorted_remote_dependencies,
179
+ current_env=current_env,
180
+ command="pull",
181
+ options=[],
182
+ service_config_file_path=service_config_file_path,
183
+ mode_dependencies=mode_dependencies,
184
+ )
185
+
186
+ with concurrent.futures.ThreadPoolExecutor() as pull_dependency_executor:
187
+ futures = [
188
+ pull_dependency_executor.submit(
189
+ _pull_dependency_images, cmd, current_env, status
190
+ )
191
+ for cmd in pull_commands
192
+ ]
193
+ for future in concurrent.futures.as_completed(futures):
194
+ _ = future.result()
195
+
196
+ # Bring up all necessary containers
197
+ up_commands = get_docker_compose_commands_to_run(
154
198
  service=service,
155
199
  remote_dependencies=sorted_remote_dependencies,
156
200
  current_env=current_env,
157
201
  command="up",
158
- options=["-d", "--pull", "always"],
202
+ options=["-d"],
159
203
  service_config_file_path=service_config_file_path,
160
204
  mode_dependencies=mode_dependencies,
161
205
  )
162
206
 
163
207
  containers_to_check = []
164
- with concurrent.futures.ThreadPoolExecutor() as dependency_executor:
208
+ with concurrent.futures.ThreadPoolExecutor() as up_dependency_executor:
165
209
  futures = [
166
- dependency_executor.submit(_bring_up_dependency, cmd, current_env, status)
167
- for cmd in docker_compose_commands
210
+ up_dependency_executor.submit(
211
+ _bring_up_dependency, cmd, current_env, status
212
+ )
213
+ for cmd in up_commands
168
214
  ]
169
215
  for future in concurrent.futures.as_completed(futures):
170
216
  _ = future.result()
171
217
 
172
- for cmd in docker_compose_commands:
218
+ for cmd in up_commands:
173
219
  try:
174
220
  container_names = get_container_names_for_project(
175
221
  cmd.project_name, cmd.config_path
@@ -11,10 +11,13 @@ from importlib import metadata
11
11
  from sentry_sdk import capture_exception
12
12
  from sentry_sdk import flush
13
13
  from sentry_sdk import init
14
+ from sentry_sdk import set_context
14
15
  from sentry_sdk import set_tag
15
16
  from sentry_sdk import set_user
16
17
  from sentry_sdk import start_transaction
17
18
  from sentry_sdk.integrations.argv import ArgvIntegration
19
+ from sentry_sdk.types import Event
20
+ from sentry_sdk.types import Hint
18
21
 
19
22
  from devservices.commands import down
20
23
  from devservices.commands import list_dependencies
@@ -42,6 +45,31 @@ disable_sentry = os.environ.get("DEVSERVICES_DISABLE_SENTRY", default="0") == "1
42
45
  logging.basicConfig(level=logging.INFO)
43
46
  current_version = metadata.version("devservices")
44
47
 
48
+ error_trace_ids = set()
49
+
50
+
51
+ def before_send_error(event: Event, hint: Hint) -> Event:
52
+ """Gets the trace_id from the errors we care about.
53
+
54
+ This function is used as a before_send callback for Sentry to track error trace IDs.
55
+ It adds the trace_id to error_trace_ids set for non-info level events.
56
+ """
57
+ if event["level"] != "info":
58
+ error_trace_ids.add(event["contexts"]["trace"]["trace_id"])
59
+ return event
60
+
61
+
62
+ def before_send_transaction(event: Event, hint: Hint) -> Event:
63
+ """Manually sets the status of a transaction.
64
+
65
+ This function is used as a before_send_transaction callback for Sentry to mark transaction status
66
+ as unknown if they don't correspond to errors we care about.
67
+ """
68
+ if event["contexts"]["trace"]["trace_id"] not in error_trace_ids:
69
+ event["contexts"]["trace"]["status"] = "unknown"
70
+ return event
71
+
72
+
45
73
  if not disable_sentry:
46
74
  init(
47
75
  dsn="https://56470da7302c16e83141f62f88e46449@o1.ingest.us.sentry.io/4507946704961536",
@@ -50,16 +78,34 @@ if not disable_sentry:
50
78
  enable_tracing=True,
51
79
  integrations=[ArgvIntegration()],
52
80
  environment=sentry_environment,
81
+ before_send=before_send_error,
82
+ before_send_transaction=before_send_transaction,
53
83
  release=current_version,
54
84
  )
55
85
  username = getpass.getuser()
56
86
  set_user({"username": username})
57
87
  set_tag("user_platform", platform.platform())
88
+ if sentry_environment == "CI":
89
+ set_context(
90
+ "github",
91
+ {
92
+ "github_action": os.environ.get("GITHUB_ACTION"),
93
+ "github_action_path": os.environ.get("GITHUB_ACTION_PATH"),
94
+ "github_repository": os.environ.get("GITHUB_REPOSITORY"),
95
+ "github_ref_name": os.environ.get("GITHUB_REF_NAME"),
96
+ "github_run_id": os.environ.get("GITHUB_RUN_ID"),
97
+ "github_url": f"{os.environ.get('GITHUB_SERVER_URL')}/{os.environ.get('GITHUB_REPOSITORY')}/actions/runs/{os.environ.get('GITHUB_RUN_ID')}",
98
+ "github_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
99
+ "github_workflow": os.environ.get("GITHUB_WORKFLOW"),
100
+ "github_workflow_run_id": os.environ.get("GITHUB_WORKFLOW_RUN_ID"),
101
+ "github_sha": os.environ.get("GITHUB_SHA"),
102
+ },
103
+ )
58
104
  try:
59
105
  git_version = get_git_version()
60
106
  set_tag("git_version", git_version)
61
107
  except GitError as e:
62
- capture_exception(e)
108
+ capture_exception(e, level="info")
63
109
  logging.debug("Failed to get git version: %s", e)
64
110
  set_tag("git_version", "unknown")
65
111
 
@@ -75,11 +121,11 @@ def main() -> None:
75
121
  try:
76
122
  check_docker_compose_version()
77
123
  except DockerDaemonNotRunningError as e:
78
- capture_exception(e)
124
+ capture_exception(e, level="info")
79
125
  console.failure(str(e))
80
126
  exit(1)
81
127
  except DockerComposeInstallationError as e:
82
- capture_exception(e)
128
+ capture_exception(e, level="info")
83
129
  console.failure("Failed to ensure docker compose is installed and up-to-date")
84
130
  exit(1)
85
131
  parser = argparse.ArgumentParser(
@@ -271,7 +271,8 @@ def get_non_shared_remote_dependencies(
271
271
  started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
272
272
  active_services = starting_services.union(started_services)
273
273
  # We don't care about the remote dependencies of the service we are stopping
274
- active_services.remove(service_to_stop.name)
274
+ if service_to_stop.name in active_services:
275
+ active_services.remove(service_to_stop.name)
275
276
 
276
277
  active_modes: dict[str, list[str]] = dict()
277
278
  for active_service in active_services:
@@ -3,6 +3,7 @@ from __future__ import annotations
3
3
  import concurrent.futures
4
4
  import subprocess
5
5
  import time
6
+ from typing import NamedTuple
6
7
 
7
8
  from devservices.constants import HEALTHCHECK_INTERVAL
8
9
  from devservices.constants import HEALTHCHECK_TIMEOUT
@@ -12,6 +13,11 @@ from devservices.exceptions import DockerError
12
13
  from devservices.utils.console import Status
13
14
 
14
15
 
16
+ class ContainerNames(NamedTuple):
17
+ name: str
18
+ short_name: str
19
+
20
+
15
21
  def check_docker_daemon_running() -> None:
16
22
  """Checks if the Docker daemon is running. Raises DockerDaemonNotRunningError if not."""
17
23
  try:
@@ -25,8 +31,11 @@ def check_docker_daemon_running() -> None:
25
31
  raise DockerDaemonNotRunningError from e
26
32
 
27
33
 
28
- def check_all_containers_healthy(status: Status, containers: list[str]) -> None:
34
+ def check_all_containers_healthy(
35
+ status: Status, containers: list[ContainerNames]
36
+ ) -> None:
29
37
  """Ensures all containers are healthy."""
38
+ status.info("Waiting for all containers to be healthy")
30
39
  with concurrent.futures.ThreadPoolExecutor() as healthcheck_executor:
31
40
  futures = [
32
41
  healthcheck_executor.submit(wait_for_healthy, container, status)
@@ -36,7 +45,7 @@ def check_all_containers_healthy(status: Status, containers: list[str]) -> None:
36
45
  future.result()
37
46
 
38
47
 
39
- def wait_for_healthy(container_name: str, status: Status) -> None:
48
+ def wait_for_healthy(container: ContainerNames, status: Status) -> None:
40
49
  """
41
50
  Polls a Docker container's health status until it becomes healthy or a timeout is reached.
42
51
  """
@@ -51,31 +60,32 @@ def wait_for_healthy(container_name: str, status: Status) -> None:
51
60
  "inspect",
52
61
  "-f",
53
62
  "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}",
54
- container_name,
63
+ container.name,
55
64
  ],
56
65
  stderr=subprocess.DEVNULL,
57
66
  text=True,
58
67
  ).strip()
59
68
  except subprocess.CalledProcessError as e:
60
69
  raise DockerError(
61
- command=f"docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' {container_name}",
70
+ command=f"docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' {container.name}",
62
71
  returncode=e.returncode,
63
72
  stdout=e.stdout,
64
73
  stderr=e.stderr,
65
74
  ) from e
66
75
 
67
76
  if result == "healthy":
77
+ status.info(f"{container.short_name} is healthy")
68
78
  return
69
79
  if result == "unknown":
70
80
  status.warning(
71
- f"WARNING: Container {container_name} does not have a healthcheck"
81
+ f"WARNING: Container {container.short_name} does not have a healthcheck"
72
82
  )
73
83
  return
74
84
 
75
85
  # If not healthy, wait and try again
76
86
  time.sleep(HEALTHCHECK_INTERVAL)
77
87
 
78
- raise ContainerHealthcheckFailedError(container_name, HEALTHCHECK_TIMEOUT)
88
+ raise ContainerHealthcheckFailedError(container.short_name, HEALTHCHECK_TIMEOUT)
79
89
 
80
90
 
81
91
  def get_matching_containers(label: str) -> list[str]:
@@ -1,5 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import json
3
4
  import logging
4
5
  import os
5
6
  import platform
@@ -23,6 +24,7 @@ from devservices.exceptions import DockerComposeInstallationError
23
24
  from devservices.utils.console import Console
24
25
  from devservices.utils.dependencies import InstalledRemoteDependency
25
26
  from devservices.utils.docker import check_docker_daemon_running
27
+ from devservices.utils.docker import ContainerNames
26
28
  from devservices.utils.install_binary import install_binary
27
29
  from devservices.utils.services import Service
28
30
 
@@ -92,9 +94,11 @@ def install_docker_compose() -> None:
92
94
  console.success(f"Verified Docker Compose installation: v{version}")
93
95
 
94
96
 
95
- def get_container_names_for_project(project_name: str, config_path: str) -> list[str]:
97
+ def get_container_names_for_project(
98
+ project_name: str, config_path: str
99
+ ) -> list[ContainerNames]:
96
100
  try:
97
- container_names = subprocess.check_output(
101
+ output = subprocess.check_output(
98
102
  [
99
103
  "docker",
100
104
  "compose",
@@ -104,11 +108,15 @@ def get_container_names_for_project(project_name: str, config_path: str) -> list
104
108
  config_path,
105
109
  "ps",
106
110
  "--format",
107
- "{{.Name}}",
111
+ '{"name":"{{.Names}}", "short_name":"{{.Service}}"}',
108
112
  ],
109
113
  text=True,
110
114
  ).splitlines()
111
- return container_names
115
+ return [
116
+ ContainerNames(name=json_data["name"], short_name=json_data["short_name"])
117
+ for line in output
118
+ if (json_data := json.loads(line))
119
+ ]
112
120
  except subprocess.CalledProcessError as e:
113
121
  raise DockerComposeError(
114
122
  command=f"docker compose -p {project_name} -f {config_path} ps --format {{.Name}}",