devservices 1.0.3__tar.gz → 1.0.4__tar.gz

{devservices-1.0.3 → devservices-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: devservices
-Version: 1.0.3
+Version: 1.0.4
 Requires-Python: >=3.10
 License-File: LICENSE.md
 Requires-Dist: pyyaml

{devservices-1.0.3 → devservices-1.0.4}/README.md

@@ -11,7 +11,7 @@ A standalone cli tool used to manage dependencies for services. It simplifies th
 The recommended way to install devservices is through a virtualenv in the requirements.txt.
 
 ```
-devservices==1.0.3
+devservices==1.0.4
 ```
 
 

{devservices-1.0.3 → devservices-1.0.4}/devservices/commands/down.py

@@ -126,7 +126,7 @@ def _down(
     ] = relative_local_dependency_directory
     docker_compose_commands = get_docker_compose_commands_to_run(
         service=service,
-        remote_dependencies=remote_dependencies,
+        remote_dependencies=list(remote_dependencies),
         current_env=current_env,
         command="down",
         options=[],

{devservices-1.0.3 → devservices-1.0.4}/devservices/commands/logs.py

@@ -99,7 +99,7 @@ def _logs(
     ] = relative_local_dependency_directory
     docker_compose_commands = get_docker_compose_commands_to_run(
         service=service,
-        remote_dependencies=remote_dependencies,
+        remote_dependencies=list(remote_dependencies),
         current_env=current_env,
         command="logs",
         options=["-n", MAX_LOG_LINES],

{devservices-1.0.3 → devservices-1.0.4}/devservices/commands/status.py

@@ -137,7 +137,7 @@ def _status(
     ] = relative_local_dependency_directory
     docker_compose_commands = get_docker_compose_commands_to_run(
         service=service,
-        remote_dependencies=remote_dependencies,
+        remote_dependencies=list(remote_dependencies),
         current_env=current_env,
         command="ps",
         options=["--format", "json"],

{devservices-1.0.3 → devservices-1.0.4}/devservices/commands/up.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import concurrent.futures
 import os
 import subprocess
 from argparse import _SubParsersAction
@@ -21,6 +22,8 @@ from devservices.exceptions import ModeDoesNotExistError
 from devservices.exceptions import ServiceNotFoundError
 from devservices.utils.console import Console
 from devservices.utils.console import Status
+from devservices.utils.dependencies import construct_dependency_graph
+from devservices.utils.dependencies import get_non_shared_remote_dependencies
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run
@@ -63,8 +66,58 @@ def up(args: Namespace) -> None:
     modes = service.config.modes
     mode = args.mode
 
+    state = State()
+    started_services = state.get_started_services()
+    running_mode = state.get_mode_for_service(service.name) or "default"
+
+    # TODO: Remove this once we properly handle mode switching
+    if service.name in started_services and running_mode != mode:
+        console.warning(
+            f"Service '{service.name}' is already running in mode: '{running_mode}', restarting in mode: '{mode}'"
+        )
+        with Status() as status:
+            try:
+                remote_dependencies = install_and_verify_dependencies(
+                    service, mode=running_mode
+                )
+            except DependencyError as de:
+                capture_exception(de)
+                status.failure(str(de))
+                exit(1)
+            except ModeDoesNotExistError as mde:
+                capture_exception(mde)
+                status.failure(str(mde))
+                exit(1)
+            service_config_file_path = os.path.join(
+                service.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
+            )
+            current_env = os.environ.copy()
+            running_mode_dependencies = modes[running_mode]
+            remote_dependencies_to_bring_down = get_non_shared_remote_dependencies(
+                service, remote_dependencies
+            )
+            down_docker_compose_commands = get_docker_compose_commands_to_run(
+                service=service,
+                remote_dependencies=list(remote_dependencies_to_bring_down),
+                current_env=current_env,
+                command="down",
+                options=[],
+                service_config_file_path=service_config_file_path,
+                mode_dependencies=running_mode_dependencies,
+            )
+
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                futures = [
+                    executor.submit(run_cmd, cmd, current_env)
+                    for cmd in down_docker_compose_commands
+                ]
+                for future in concurrent.futures.as_completed(futures):
+                    future.result()
+
+            state.remove_started_service(service.name)
+
     with Status(
-        lambda: console.warning(f"Starting {service.name}"),
+        lambda: console.warning(f"Starting '{service.name}' in mode: '{mode}'"),
         lambda: console.success(f"{service.name} started"),
     ) as status:
         try:
@@ -79,6 +132,11 @@ def up(args: Namespace) -> None:
         except ModeDoesNotExistError as mde:
             status.failure(str(mde))
             exit(1)
+        try:
+            _create_devservices_network()
+        except subprocess.CalledProcessError:
+            # Network already exists, ignore the error
+            pass
         try:
             mode_dependencies = modes[mode]
             _up(service, remote_dependencies, mode_dependencies, status)
@@ -119,9 +177,14 @@ def _up(
         DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
     ] = relative_local_dependency_directory
     options = ["-d"]
+    dependency_graph = construct_dependency_graph(service)
+    starting_order = dependency_graph.get_starting_order()
+    sorted_remote_dependencies = sorted(
+        remote_dependencies, key=lambda dep: starting_order.index(dep.service_name)
+    )
     docker_compose_commands = get_docker_compose_commands_to_run(
         service=service,
-        remote_dependencies=remote_dependencies,
+        remote_dependencies=sorted_remote_dependencies,
         current_env=current_env,
         command="up",
         options=options,
@@ -131,3 +194,11 @@ def _up(
 
     for cmd in docker_compose_commands:
         _bring_up_dependency(cmd, current_env, status, len(options))
+
+
+def _create_devservices_network() -> None:
+    subprocess.run(
+        ["docker", "network", "create", "devservices"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )

{devservices-1.0.3 → devservices-1.0.4}/devservices/utils/dependencies.py

@@ -5,6 +5,7 @@ import os
 import shutil
 import subprocess
 import tempfile
+from collections import deque
 from concurrent.futures import as_completed
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
@@ -14,6 +15,7 @@ from typing import TypeGuard
 from devservices.configs.service_config import Dependency
 from devservices.configs.service_config import load_service_config_from_file
 from devservices.configs.service_config import RemoteConfig
+from devservices.configs.service_config import ServiceConfig
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS
@@ -35,6 +37,62 @@ from devservices.utils.services import Service
 from devservices.utils.state import State
 
 
+class DependencyGraph:
+    def __init__(self) -> None:
+        self.graph: dict[str, set[str]] = dict()
+
+    def add_dependency(self, service_name: str) -> None:
+        if service_name not in self.graph:
+            self.graph[service_name] = set()
+
+    def add_edge(self, service_name: str, dependency_name: str) -> None:
+        # TODO: We should rename services that depend on themselves
+        if service_name == dependency_name:
+            return
+        if service_name not in self.graph:
+            self.add_dependency(service_name)
+        if dependency_name not in self.graph:
+            self.add_dependency(dependency_name)
+
+        # TODO: Should we check for cycles here?
+
+        self.graph[service_name].add(dependency_name)
+
+    def topological_sort(self) -> list[str]:
+        in_degree = {service_name: 0 for service_name in self.graph}
+
+        for service_name in self.graph.keys():
+            for dependency in self.graph[service_name]:
+                in_degree[dependency] += 1
+
+        queue = deque(
+            [
+                service_name
+                for service_name in self.graph
+                if in_degree[service_name] == 0
+            ]
+        )
+        topological_order = list()
+
+        while queue:
+            service_name = queue.popleft()
+            topological_order.append(service_name)
+
+            for dependency in self.graph[service_name]:
+                in_degree[dependency] -= 1
+                if in_degree[dependency] == 0:
+                    queue.append(dependency)
+
+        if len(topological_order) != len(self.graph):
+            # TODO: Add a better exception
+            raise ValueError("Cycle detected in the dependency graph")
+
+        return topological_order
+
+    def get_starting_order(self) -> list[str]:
+        return list(reversed(self.topological_sort()))
+
+
 @dataclass(frozen=True)
 class InstalledRemoteDependency:
     service_name: str
@@ -457,3 +515,26 @@ def _run_command(
     logger = logging.getLogger(LOGGER_NAME)
     logger.debug(f"Running command: {' '.join(cmd)} in {cwd}")
     subprocess.run(cmd, cwd=cwd, check=True, stdout=stdout, stderr=subprocess.DEVNULL)
+
+
+def get_remote_dependency_config(remote_config: RemoteConfig) -> ServiceConfig:
+    dependency_repo_dir = os.path.join(
+        DEVSERVICES_DEPENDENCIES_CACHE_DIR,
+        DEPENDENCY_CONFIG_VERSION,
+        remote_config.repo_name,
+    )
+    return load_service_config_from_file(dependency_repo_dir)
+
+
+def construct_dependency_graph(service: Service) -> DependencyGraph:
+    dependency_graph = DependencyGraph()
+
+    def _construct_dependency_graph(service_config: ServiceConfig) -> None:
+        for dependency_name, dependency in service_config.dependencies.items():
+            dependency_graph.add_edge(service_config.service_name, dependency_name)
+            if _has_remote_config(dependency.remote):
+                dependency_config = get_remote_dependency_config(dependency.remote)
+                _construct_dependency_graph(dependency_config)
+
+    _construct_dependency_graph(service.config)
+    return dependency_graph

{devservices-1.0.3 → devservices-1.0.4}/devservices/utils/docker_compose.py

@@ -163,7 +163,7 @@ def _get_non_remote_services(
 
 def get_docker_compose_commands_to_run(
     service: Service,
-    remote_dependencies: set[InstalledRemoteDependency],
+    remote_dependencies: list[InstalledRemoteDependency],
     current_env: dict[str, str],
     command: str,
     options: list[str],
@@ -184,8 +184,7 @@ def get_docker_compose_commands_to_run(
         + sorted(list(services_to_use))  # Sort the services to prevent flaky tests
         + options
     )
-    # Sort the remote dependencies by service name to ensure a deterministic order
-    for dependency in sorted(remote_dependencies, key=lambda x: x.service_name):
+    for dependency in remote_dependencies:
         # TODO: Consider passing in service config in InstalledRemoteDependency instead of loading it here
         dependency_service_config = load_service_config_from_file(dependency.repo_path)
         dependency_config_path = os.path.join(

{devservices-1.0.3 → devservices-1.0.4}/devservices.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: devservices
-Version: 1.0.3
+Version: 1.0.4
 Requires-Python: >=3.10
 License-File: LICENSE.md
 Requires-Dist: pyyaml

{devservices-1.0.3 → devservices-1.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "devservices"
-version = "1.0.3"
+version = "1.0.4"
 # 3.10 is just for internal pypi compat
 requires-python = ">=3.10"
 dependencies = [

{devservices-1.0.3 → devservices-1.0.4}/tests/commands/test_up.py

@@ -14,7 +14,10 @@ from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
 from devservices.constants import DEVSERVICES_DIR_NAME
 from devservices.exceptions import DependencyError
+from devservices.utils.state import State
 from testing.utils import create_config_file
+from testing.utils import create_mock_git_repo
+from testing.utils import run_git_command
 
 
 @mock.patch(
@@ -26,7 +29,9 @@ from testing.utils import create_config_file
     ),
 )
 @mock.patch("devservices.utils.state.State.add_started_service")
+@mock.patch("devservices.commands.up._create_devservices_network")
 def test_up_simple(
+    mock_create_devservices_network: mock.Mock,
     mock_add_started_service: mock.Mock,
     mock_run: mock.Mock,
     tmp_path: Path,
@@ -69,6 +74,8 @@ def test_up_simple(
             == f"../dependency-dir/{DEPENDENCY_CONFIG_VERSION}"
         )
 
+        mock_create_devservices_network.assert_called_once()
+
         mock_run.assert_called_with(
             [
                 "docker",
@@ -91,13 +98,16 @@ def test_up_simple(
         mock_add_started_service.assert_called_with("example-service", "default")
         captured = capsys.readouterr()
         assert "Retrieving dependencies" in captured.out.strip()
+        assert "Starting 'example-service' in mode: 'default'" in captured.out.strip()
         assert "Starting clickhouse" in captured.out.strip()
         assert "Starting redis" in captured.out.strip()
 
 
 @mock.patch("devservices.utils.docker_compose.subprocess.run")
 @mock.patch("devservices.utils.state.State.add_started_service")
+@mock.patch("devservices.commands.up._create_devservices_network")
 def test_up_dependency_error(
+    mock_create_devservices_network: mock.Mock,
     mock_add_started_service: mock.Mock,
     mock_run: mock.Mock,
     capsys: pytest.CaptureFixture[str],
@@ -135,6 +145,7 @@ def test_up_dependency_error(
         with pytest.raises(SystemExit):
             up(args)
 
+        mock_create_devservices_network.assert_not_called()
         # Capture the printed output
         captured = capsys.readouterr()
 
@@ -144,13 +155,18 @@ def test_up_dependency_error(
 
         captured = capsys.readouterr()
         assert "Retrieving dependencies" not in captured.out.strip()
+        assert (
+            "Starting 'example-service' in mode: 'default'" not in captured.out.strip()
+        )
         assert "Starting clickhouse" not in captured.out.strip()
         assert "Starting redis" not in captured.out.strip()
 
 
 @mock.patch("devservices.utils.docker_compose.subprocess.run")
 @mock.patch("devservices.utils.state.State.add_started_service")
+@mock.patch("devservices.commands.up._create_devservices_network")
 def test_up_error(
+    mock_create_devservices_network: mock.Mock,
     mock_add_started_service: mock.Mock,
     mock_run: mock.Mock,
     capsys: pytest.CaptureFixture[str],
@@ -185,6 +201,8 @@ def test_up_error(
     with pytest.raises(SystemExit):
         up(args)
 
+    mock_create_devservices_network.assert_called_once()
+
     # Capture the printed output
     captured = capsys.readouterr()
 
@@ -196,6 +214,7 @@ def test_up_error(
 
     captured = capsys.readouterr()
     assert "Retrieving dependencies" not in captured.out.strip()
+    assert "Starting 'example-service' in mode: 'default'" not in captured.out.strip()
     assert "Starting clickhouse" not in captured.out.strip()
     assert "Starting redis" not in captured.out.strip()
 
@@ -209,7 +228,9 @@ def test_up_error(
     ),
 )
 @mock.patch("devservices.utils.state.State.add_started_service")
+@mock.patch("devservices.commands.up._create_devservices_network")
 def test_up_mode_simple(
+    mock_create_devservices_network: mock.Mock,
     mock_add_started_service: mock.Mock,
     mock_run: mock.Mock,
     tmp_path: Path,
@@ -252,6 +273,8 @@ def test_up_mode_simple(
             == f"../dependency-dir/{DEPENDENCY_CONFIG_VERSION}"
         )
 
+        mock_create_devservices_network.assert_called_once()
+
         mock_run.assert_called_with(
             [
                 "docker",
@@ -273,6 +296,7 @@ def test_up_mode_simple(
         mock_add_started_service.assert_called_with("example-service", "test")
         captured = capsys.readouterr()
         assert "Retrieving dependencies" in captured.out.strip()
+        assert "Starting 'example-service' in mode: 'test'" in captured.out.strip()
         assert "Starting redis" in captured.out.strip()
 
 
@@ -334,5 +358,244 @@ def test_up_mode_does_not_exist(
 
         captured = capsys.readouterr()
         assert "Retrieving dependencies" not in captured.out.strip()
+        assert "Starting 'example-service' in mode: 'test'" not in captured.out.strip()
         assert "Starting clickhouse" not in captured.out.strip()
         assert "Starting redis" not in captured.out.strip()
+
+
+@mock.patch(
+    "devservices.utils.docker_compose.subprocess.run",
+    return_value=subprocess.CompletedProcess(
+        args=["docker", "compose", "config", "--services"],
+        returncode=0,
+        stdout="clickhouse\nredis\n",
+    ),
+)
+def test_up_switching_modes(
+    mock_run: mock.Mock,
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    with (
+        mock.patch(
+            "devservices.commands.up.DEVSERVICES_DEPENDENCIES_CACHE_DIR",
+            str(tmp_path / "dependency-dir"),
+        ),
+        mock.patch("devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")),
+    ):
+        config = {
+            "x-sentry-service-config": {
+                "version": 0.1,
+                "service_name": "example-service",
+                "dependencies": {
+                    "redis": {"description": "Redis"},
+                    "clickhouse": {"description": "Clickhouse"},
+                },
+                "modes": {"default": ["redis", "clickhouse"], "test": ["redis"]},
+            },
+            "services": {
+                "redis": {"image": "redis:6.2.14-alpine"},
+                "clickhouse": {
+                    "image": "altinity/clickhouse-server:23.8.11.29.altinitystable"
+                },
+            },
+        }
+
+        service_path = tmp_path / "example-service"
+        create_config_file(service_path, config)
+        os.chdir(service_path)
+
+        state = State()
+        state.add_started_service("example-service", "default")
+
+        args = Namespace(service_name=None, debug=False, mode="test")
+        up(args)
+
+        mock_run.assert_has_calls(
+            [
+                mock.call(
+                    [
+                        "docker",
+                        "compose",
+                        "-p",
+                        "example-service",
+                        "-f",
+                        f"{service_path}/{DEVSERVICES_DIR_NAME}/{CONFIG_FILE_NAME}",
+                        "down",
+                        "clickhouse",
+                        "redis",
+                    ],
+                    check=True,
+                    capture_output=True,
+                    text=True,
+                    env=mock.ANY,
+                ),
+                mock.call(
+                    [
+                        "docker",
+                        "compose",
+                        "-p",
+                        "example-service",
+                        "-f",
+                        f"{service_path}/{DEVSERVICES_DIR_NAME}/{CONFIG_FILE_NAME}",
+                        "up",
+                        "redis",
+                        "-d",
+                    ],
+                    check=True,
+                    capture_output=True,
+                    text=True,
+                    env=mock.ANY,
+                ),
+            ],
+            any_order=True,
+        )
+
+        captured = capsys.readouterr()
+        assert (
+            "Service 'example-service' is already running in mode: 'default', restarting in mode: 'test'"
+            in captured.out.strip()
+        )
+        assert "Starting 'example-service' in mode: 'test'" in captured.out.strip()
+        assert "Retrieving dependencies" in captured.out.strip()
+        assert "Starting redis" in captured.out.strip()
+
+
+def test_up_switching_modes_overlapping_running_service(
+    tmp_path: Path,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    with (
+        mock.patch(
+            "devservices.commands.up.DEVSERVICES_DEPENDENCIES_CACHE_DIR",
+            str(tmp_path / "dependency-dir"),
+        ),
+        mock.patch(
+            "devservices.utils.dependencies.DEVSERVICES_DEPENDENCIES_CACHE_DIR",
+            str(tmp_path / "dependency-dir"),
+        ),
+        mock.patch(
+            "devservices.utils.services.get_coderoot",
+            return_value=str(tmp_path / "code"),
+        ),
+        mock.patch("devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")),
+    ):
+        redis_repo_path = tmp_path / "redis"
+        create_mock_git_repo("blank_repo", redis_repo_path)
+        mock_git_repo_config = {
+            "x-sentry-service-config": {
+                "version": 0.1,
+                "service_name": "shared-redis",
+                "dependencies": {},
+                "modes": {"default": []},
+            },
+            "services": {
+                "redis": {"image": "redis:6.2.14-alpine"},
+            },
+        }
+        create_config_file(redis_repo_path, mock_git_repo_config)
+        run_git_command(["add", "."], cwd=redis_repo_path)
+        run_git_command(["commit", "-m", "Add devservices config"], cwd=redis_repo_path)
+        config = {
+            "x-sentry-service-config": {
+                "version": 0.1,
+                "service_name": "example-service",
+                "dependencies": {
+                    "redis": {
+                        "description": "Redis",
+                        "remote": {
+                            "repo_name": "redis",
+                            "branch": "main",
+                            "repo_link": f"file://{redis_repo_path}",
+                        },
+                    },
+                    "clickhouse": {"description": "Clickhouse"},
+                },
+                "modes": {"default": ["redis", "clickhouse"], "test": ["clickhouse"]},
+            },
+            "services": {
+                "clickhouse": {
+                    "image": "altinity/clickhouse-server:23.8.11.29.altinitystable"
+                },
+            },
+        }
+        other_config = {
+            "x-sentry-service-config": {
+                "version": 0.1,
+                "service_name": "other-service",
+                "dependencies": {
+                    "redis": {
+                        "description": "Redis",
+                        "remote": {
+                            "repo_name": "redis",
+                            "branch": "main",
+                            "repo_link": f"file://{redis_repo_path}",
+                        },
+                    },
+                },
+                "modes": {"default": ["redis"]},
+            },
+        }
+
+        service_path = tmp_path / "code" / "example-service"
+        other_service_path = tmp_path / "code" / "other-service"
+        create_config_file(service_path, config)
+        create_config_file(other_service_path, other_config)
+        os.chdir(service_path)
+
+        state = State()
+        state.add_started_service("example-service", "default")
+        state.add_started_service("other-service", "default")
+
+        args = Namespace(service_name="example-service", debug=False, mode="test")
+
+        with mock.patch(
+            "devservices.commands.up.run_cmd",
+            return_value=subprocess.CompletedProcess(
+                args=["docker", "compose", "config", "--services"],
+                returncode=0,
+                stdout="clickhouse\n",
+            ),
+        ) as mock_run:
+            up(args)
+
+            mock_run.assert_has_calls(
+                [
+                    mock.call(
+                        [
+                            "docker",
+                            "compose",
+                            "-p",
+                            "example-service",
+                            "-f",
+                            f"{service_path}/{DEVSERVICES_DIR_NAME}/{CONFIG_FILE_NAME}",
+                            "down",
+                            "clickhouse",
+                        ],
+                        mock.ANY,
+                    ),
+                    mock.call(
+                        [
+                            "docker",
+                            "compose",
+                            "-p",
+                            "example-service",
+                            "-f",
+                            f"{service_path}/{DEVSERVICES_DIR_NAME}/{CONFIG_FILE_NAME}",
+                            "up",
+                            "clickhouse",
+                            "-d",
+                        ],
+                        mock.ANY,
+                    ),
+                ],
+            )
+
+        captured = capsys.readouterr()
+        assert (
+            "Service 'example-service' is already running in mode: 'default', restarting in mode: 'test'"
+            in captured.out.strip()
+        )
+        assert "Starting 'example-service' in mode: 'test'" in captured.out.strip()
+        assert "Retrieving dependencies" in captured.out.strip()
+        assert "Starting clickhouse" in captured.out.strip()