devservices 1.0.5__tar.gz → 1.0.7__tar.gz

{devservices-1.0.5 → devservices-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: devservices
-Version: 1.0.5
+Version: 1.0.7
 Requires-Python: >=3.10
 License-File: LICENSE.md
 Requires-Dist: pyyaml

{devservices-1.0.5 → devservices-1.0.7}/README.md

@@ -11,7 +11,7 @@ A standalone cli tool used to manage dependencies for services. It simplifies th
 The recommended way to install devservices is through a virtualenv in the requirements.txt.
 
 ```
-devservices==1.0.5
+devservices==1.0.7
 ```
 
 

{devservices-1.0.5 → devservices-1.0.7}/devservices/commands/down.py

@@ -14,7 +14,6 @@ from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
 from devservices.constants import DEVSERVICES_DIR_NAME
-from devservices.constants import DOCKER_COMPOSE_COMMAND_LENGTH
 from devservices.exceptions import ConfigError
 from devservices.exceptions import DependencyError
 from devservices.exceptions import DockerComposeError
@@ -24,6 +23,7 @@ from devservices.utils.console import Status
 from devservices.utils.dependencies import get_non_shared_remote_dependencies
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
+from devservices.utils.docker_compose import DockerComposeCommand
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
@@ -90,9 +90,14 @@ def down(args: Namespace) -> None:
             capture_exception(de)
             status.failure(str(de))
             exit(1)
-        remote_dependencies = get_non_shared_remote_dependencies(
-            service, remote_dependencies
-        )
+        try:
+            remote_dependencies = get_non_shared_remote_dependencies(
+                service, remote_dependencies
+            )
+        except DependencyError as de:
+            capture_exception(de)
+            status.failure(str(de))
+            exit(1)
         try:
             _down(service, remote_dependencies, list(mode_dependencies), status)
         except DockerComposeError as dce:
@@ -106,12 +111,12 @@ def down(args: Namespace) -> None:
 
 
 def _bring_down_dependency(
-    cmd: list[str], current_env: dict[str, str], status: Status
+    cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
 ) -> subprocess.CompletedProcess[str]:
     # TODO: Get rid of these constants, we need a smarter way to determine the containers being brought down
-    for dependency in cmd[DOCKER_COMPOSE_COMMAND_LENGTH:]:
+    for dependency in cmd.services:
         status.info(f"Stopping {dependency}")
-    return run_cmd(cmd, current_env)
+    return run_cmd(cmd.full_command, current_env)
 
 
 def _down(

{devservices-1.0.5 → devservices-1.0.7}/devservices/commands/logs.py

@@ -114,7 +114,7 @@ def _logs(
 
     with concurrent.futures.ThreadPoolExecutor() as executor:
         futures = [
-            executor.submit(run_cmd, cmd, current_env)
+            executor.submit(run_cmd, cmd.full_command, current_env)
             for cmd in docker_compose_commands
         ]
         for future in concurrent.futures.as_completed(futures):

devservices-1.0.7/devservices/commands/purge.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import os
+import shutil
+from argparse import _SubParsersAction
+from argparse import ArgumentParser
+from argparse import Namespace
+
+from devservices.constants import DEVSERVICES_CACHE_DIR
+from devservices.constants import DEVSERVICES_ORCHESTRATOR_LABEL
+from devservices.constants import DOCKER_NETWORK_NAME
+from devservices.exceptions import DockerDaemonNotRunningError
+from devservices.exceptions import DockerError
+from devservices.utils.console import Console
+from devservices.utils.console import Status
+from devservices.utils.docker import get_matching_containers
+from devservices.utils.docker import get_matching_networks
+from devservices.utils.docker import get_volumes_for_containers
+from devservices.utils.docker import remove_docker_resources
+from devservices.utils.docker import stop_containers
+from devservices.utils.state import State
+
+
+def add_parser(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    parser = subparsers.add_parser("purge", help="Purge the local devservices cache")
+    parser.set_defaults(func=purge)
+
+
+def purge(_args: Namespace) -> None:
+    """Purge the local devservices state and cache and remove all devservices containers and volumes."""
+    console = Console()
+
+    if os.path.exists(DEVSERVICES_CACHE_DIR):
+        try:
+            shutil.rmtree(DEVSERVICES_CACHE_DIR)
+        except PermissionError as e:
+            console.failure(f"Failed to purge cache: {e}")
+            exit(1)
+    state = State()
+    state.clear_state()
+
+    try:
+        devservices_containers = get_matching_containers(DEVSERVICES_ORCHESTRATOR_LABEL)
+    except DockerDaemonNotRunningError as e:
+        console.warning(str(e))
+        return
+    except DockerError as de:
+        console.failure(f"Failed to get devservices containers {de.stderr}")
+        exit(1)
+
+    try:
+        devservices_volumes = get_volumes_for_containers(devservices_containers)
+    except DockerError as e:
+        console.failure(f"Failed to get devservices volumes {e.stderr}")
+        exit(1)
+
+    with Status(
+        lambda: console.warning("Stopping all devservices containers"),
+        lambda: console.success("All devservices containers have been stopped"),
+    ):
+        try:
+            stop_containers(devservices_containers, should_remove=True)
+        except DockerError as e:
+            console.failure(f"Failed to stop devservices containers {e.stderr}")
+            exit(1)
+
+    console.warning("Removing any devservices docker volumes")
+    if len(devservices_volumes) == 0:
+        console.success("No devservices volumes found to remove")
+    else:
+        try:
+            remove_docker_resources("volume", list(devservices_volumes))
+            console.success("All devservices volumes removed")
+        except DockerError as e:
+            # We don't want to exit here since we still want to try to remove the networks
+            console.failure(f"Failed to remove devservices volumes {e.stderr}")
+
+    console.warning("Removing any devservices networks")
+    try:
+        devservices_networks = get_matching_networks(DOCKER_NETWORK_NAME)
+    except DockerError as e:
+        console.failure(f"Failed to get devservices networks {e.stderr}")
+        exit(1)
+    if len(devservices_networks) == 0:
+        console.success("No devservices networks found to remove")
+    else:
+        try:
+            remove_docker_resources("network", devservices_networks)
+            console.success("All devservices networks removed")
+        except DockerError as e:
+            console.failure(f"Failed to remove devservices networks {e.stderr}")
+            exit(1)
+
+    console.success("The local devservices cache and state has been purged")

{devservices-1.0.5 → devservices-1.0.7}/devservices/commands/status.py

@@ -152,7 +152,7 @@ def _status(
 
     with concurrent.futures.ThreadPoolExecutor() as executor:
         futures = [
-            executor.submit(run_cmd, cmd, current_env)
+            executor.submit(run_cmd, cmd.full_command, current_env)
             for cmd in docker_compose_commands
         ]
         for future in concurrent.futures.as_completed(futures):

{devservices-1.0.5 → devservices-1.0.7}/devservices/commands/up.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import concurrent.futures
 import os
 import subprocess
 from argparse import _SubParsersAction
@@ -13,8 +14,8 @@ from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
 from devservices.constants import DEVSERVICES_DIR_NAME
-from devservices.constants import DOCKER_COMPOSE_COMMAND_LENGTH
 from devservices.exceptions import ConfigError
+from devservices.exceptions import ContainerHealthcheckFailedError
 from devservices.exceptions import DependencyError
 from devservices.exceptions import DockerComposeError
 from devservices.exceptions import ModeDoesNotExistError
@@ -24,6 +25,9 @@ from devservices.utils.console import Status
 from devservices.utils.dependencies import construct_dependency_graph
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
+from devservices.utils.docker import check_all_containers_healthy
+from devservices.utils.docker_compose import DockerComposeCommand
+from devservices.utils.docker_compose import get_container_names_for_project
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
@@ -101,12 +105,12 @@ def up(args: Namespace) -> None:
 
 
 def _bring_up_dependency(
-    cmd: list[str], current_env: dict[str, str], status: Status, len_options: int
+    cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
 ) -> subprocess.CompletedProcess[str]:
     # TODO: Get rid of these constants, we need a smarter way to determine the containers being brought up
-    for dependency in cmd[DOCKER_COMPOSE_COMMAND_LENGTH:-len_options]:
+    for dependency in cmd.services:
         status.info(f"Starting {dependency}")
-    return run_cmd(cmd, current_env)
+    return run_cmd(cmd.full_command, current_env)
 
 
 def _up(
@@ -128,7 +132,7 @@ def _up(
     current_env[
         DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
     ] = relative_local_dependency_directory
-    options = ["-d"]
+    options = ["-d", "--pull", "always"]
     dependency_graph = construct_dependency_graph(service, modes=modes)
     starting_order = dependency_graph.get_starting_order()
     sorted_remote_dependencies = sorted(
@@ -144,8 +148,31 @@ def _up(
         mode_dependencies=mode_dependencies,
     )
 
+    containers_to_check = []
+    with concurrent.futures.ThreadPoolExecutor() as dependency_executor:
+        futures = [
+            dependency_executor.submit(_bring_up_dependency, cmd, current_env, status)
+            for cmd in docker_compose_commands
+        ]
+        for future in concurrent.futures.as_completed(futures):
+            _ = future.result()
+
     for cmd in docker_compose_commands:
-        _bring_up_dependency(cmd, current_env, status, len(options))
+        try:
+            container_names = get_container_names_for_project(
+                cmd.project_name, cmd.config_path
+            )
+            containers_to_check.extend(container_names)
+        except DockerComposeError as dce:
+            status.failure(
+                f"Failed to get containers to healthcheck for {cmd.project_name}: {dce.stderr}"
+            )
+            exit(1)
+    try:
+        check_all_containers_healthy(status, containers_to_check)
+    except ContainerHealthcheckFailedError as e:
+        status.failure(str(e))
+        exit(1)
 
 
 def _create_devservices_network() -> None:

{devservices-1.0.5 → devservices-1.0.7}/devservices/commands/update.py

@@ -7,10 +7,10 @@ from argparse import ArgumentParser
 from argparse import Namespace
 from importlib import metadata
 
-from devservices.commands.check_for_update import check_for_update
 from devservices.constants import DEVSERVICES_DOWNLOAD_URL
 from devservices.exceptions import BinaryInstallError
 from devservices.exceptions import DevservicesUpdateError
+from devservices.utils.check_for_update import check_for_update
 from devservices.utils.console import Console
 from devservices.utils.install_binary import install_binary
 

{devservices-1.0.5 → devservices-1.0.7}/devservices/constants.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from datetime import timedelta
 
 MINIMUM_DOCKER_COMPOSE_VERSION = "2.29.7"
 DEVSERVICES_DIR_NAME = "devservices"
@@ -12,7 +13,7 @@ DEVSERVICES_LOCAL_DIR = os.path.expanduser("~/.local/share/sentry-devservices")
 DEVSERVICES_DEPENDENCIES_CACHE_DIR = os.path.join(DEVSERVICES_CACHE_DIR, "dependencies")
 DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY = "DEVSERVICES_DEPENDENCIES_CACHE_DIR"
 STATE_DB_FILE = os.path.join(DEVSERVICES_LOCAL_DIR, "state")
-DOCKER_COMPOSE_COMMAND_LENGTH = 7
+DEVSERVICES_ORCHESTRATOR_LABEL = "orchestrator=devservices"
 
 DEPENDENCY_CONFIG_VERSION = "v1"
 DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS = {
@@ -21,9 +22,20 @@ DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS = {
     "core.sparseCheckout": "true",
 }
 
+DEVSERVICES_RELEASES_URL = (
+    "https://api.github.com/repos/getsentry/devservices/releases/latest"
+)
 DOCKER_COMPOSE_DOWNLOAD_URL = "https://github.com/docker/compose/releases/download"
 DEVSERVICES_DOWNLOAD_URL = "https://github.com/getsentry/devservices/releases/download"
 BINARY_PERMISSIONS = 0o755
 MAX_LOG_LINES = "100"
 LOGGER_NAME = "devservices"
 DOCKER_NETWORK_NAME = "devservices"
+
+# Latest Version Cache
+DEVSERVICES_LATEST_VERSION_CACHE_FILE = os.path.join(
+    DEVSERVICES_CACHE_DIR, "latest_version.txt"
+)
+DEVSERVICES_LATEST_VERSION_CACHE_TTL = timedelta(minutes=15)
+HEALTHCHECK_TIMEOUT = 30
+HEALTHCHECK_INTERVAL = 5

{devservices-1.0.5 → devservices-1.0.7}/devservices/exceptions.py

@@ -57,8 +57,8 @@ class DockerComposeInstallationError(BinaryInstallError):
     pass
 
 
-class DockerComposeError(Exception):
-    """Base class for Docker Compose related errors."""
+class DockerError(Exception):
+    """Base class for Docker related errors."""
 
     def __init__(self, command: str, returncode: int, stdout: str, stderr: str):
         self.command = command
@@ -67,6 +67,12 @@ class DockerComposeError(Exception):
         self.stderr = stderr
 
 
+class DockerComposeError(DockerError):
+    """Base class for Docker Compose related errors."""
+
+    pass
+
+
 class ModeDoesNotExistError(Exception):
     """Raised when a mode does not exist."""
 
@@ -121,3 +127,14 @@ class FailedToSetGitConfigError(GitConfigError):
     """Raised when a git config cannot be set."""
 
     pass
+
+
+class ContainerHealthcheckFailedError(Exception):
+    """Raised when a container is not healthy."""
+
+    def __init__(self, container_name: str, timeout: int):
+        self.container_name = container_name
+        self.timeout = timeout
+
+    def __str__(self) -> str:
+        return f"Container {self.container_name} did not become healthy within {self.timeout} seconds."

{devservices-1.0.5 → devservices-1.0.7}/devservices/main.py

@@ -23,10 +23,10 @@ from devservices.commands import purge
 from devservices.commands import status
 from devservices.commands import up
 from devservices.commands import update
-from devservices.commands.check_for_update import check_for_update
 from devservices.constants import LOGGER_NAME
 from devservices.exceptions import DockerComposeInstallationError
 from devservices.exceptions import DockerDaemonNotRunningError
+from devservices.utils.check_for_update import check_for_update
 from devservices.utils.console import Console
 from devservices.utils.docker_compose import check_docker_compose_version
 
@@ -102,7 +102,7 @@ def main() -> None:
     else:
         parser.print_help()
 
-    if args.command != "update":
+    if args.command != "update" and os.environ.get("CI") != "true":
         newest_version = check_for_update()
         if newest_version != current_version:
             console.warning(

devservices-1.0.7/devservices/utils/check_for_update.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime
+from datetime import timedelta
+from urllib.request import urlopen
+
+from devservices.constants import DEVSERVICES_CACHE_DIR
+from devservices.constants import DEVSERVICES_LATEST_VERSION_CACHE_FILE
+from devservices.constants import DEVSERVICES_LATEST_VERSION_CACHE_TTL
+from devservices.constants import DEVSERVICES_RELEASES_URL
+
+
+def _delete_cached_version() -> None:
+    if os.path.exists(DEVSERVICES_LATEST_VERSION_CACHE_FILE):
+        os.remove(DEVSERVICES_LATEST_VERSION_CACHE_FILE)
+
+
+def _get_cache_age() -> timedelta:
+    if os.path.exists(DEVSERVICES_LATEST_VERSION_CACHE_FILE):
+        file_modification_time = datetime.fromtimestamp(
+            os.path.getmtime(DEVSERVICES_LATEST_VERSION_CACHE_FILE)
+        )
+        return datetime.now() - file_modification_time
+    return timedelta.max
+
+
+def _get_cached_version() -> str | None:
+    cache_age = _get_cache_age()
+    if cache_age < DEVSERVICES_LATEST_VERSION_CACHE_TTL:
+        with open(DEVSERVICES_LATEST_VERSION_CACHE_FILE, "r", encoding="utf-8") as f:
+            return f.read()
+    else:
+        _delete_cached_version()
+        return None
+
+
+def _set_cached_version(latest_version: str) -> None:
+    with open(DEVSERVICES_LATEST_VERSION_CACHE_FILE, "w", encoding="utf-8") as f:
+        f.write(latest_version)
+
+
+def check_for_update() -> str | None:
+    os.makedirs(DEVSERVICES_CACHE_DIR, exist_ok=True)
+
+    cached_version = _get_cached_version()
+    if cached_version is not None:
+        return cached_version
+
+    with urlopen(DEVSERVICES_RELEASES_URL) as response:
+        if response.status == 200:
+            data = json.loads(response.read())
+            latest_version = str(data["tag_name"])
+
+            _set_cached_version(latest_version)
+
+            return latest_version
+    return None

{devservices-1.0.5 → devservices-1.0.7}/devservices/utils/dependencies.py

@@ -12,6 +12,8 @@ from dataclasses import dataclass
 from typing import TextIO
 from typing import TypeGuard
 
+from sentry_sdk import set_context
+
 from devservices.configs.service_config import Dependency
 from devservices.configs.service_config import load_service_config_from_file
 from devservices.configs.service_config import RemoteConfig
@@ -36,6 +38,17 @@ from devservices.utils.services import find_matching_service
 from devservices.utils.services import Service
 from devservices.utils.state import State
 
+RELEVANT_GIT_CONFIG_KEYS = [
+    "init.defaultbranch",
+    "core.sparsecheckout",
+    "remote.origin.url",
+    "remote.origin.fetch",
+    "remote.origin.promisor",
+    "remote.origin.partialclonefilter",
+    "protocol.version",
+    "extensions.partialclone",
+]
+
 
 class DependencyGraph:
     def __init__(self) -> None:
@@ -149,6 +162,28 @@ class GitConfigManager:
         if self.sparse_pattern:
             self.sparse_checkout_manager.set_sparse_checkout(self.sparse_pattern)
 
+    def get_relevant_config(self) -> dict[str, str]:
+        """
+        Get the relevant git config entries (to avoid logging sensitive information)
+        """
+        git_config = (
+            subprocess.check_output(
+                ["git", "config", "--list"],
+                cwd=self.repo_dir,
+                stderr=subprocess.PIPE,
+            )
+            .decode()
+            .strip()
+        )
+        git_config_dict = dict()
+        for line in git_config.split("\n"):
+            if not line:
+                continue
+            key, value = line.split("=")
+            if key in RELEVANT_GIT_CONFIG_KEYS:
+                git_config_dict[key] = value
+        return git_config_dict
+
     def _set_config(self, key: str, value: str) -> None:
         """
         Set a git config option for the repo
@@ -411,12 +446,15 @@ def _update_dependency(
             repo_link=dependency.repo_link,
             branch=dependency.branch,
         ) from e
+
     try:
         _run_command(
             ["git", "fetch", "origin", dependency.branch, "--filter=blob:none"],
             cwd=dependency_repo_dir,
         )
     except subprocess.CalledProcessError as e:
+        # Try to set the git config context to help with debugging
+        _try_set_git_config_context(git_config_manager)
         raise DependencyError(
             repo_name=dependency.repo_name,
             repo_link=dependency.repo_link,
@@ -536,6 +574,17 @@ def _run_command(
     subprocess.run(cmd, cwd=cwd, check=True, stdout=stdout, stderr=subprocess.DEVNULL)
 
 
+def _try_set_git_config_context(
+    git_config_manager: GitConfigManager,
+) -> None:
+    try:
+        git_config = git_config_manager.get_relevant_config()
+        set_context("git_config", git_config)
+    except subprocess.CalledProcessError as e:
+        logger = logging.getLogger(LOGGER_NAME)
+        logger.exception(e)
+
+
 def get_remote_dependency_config(remote_config: RemoteConfig) -> ServiceConfig:
     dependency_repo_dir = os.path.join(
         DEVSERVICES_DEPENDENCIES_CACHE_DIR,