devservices 1.0.6__tar.gz → 1.0.8__tar.gz

{devservices-1.0.6 → devservices-1.0.8}/PKG-INFO

@@ -1,10 +1,12 @@
 Metadata-Version: 2.1
 Name: devservices
-Version: 1.0.6
+Version: 1.0.8
 Requires-Python: >=3.10
 License-File: LICENSE.md
 Requires-Dist: pyyaml
 Requires-Dist: sentry-devenv
+Requires-Dist: sentry-sdk
+Requires-Dist: packaging
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: mypy; extra == "dev"

{devservices-1.0.6 → devservices-1.0.8}/README.md

@@ -11,7 +11,7 @@ A standalone cli tool used to manage dependencies for services. It simplifies th
 The recommended way to install devservices is through a virtualenv in the requirements.txt.
 
 ```
-devservices==1.0.6
+devservices==1.0.8
 ```
 
 

{devservices-1.0.6 → devservices-1.0.8}/devservices/commands/down.py

@@ -14,7 +14,6 @@ from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
 from devservices.constants import DEVSERVICES_DIR_NAME
-from devservices.constants import DOCKER_COMPOSE_COMMAND_LENGTH
 from devservices.exceptions import ConfigError
 from devservices.exceptions import DependencyError
 from devservices.exceptions import DockerComposeError
@@ -24,6 +23,7 @@ from devservices.utils.console import Status
 from devservices.utils.dependencies import get_non_shared_remote_dependencies
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
+from devservices.utils.docker_compose import DockerComposeCommand
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
@@ -111,12 +111,12 @@ def down(args: Namespace) -> None:
 
 
 def _bring_down_dependency(
-    cmd: list[str], current_env: dict[str, str], status: Status
+    cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
 ) -> subprocess.CompletedProcess[str]:
     # TODO: Get rid of these constants, we need a smarter way to determine the containers being brought down
-    for dependency in cmd[DOCKER_COMPOSE_COMMAND_LENGTH:]:
+    for dependency in cmd.services:
         status.info(f"Stopping {dependency}")
-    return run_cmd(cmd, current_env)
+    return run_cmd(cmd.full_command, current_env)
 
 
 def _down(

{devservices-1.0.6 → devservices-1.0.8}/devservices/commands/logs.py

@@ -114,7 +114,7 @@ def _logs(
 
     with concurrent.futures.ThreadPoolExecutor() as executor:
         futures = [
-            executor.submit(run_cmd, cmd, current_env)
+            executor.submit(run_cmd, cmd.full_command, current_env)
             for cmd in docker_compose_commands
         ]
         for future in concurrent.futures.as_completed(futures):

devservices-1.0.8/devservices/commands/purge.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import os
+import shutil
+from argparse import _SubParsersAction
+from argparse import ArgumentParser
+from argparse import Namespace
+
+from devservices.constants import DEVSERVICES_CACHE_DIR
+from devservices.constants import DEVSERVICES_ORCHESTRATOR_LABEL
+from devservices.constants import DOCKER_NETWORK_NAME
+from devservices.exceptions import DockerDaemonNotRunningError
+from devservices.exceptions import DockerError
+from devservices.utils.console import Console
+from devservices.utils.console import Status
+from devservices.utils.docker import get_matching_containers
+from devservices.utils.docker import get_matching_networks
+from devservices.utils.docker import get_volumes_for_containers
+from devservices.utils.docker import remove_docker_resources
+from devservices.utils.docker import stop_containers
+from devservices.utils.state import State
+
+
+def add_parser(subparsers: _SubParsersAction[ArgumentParser]) -> None:
+    parser = subparsers.add_parser("purge", help="Purge the local devservices cache")
+    parser.set_defaults(func=purge)
+
+
+def purge(_args: Namespace) -> None:
+    """Purge the local devservices state and cache and remove all devservices containers and volumes."""
+    console = Console()
+
+    if os.path.exists(DEVSERVICES_CACHE_DIR):
+        try:
+            shutil.rmtree(DEVSERVICES_CACHE_DIR)
+        except PermissionError as e:
+            console.failure(f"Failed to purge cache: {e}")
+            exit(1)
+    state = State()
+    state.clear_state()
+
+    try:
+        devservices_containers = get_matching_containers(DEVSERVICES_ORCHESTRATOR_LABEL)
+    except DockerDaemonNotRunningError as e:
+        console.warning(str(e))
+        return
+    except DockerError as de:
+        console.failure(f"Failed to get devservices containers {de.stderr}")
+        exit(1)
+
+    try:
+        devservices_volumes = get_volumes_for_containers(devservices_containers)
+    except DockerError as e:
+        console.failure(f"Failed to get devservices volumes {e.stderr}")
+        exit(1)
+
+    with Status(
+        lambda: console.warning("Stopping all devservices containers"),
+        lambda: console.success("All devservices containers have been stopped"),
+    ):
+        try:
+            stop_containers(devservices_containers, should_remove=True)
+        except DockerError as e:
+            console.failure(f"Failed to stop devservices containers {e.stderr}")
+            exit(1)
+
+    console.warning("Removing any devservices docker volumes")
+    if len(devservices_volumes) == 0:
+        console.success("No devservices volumes found to remove")
+    else:
+        try:
+            remove_docker_resources("volume", list(devservices_volumes))
+            console.success("All devservices volumes removed")
+        except DockerError as e:
+            # We don't want to exit here since we still want to try to remove the networks
+            console.failure(f"Failed to remove devservices volumes {e.stderr}")
+
+    console.warning("Removing any devservices networks")
+    try:
+        devservices_networks = get_matching_networks(DOCKER_NETWORK_NAME)
+    except DockerError as e:
+        console.failure(f"Failed to get devservices networks {e.stderr}")
+        exit(1)
+    if len(devservices_networks) == 0:
+        console.success("No devservices networks found to remove")
+    else:
+        try:
+            remove_docker_resources("network", devservices_networks)
+            console.success("All devservices networks removed")
+        except DockerError as e:
+            console.failure(f"Failed to remove devservices networks {e.stderr}")
+            exit(1)
+
+    console.success("The local devservices cache and state has been purged")

{devservices-1.0.6 → devservices-1.0.8}/devservices/commands/status.py

@@ -7,6 +7,7 @@ import subprocess
 from argparse import _SubParsersAction
 from argparse import ArgumentParser
 from argparse import Namespace
+from collections import namedtuple
 
 from sentry_sdk import capture_exception
 
@@ -30,6 +31,9 @@ from devservices.utils.services import Service
 LINE_LENGTH = 40
 
 
+ServiceStatus = namedtuple("ServiceStatus", ["name", "formatted_output"])
+
+
 def add_parser(subparsers: _SubParsersAction[ArgumentParser]) -> None:
     parser = subparsers.add_parser("status", help="View status of a service")
     parser.add_argument(
@@ -41,12 +45,12 @@ def add_parser(subparsers: _SubParsersAction[ArgumentParser]) -> None:
     parser.set_defaults(func=status)
 
 
-def format_status_output(status_json: str) -> str:
+def format_status_output(service_status_json: str) -> list[ServiceStatus]:
     # Docker compose ps is line delimited json, so this constructs this into an array we can use
-    service_statuses = status_json.split("\n")[:-1]
-    output = []
-    output.append("-" * LINE_LENGTH)
+    service_statuses = service_status_json.split("\n")[:-1]
+    outputs = []
     for service_status in service_statuses:
+        output = []
         service = json.loads(service_status)
         name = service["Service"]
         state = service["State"]
@@ -71,8 +75,9 @@ def format_status_output(status_json: str) -> str:
             output.append("No ports exposed")
 
         output.append("")  # Empty line for readability
+        outputs.append(ServiceStatus(name=name, formatted_output="\n".join(output)))
 
-    return "\n".join(output)
+    return outputs
 
 
 def status(args: Namespace) -> None:
@@ -115,10 +120,15 @@ def status(args: Namespace) -> None:
         console.warning(f"{service.name} is not running")
         return
     output = f"Service: {service.name}\n\n"
+    output += "=" * LINE_LENGTH + "\n"
+    formatted_status_outputs = []
     for status_json in status_json_results:
-        output += format_status_output(status_json.stdout)
-    output += "=" * LINE_LENGTH
-    console.info(output + "\n")
+        formatted_status_outputs.extend(format_status_output(status_json.stdout))
+    formatted_status_outputs.sort(key=lambda x: x.name)
+    for formatted_status_output in formatted_status_outputs:
+        output += formatted_status_output[1]
+        output += "-" * LINE_LENGTH + "\n"
+    console.info(output)
 
 
 def _status(
@@ -152,7 +162,7 @@ def _status(
 
     with concurrent.futures.ThreadPoolExecutor() as executor:
         futures = [
-            executor.submit(run_cmd, cmd, current_env)
+            executor.submit(run_cmd, cmd.full_command, current_env)
             for cmd in docker_compose_commands
         ]
         for future in concurrent.futures.as_completed(futures):

{devservices-1.0.6 → devservices-1.0.8}/devservices/commands/up.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import concurrent.futures
 import os
 import subprocess
 from argparse import _SubParsersAction
@@ -13,8 +14,8 @@ from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
 from devservices.constants import DEVSERVICES_DIR_NAME
-from devservices.constants import DOCKER_COMPOSE_COMMAND_LENGTH
 from devservices.exceptions import ConfigError
+from devservices.exceptions import ContainerHealthcheckFailedError
 from devservices.exceptions import DependencyError
 from devservices.exceptions import DockerComposeError
 from devservices.exceptions import ModeDoesNotExistError
@@ -24,6 +25,9 @@ from devservices.utils.console import Status
 from devservices.utils.dependencies import construct_dependency_graph
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
+from devservices.utils.docker import check_all_containers_healthy
+from devservices.utils.docker_compose import DockerComposeCommand
+from devservices.utils.docker_compose import get_container_names_for_project
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
@@ -101,12 +105,11 @@ def up(args: Namespace) -> None:
 
 
 def _bring_up_dependency(
-    cmd: list[str], current_env: dict[str, str], status: Status, len_options: int
+    cmd: DockerComposeCommand, current_env: dict[str, str], status: Status
 ) -> subprocess.CompletedProcess[str]:
-    # TODO: Get rid of these constants, we need a smarter way to determine the containers being brought up
-    for dependency in cmd[DOCKER_COMPOSE_COMMAND_LENGTH:-len_options]:
+    for dependency in cmd.services:
         status.info(f"Starting {dependency}")
-    return run_cmd(cmd, current_env)
+    return run_cmd(cmd.full_command, current_env)
 
 
 def _up(
@@ -128,7 +131,6 @@ def _up(
     current_env[
         DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
     ] = relative_local_dependency_directory
-    options = ["-d"]
     dependency_graph = construct_dependency_graph(service, modes=modes)
     starting_order = dependency_graph.get_starting_order()
     sorted_remote_dependencies = sorted(
@@ -139,13 +141,36 @@ def _up(
         remote_dependencies=sorted_remote_dependencies,
         current_env=current_env,
         command="up",
-        options=options,
+        options=["-d", "--pull", "always"],
         service_config_file_path=service_config_file_path,
         mode_dependencies=mode_dependencies,
     )
 
+    containers_to_check = []
+    with concurrent.futures.ThreadPoolExecutor() as dependency_executor:
+        futures = [
+            dependency_executor.submit(_bring_up_dependency, cmd, current_env, status)
+            for cmd in docker_compose_commands
+        ]
+        for future in concurrent.futures.as_completed(futures):
+            _ = future.result()
+
     for cmd in docker_compose_commands:
-        _bring_up_dependency(cmd, current_env, status, len(options))
+        try:
+            container_names = get_container_names_for_project(
+                cmd.project_name, cmd.config_path
+            )
+            containers_to_check.extend(container_names)
+        except DockerComposeError as dce:
+            status.failure(
+                f"Failed to get containers to healthcheck for {cmd.project_name}: {dce.stderr}"
+            )
+            exit(1)
+    try:
+        check_all_containers_healthy(status, containers_to_check)
+    except ContainerHealthcheckFailedError as e:
+        status.failure(str(e))
+        exit(1)
 
 
 def _create_devservices_network() -> None:
@@ -153,4 +178,5 @@ def _create_devservices_network() -> None:
         ["docker", "network", "create", "devservices"],
         stdout=subprocess.DEVNULL,
         stderr=subprocess.DEVNULL,
+        check=True,
     )

{devservices-1.0.6 → devservices-1.0.8}/devservices/configs/service_config.py

@@ -68,7 +68,7 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig:
     config_path = os.path.join(repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME)
     if not os.path.exists(config_path):
         raise ConfigNotFoundError(f"Config file not found in directory: {config_path}")
-    with open(config_path, "r") as stream:
+    with open(config_path, "r", encoding="utf-8") as stream:
         try:
             config = yaml.safe_load(stream)
         except yaml.YAMLError as yml_error:

{devservices-1.0.6 → devservices-1.0.8}/devservices/constants.py

@@ -14,7 +14,6 @@ DEVSERVICES_DEPENDENCIES_CACHE_DIR = os.path.join(DEVSERVICES_CACHE_DIR, "depend
 DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY = "DEVSERVICES_DEPENDENCIES_CACHE_DIR"
 STATE_DB_FILE = os.path.join(DEVSERVICES_LOCAL_DIR, "state")
 DEVSERVICES_ORCHESTRATOR_LABEL = "orchestrator=devservices"
-DOCKER_COMPOSE_COMMAND_LENGTH = 7
 
 DEPENDENCY_CONFIG_VERSION = "v1"
 DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS = {
@@ -38,3 +37,5 @@ DEVSERVICES_LATEST_VERSION_CACHE_FILE = os.path.join(
     DEVSERVICES_CACHE_DIR, "latest_version.txt"
 )
 DEVSERVICES_LATEST_VERSION_CACHE_TTL = timedelta(minutes=15)
+HEALTHCHECK_TIMEOUT = 45
+HEALTHCHECK_INTERVAL = 5

{devservices-1.0.6 → devservices-1.0.8}/devservices/exceptions.py

@@ -127,3 +127,14 @@ class FailedToSetGitConfigError(GitConfigError):
     """Raised when a git config cannot be set."""
 
     pass
+
+
+class ContainerHealthcheckFailedError(Exception):
+    """Raised when a container is not healthy."""
+
+    def __init__(self, container_name: str, timeout: int):
+        self.container_name = container_name
+        self.timeout = timeout
+
+    def __str__(self) -> str:
+        return f"Container {self.container_name} did not become healthy within {self.timeout} seconds."

{devservices-1.0.6 → devservices-1.0.8}/devservices/main.py

@@ -26,7 +26,6 @@ from devservices.commands import update
 from devservices.constants import LOGGER_NAME
 from devservices.exceptions import DockerComposeInstallationError
 from devservices.exceptions import DockerDaemonNotRunningError
-from devservices.utils.check_for_update import check_for_update
 from devservices.utils.console import Console
 from devservices.utils.docker_compose import check_docker_compose_version
 
@@ -102,14 +101,6 @@ def main() -> None:
     else:
         parser.print_help()
 
-    if args.command != "update" and os.environ.get("CI") != "true":
-        newest_version = check_for_update()
-        if newest_version != current_version:
-            console.warning(
-                f"WARNING: A new version of devservices is available: {newest_version}"
-            )
-            console.warning('To update, run: "devservices update"')
-
 
 if __name__ == "__main__":
     main()

{devservices-1.0.6 → devservices-1.0.8}/devservices/utils/dependencies.py

@@ -12,6 +12,8 @@ from dataclasses import dataclass
 from typing import TextIO
 from typing import TypeGuard
 
+from sentry_sdk import set_context
+
 from devservices.configs.service_config import Dependency
 from devservices.configs.service_config import load_service_config_from_file
 from devservices.configs.service_config import RemoteConfig
@@ -36,6 +38,17 @@ from devservices.utils.services import find_matching_service
 from devservices.utils.services import Service
 from devservices.utils.state import State
 
+RELEVANT_GIT_CONFIG_KEYS = [
+    "init.defaultbranch",
+    "core.sparsecheckout",
+    "remote.origin.url",
+    "remote.origin.fetch",
+    "remote.origin.promisor",
+    "remote.origin.partialclonefilter",
+    "protocol.version",
+    "extensions.partialclone",
+]
+
 
 class DependencyGraph:
     def __init__(self) -> None:
@@ -149,6 +162,28 @@ class GitConfigManager:
         if self.sparse_pattern:
             self.sparse_checkout_manager.set_sparse_checkout(self.sparse_pattern)
 
+    def get_relevant_config(self) -> dict[str, str]:
+        """
+        Get the relevant git config entries (to avoid logging sensitive information)
+        """
+        git_config = (
+            subprocess.check_output(
+                ["git", "config", "--list"],
+                cwd=self.repo_dir,
+                stderr=subprocess.PIPE,
+            )
+            .decode()
+            .strip()
+        )
+        git_config_dict = dict()
+        for line in git_config.split("\n"):
+            if not line:
+                continue
+            key, value = line.split("=")
+            if key in RELEVANT_GIT_CONFIG_KEYS:
+                git_config_dict[key] = value
+        return git_config_dict
+
     def _set_config(self, key: str, value: str) -> None:
         """
         Set a git config option for the repo
@@ -411,12 +446,15 @@ def _update_dependency(
             repo_link=dependency.repo_link,
             branch=dependency.branch,
         ) from e
+
     try:
         _run_command(
             ["git", "fetch", "origin", dependency.branch, "--filter=blob:none"],
             cwd=dependency_repo_dir,
         )
     except subprocess.CalledProcessError as e:
+        # Try to set the git config context to help with debugging
+        _try_set_git_config_context(git_config_manager)
         raise DependencyError(
             repo_name=dependency.repo_name,
             repo_link=dependency.repo_link,
@@ -536,6 +574,17 @@ def _run_command(
     subprocess.run(cmd, cwd=cwd, check=True, stdout=stdout, stderr=subprocess.DEVNULL)
 
 
+def _try_set_git_config_context(
+    git_config_manager: GitConfigManager,
+) -> None:
+    try:
+        git_config = git_config_manager.get_relevant_config()
+        set_context("git_config", git_config)
+    except subprocess.CalledProcessError as e:
+        logger = logging.getLogger(LOGGER_NAME)
+        logger.exception(e)
+
+
 def get_remote_dependency_config(remote_config: RemoteConfig) -> ServiceConfig:
     dependency_repo_dir = os.path.join(
         DEVSERVICES_DEPENDENCIES_CACHE_DIR,