devservices 1.5.0__tar.gz → 1.5.1__tar.gz

{devservices-1.5.0 → devservices-1.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devservices
-Version: 1.5.0
+Version: 1.5.1
 Requires-Python: >=3.11
 License-File: LICENSE.md
 Requires-Dist: pyyaml

{devservices-1.5.0 → devservices-1.5.1}/README.md

@@ -33,7 +33,7 @@ NOTE: service-name is an optional parameter. If not provided, devservices will a
 The recommended way to install devservices is through a virtualenv in the requirements.txt. Once that is installed and a devservices config file is added, you should be able to run `devservices up` to begin local development.
 
 ```
-devservices==1.5.0
+devservices==1.5.1
 ```
 
 ### 2. Add devservices config files

{devservices-1.5.0 → devservices-1.5.1}/devservices/commands/up.py

@@ -30,6 +30,7 @@ from devservices.exceptions import SupervisorError
 from devservices.utils.console import Console
 from devservices.utils.console import Status
 from devservices.utils.dependencies import DependencyNode
+from devservices.utils.dependencies import DependencyUpdateMode
 from devservices.utils.dependencies import InstalledRemoteDependency
 from devservices.utils.dependencies import construct_dependency_graph
 from devservices.utils.dependencies import install_and_verify_dependencies
@@ -68,6 +69,12 @@ def add_parser(subparsers: _SubParsersAction[ArgumentParser]) -> None:
         action="store_true",
         default=False,
     )
+    parser.add_argument(
+        "--offline",
+        help="Use cached dependencies and images without accessing the network",
+        action="store_true",
+        default=False,
+    )
     parser.set_defaults(func=up)
 
 
@@ -96,6 +103,7 @@ def up(args: Namespace, existing_status: Status | None = None) -> None:
     modes = service.config.modes
     mode = args.mode
     exclude_local = getattr(args, "exclude_local", False)
+    offline = getattr(args, "offline", False)
 
     sentry_logger.info(
         "Starting service",
@@ -162,7 +170,9 @@ def up(args: Namespace, existing_status: Status | None = None) -> None:
             },
         )
 
-        remote_dependencies = _install_service_dependencies(service, mode, status)
+        remote_dependencies = _install_service_dependencies(
+            service, mode, status, offline
+        )
         _create_devservices_network()
         # Add the service to the starting services table
         state.update_service_entry(service.name, mode, StateTables.STARTING_SERVICES)
@@ -204,6 +214,7 @@ def up(args: Namespace, existing_status: Status | None = None) -> None:
                         service_name=local_runtime_dependency_name,
                         mode="default",  # We intentionally don't use the mode from the parent command here
                         exclude_local=True,  # TODO: This should be False (or maybe whatever the parent command is set to)
+                        offline=offline,
                     ),
                     status,
                 )
@@ -216,7 +227,12 @@ def up(args: Namespace, existing_status: Status | None = None) -> None:
             span.set_data("exclude_local", exclude_local)
             try:
                 bring_up_docker_compose_services(
-                    service, [mode], remote_dependencies, mode_dependencies, status
+                    service,
+                    [mode],
+                    remote_dependencies,
+                    mode_dependencies,
+                    status,
+                    offline,
                 )
             except DockerComposeError as dce:
                 capture_exception(dce, level="info")
@@ -239,25 +255,39 @@ def up(args: Namespace, existing_status: Status | None = None) -> None:
 
 
 def _install_service_dependencies(
-    service: Service, mode: str, status: Status
+    service: Service, mode: str, status: Status, offline: bool = False
 ) -> set[InstalledRemoteDependency]:
     with start_span(
         op="service.dependencies.install", name="Install dependencies"
     ) as span:
-        status.info("Retrieving dependencies")
+        status.info(
+            "Using cached dependencies" if offline else "Retrieving dependencies"
+        )
         span.set_data("service_name", service.name)
         span.set_data("mode", mode)
+        span.set_data("offline", offline)
+
+        update_mode = (
+            DependencyUpdateMode.OFFLINE if offline else DependencyUpdateMode.FORCE
+        )
         try:
             remote_dependencies = install_and_verify_dependencies(
-                service, force_update_dependencies=True, modes=[mode]
+                service,
+                update_mode,
+                modes=[mode],
             )
             span.set_data("remote_dependency_count", len(remote_dependencies))
             return remote_dependencies
         except DependencyError as de:
             capture_exception(de)
-            status.failure(
-                f"{str(de)}. If this error persists, try running `devservices purge`"
-            )
+            if offline:
+                status.failure(
+                    f"{str(de)}. Run `devservices up` without `--offline` to fetch and cache dependencies"
+                )
+            else:
+                status.failure(
+                    f"{str(de)}. If this error persists, try running `devservices purge`"
+                )
             exit(1)
         except ModeDoesNotExistError as mde:
             status.failure(str(mde))
@@ -310,6 +340,7 @@ def bring_up_docker_compose_services(
     remote_dependencies: set[InstalledRemoteDependency],
     mode_dependencies: list[str],
     status: Status,
+    offline: bool = False,
 ) -> None:
     relative_local_dependency_directory = os.path.relpath(
         os.path.join(DEVSERVICES_DEPENDENCIES_CACHE_DIR, DEPENDENCY_CONFIG_VERSION),
@@ -334,35 +365,41 @@ def bring_up_docker_compose_services(
         ),
     )
 
-    # Pull all images in parallel
-    status.info("Pulling images")
-    pull_commands = get_docker_compose_commands_to_run(
-        service=service,
-        remote_dependencies=sorted_remote_dependencies,
-        current_env=current_env,
-        command="pull",
-        options=[],
-        service_config_file_path=service_config_file_path,
-        mode_dependencies=mode_dependencies,
-    )
-
-    with concurrent.futures.ThreadPoolExecutor() as pull_dependency_executor:
-        futures = [
-            pull_dependency_executor.submit(
-                _pull_dependency_images, cmd, current_env, status
-            )
-            for cmd in pull_commands
-        ]
-        for future in concurrent.futures.as_completed(futures):
-            _ = future.result()
+    if offline:
+        status.info("Skipping image pull (offline mode)")
+    else:
+        # Pull all images in parallel
+        status.info("Pulling images")
+        pull_commands = get_docker_compose_commands_to_run(
+            service=service,
+            remote_dependencies=sorted_remote_dependencies,
+            current_env=current_env,
+            command="pull",
+            options=[],
+            service_config_file_path=service_config_file_path,
+            mode_dependencies=mode_dependencies,
+        )
 
-    # Bring up all necessary containers
+        with concurrent.futures.ThreadPoolExecutor() as pull_dependency_executor:
+            futures = [
+                pull_dependency_executor.submit(
+                    _pull_dependency_images, cmd, current_env, status
+                )
+                for cmd in pull_commands
+            ]
+            for future in concurrent.futures.as_completed(futures):
+                _ = future.result()
+
+    # Bring up all necessary containers. In offline mode, never pull missing images.
+    up_options = ["-d"]
+    if offline:
+        up_options += ["--pull", "never"]
     up_commands = get_docker_compose_commands_to_run(
         service=service,
         remote_dependencies=sorted_remote_dependencies,
         current_env=current_env,
         command="up",
-        options=["-d"],
+        options=up_options,
         service_config_file_path=service_config_file_path,
         mode_dependencies=mode_dependencies,
     )

{devservices-1.5.0 → devservices-1.5.1}/devservices/utils/dependencies.py

@@ -10,6 +10,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import as_completed
 from dataclasses import dataclass
+from enum import StrEnum
 from typing import TypeGuard
 
 from sentry_sdk import logger as sentry_logger
@@ -113,13 +114,29 @@ class InstalledRemoteDependency:
     mode: str = "default"
 
 
+class DependencyUpdateMode(StrEnum):
+    """Controls how remote dependencies are resolved during installation.
+
+    FORCE always fetches from the network, IF_NEEDED uses the cache when valid and
+    otherwise fetches, and OFFLINE never fetches and raises if the cache is
+    incomplete.
+    """
+
+    FORCE = "force"
+    IF_NEEDED = "if_needed"
+    OFFLINE = "offline"
+
+
 def install_and_verify_dependencies(
     service: Service,
-    force_update_dependencies: bool = False,
+    update_mode: DependencyUpdateMode = DependencyUpdateMode.IF_NEEDED,
     modes: list[str] | None = None,
 ) -> set[InstalledRemoteDependency]:
     """
-    Install and verify dependencies for a service
+    Install and verify dependencies for a service.
+
+    update_mode controls whether dependencies are fetched from the network. See
+    DependencyUpdateMode.
     """
     if modes is None:
         modes = ["default"]
@@ -138,18 +155,18 @@ def install_and_verify_dependencies(
         if dependency_key in mode_dependencies
     ]
 
-    if force_update_dependencies:
-        remote_dependencies = install_dependencies(matching_dependencies)
+    skip_dependency_install = update_mode is DependencyUpdateMode.OFFLINE or (
+        update_mode is DependencyUpdateMode.IF_NEEDED
+        and verify_local_dependencies(matching_dependencies)
+    )
+
+    if skip_dependency_install:
+        remote_dependencies = get_installed_remote_dependencies(matching_dependencies)
     else:
-        are_dependencies_valid = verify_local_dependencies(matching_dependencies)
-        if not are_dependencies_valid:
-            # TODO: Figure out how to handle this case as installing dependencies may not be the right thing to do
-            #       since the dependencies may have changed since the service was started.
-            remote_dependencies = install_dependencies(matching_dependencies)
-        else:
-            remote_dependencies = get_installed_remote_dependencies(
-                matching_dependencies
-            )
+        # TODO: Figure out how to handle this case as installing dependencies may not be the right thing to do
+        #       since the dependencies may have changed since the service was started.
+        remote_dependencies = install_dependencies(matching_dependencies)
+
     return remote_dependencies
 
 
@@ -275,9 +292,10 @@ def get_installed_remote_dependencies(
                 mode=remote_config.mode,
             )
         )
-        nested_remote_configs = _get_remote_configs(
-            list(service_config.dependencies.values())
+        active_nested_dependencies = _get_active_nested_dependencies(
+            service_config, remote_config.mode
         )
+        nested_remote_configs = _get_remote_configs(active_nested_dependencies)
         remote_configs.extend(nested_remote_configs)
 
     return installed_dependencies
@@ -359,18 +377,9 @@ def install_dependency(dependency: RemoteConfig) -> set[InstalledRemoteDependenc
                 branch=dependency.branch,
             ) from e
 
-    if dependency.mode not in installed_config.modes:
-        raise ModeDoesNotExistError(
-            service_name=installed_config.service_name,
-            mode=dependency.mode,
-            available_modes=list(installed_config.modes.keys()),
-        )
-
-    active_nested_dependencies = [
-        nested_dependency
-        for nested_dependency_name, nested_dependency in installed_config.dependencies.items()
-        if nested_dependency_name in installed_config.modes[dependency.mode]
-    ]
+    active_nested_dependencies = _get_active_nested_dependencies(
+        installed_config, dependency.mode
+    )
     nested_remote_configs = _get_remote_configs(active_nested_dependencies)
 
     installed_dependencies: set[InstalledRemoteDependency] = set(
@@ -484,6 +493,22 @@ def _fetch_dependency(
         ) from e
 
 
+def _get_active_nested_dependencies(
+    service_config: ServiceConfig, mode: str
+) -> list[Dependency]:
+    if mode not in service_config.modes:
+        raise ModeDoesNotExistError(
+            service_name=service_config.service_name,
+            mode=mode,
+            available_modes=list(service_config.modes.keys()),
+        )
+    return [
+        dependency
+        for dependency_name, dependency in service_config.dependencies.items()
+        if dependency_name in service_config.modes[mode]
+    ]
+
+
 def _get_remote_configs(dependencies: list[Dependency]) -> list[RemoteConfig]:
     return [
         dependency.remote

{devservices-1.5.0 → devservices-1.5.1}/devservices.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devservices
-Version: 1.5.0
+Version: 1.5.1
 Requires-Python: >=3.11
 License-File: LICENSE.md
 Requires-Dist: pyyaml

{devservices-1.5.0 → devservices-1.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "devservices"
-version = "1.5.0"
+version = "1.5.1"
 # 3.11 is just for internal pypi compat
 # but we test/support on 3.13
 requires-python = ">=3.11"

{devservices-1.5.0 → devservices-1.5.1}/tests/utils/test_dependencies.py

@@ -316,6 +316,87 @@ def test_get_installed_remote_dependencies_single_dep_installed(tmp_path: Path)
         }
 
 
+def test_get_installed_remote_dependencies_does_not_return_unnecessary_dependencies(
+    tmp_path: Path,
+) -> None:
+    """Cached traversal only returns nested deps in the active mode."""
+    with mock.patch(
+        "devservices.utils.dependencies.DEVSERVICES_DEPENDENCIES_CACHE_DIR",
+        str(tmp_path / "dependency-dir"),
+    ):
+        repo_a_config = {
+            "x-sentry-service-config": {
+                "version": 0.1,
+                "service_name": "repo-a",
+                "dependencies": {
+                    "repo-b": {
+                        "description": "nested dependency",
+                        "remote": {
+                            "repo_name": "repo-b",
+                            "repo_link": "https://github.com/getsentry/repo-b",
+                            "branch": "main",
+                        },
+                    },
+                    "unnecessary-repo": {
+                        "description": "unnecessary nested dependency",
+                        "remote": {
+                            "repo_name": "unnecessary-repo",
+                            "repo_link": "invalid-link",
+                            "branch": "main",
+                        },
+                    },
+                },
+                "modes": {"default": ["repo-b"], "other": ["unnecessary-repo"]},
+            },
+        }
+
+        with mock.patch(
+            "devservices.utils.dependencies.urllib.request.urlopen",
+            side_effect=_url_dispatch(
+                {
+                    "repo-a": _make_zip_bytes(repo_a_config),
+                    "repo-b": _make_zip_bytes(BASIC_SERVICE_CONFIG),
+                }
+            ),
+        ):
+            install_dependency(
+                RemoteConfig(
+                    repo_name="repo-a",
+                    branch="main",
+                    repo_link="https://github.com/getsentry/repo-a",
+                )
+            )
+
+            installed = get_installed_remote_dependencies(
+                dependencies=[
+                    Dependency(
+                        description="test repo",
+                        remote=RemoteConfig(
+                            repo_name="repo-a",
+                            branch="main",
+                            repo_link="https://github.com/getsentry/repo-a",
+                        ),
+                        dependency_type=DependencyType.SERVICE,
+                    )
+                ]
+            )
+
+        assert installed == {
+            InstalledRemoteDependency(
+                service_name="repo-a",
+                repo_path=str(
+                    tmp_path / "dependency-dir" / DEPENDENCY_CONFIG_VERSION / "repo-a"
+                ),
+            ),
+            InstalledRemoteDependency(
+                service_name="basic",
+                repo_path=str(
+                    tmp_path / "dependency-dir" / DEPENDENCY_CONFIG_VERSION / "repo-b"
+                ),
+            ),
+        }
+
+
 def test_install_dependency_invalid_repo(tmp_path: Path) -> None:
     with mock.patch(
         "devservices.utils.dependencies.DEVSERVICES_DEPENDENCIES_CACHE_DIR",