devservices 1.3.2__tar.gz → 1.4.1__tar.gz

{devservices-1.3.2 → devservices-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devservices
-Version: 1.3.2
+Version: 1.4.1
 Requires-Python: >=3.11
 License-File: LICENSE.md
 Requires-Dist: pyyaml

{devservices-1.3.2 → devservices-1.4.1}/README.md

@@ -33,7 +33,7 @@ NOTE: service-name is an optional parameter. If not provided, devservices will a
 The recommended way to install devservices is through a virtualenv in the requirements.txt. Once that is installed and a devservices config file is added, you should be able to run `devservices up` to begin local development.
 
 ```
-devservices==1.3.2
+devservices==1.4.1
 ```
 
 ### 2. Add devservices config files

{devservices-1.3.2 → devservices-1.4.1}/devservices/constants.py

@@ -37,11 +37,6 @@ STATE_DB_FILE = os.path.join(DEVSERVICES_LOCAL_DIR, "state")
 DEVSERVICES_ORCHESTRATOR_LABEL = "orchestrator=devservices"
 
 DEPENDENCY_CONFIG_VERSION = "v1"
-DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS = {
-    "protocol.version": "2",
-    "extensions.partialClone": "true",
-    "core.sparseCheckout": "true",
-}
 
 DEVSERVICES_RELEASES_URL = (
     "https://api.github.com/repos/getsentry/devservices/releases/latest"

{devservices-1.3.2 → devservices-1.4.1}/devservices/exceptions.py

@@ -102,13 +102,6 @@ class DependencyError(Exception):
         return f"DependencyError: {self.repo_name} ({self.repo_link}) on {self.branch}"
 
 
-class UnableToCloneDependencyError(DependencyError):
-    """Raised when a dependency is unable to be cloned."""
-
-    def __str__(self) -> str:
-        return f"Unable to clone dependency: {self.repo_name} ({self.repo_link}) on {self.branch}"
-
-
 class InvalidDependencyConfigError(DependencyError):
     """Raised when a dependency's config is invalid."""
 
@@ -142,18 +135,6 @@ class GitError(Exception):
         self.stderr = stderr
 
 
-class GitConfigError(Exception):
-    """Base class for git config related errors."""
-
-    pass
-
-
-class FailedToSetGitConfigError(GitConfigError):
-    """Raised when a git config cannot be set."""
-
-    pass
-
-
 class ContainerHealthcheckFailedError(Exception):
     """Raised when a container is not healthy."""
 

{devservices-1.3.2 → devservices-1.4.1}/devservices/utils/dependencies.py

@@ -1,21 +1,18 @@
 from __future__ import annotations
 
-import logging
+import io
 import os
 import shutil
-import subprocess
-import tempfile
-import time
+import urllib.error
+import urllib.request
+import zipfile
 from collections import deque
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import as_completed
 from dataclasses import dataclass
-from typing import TextIO
 from typing import TypeGuard
 
-from sentry_sdk import capture_message
 from sentry_sdk import logger as sentry_logger
-from sentry_sdk import set_context
 
 from devservices.configs.service_config import Dependency
 from devservices.configs.service_config import RemoteConfig
@@ -23,21 +20,19 @@ from devservices.configs.service_config import ServiceConfig
 from devservices.configs.service_config import load_service_config_from_file
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
-from devservices.constants import DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
 from devservices.constants import DEVSERVICES_DIR_NAME
-from devservices.constants import LOGGER_NAME
 from devservices.constants import DependencyType
 from devservices.exceptions import ConfigNotFoundError
 from devservices.exceptions import ConfigParseError
 from devservices.exceptions import ConfigValidationError
 from devservices.exceptions import DependencyError
 from devservices.exceptions import DependencyNotInstalledError
-from devservices.exceptions import FailedToSetGitConfigError
 from devservices.exceptions import InvalidDependencyConfigError
 from devservices.exceptions import ModeDoesNotExistError
-from devservices.exceptions import UnableToCloneDependencyError
+from devservices.utils import github
 from devservices.utils.file_lock import lock
+from devservices.utils.retry import retry
 from devservices.utils.services import Service
 from devservices.utils.services import find_matching_service
 from devservices.utils.services import get_active_service_names
@@ -45,17 +40,6 @@ from devservices.utils.state import ServiceRuntime
 from devservices.utils.state import State
 from devservices.utils.state import StateTables
 
-RELEVANT_GIT_CONFIG_KEYS = [
-    "init.defaultbranch",
-    "core.sparsecheckout",
-    "remote.origin.url",
-    "remote.origin.fetch",
-    "remote.origin.promisor",
-    "remote.origin.partialclonefilter",
-    "protocol.version",
-    "extensions.partialclone",
-]
-
 
 @dataclass(frozen=True, eq=True)
 class DependencyNode:
@@ -129,87 +113,6 @@ class InstalledRemoteDependency:
     mode: str = "default"
 
 
-class SparseCheckoutManager:
-    """
-    Manages sparse checkout for a repo
-    """
-
-    def __init__(self, repo_dir: str):
-        self.repo_dir = repo_dir
-
-    def init_sparse_checkout(self) -> None:
-        """
-        Initialize sparse checkout for the repo
-        """
-        _run_command(["git", "sparse-checkout", "init"], cwd=self.repo_dir)
-
-    def set_sparse_checkout(self, pattern: str) -> None:
-        """
-        Set sparse checkout patterns for the repo
-        """
-        self.init_sparse_checkout()
-        _run_command(["git", "sparse-checkout", "set", pattern], cwd=self.repo_dir)
-
-
-class GitConfigManager:
-    """
-    Manages git config for a repo
-    """
-
-    def __init__(
-        self,
-        repo_dir: str,
-        config_options: dict[str, str],
-        sparse_pattern: str | None = None,
-    ) -> None:
-        self.repo_dir = repo_dir
-        self.config_options = config_options
-        self.sparse_pattern = sparse_pattern
-        self.sparse_checkout_manager = SparseCheckoutManager(repo_dir)
-
-    def ensure_config(self) -> None:
-        """
-        Ensure that the git config is set correctly for the repo
-        """
-        # Otherwise, set the config options
-        for key, value in self.config_options.items():
-            self._set_config(key, value)
-
-        if self.sparse_pattern:
-            self.sparse_checkout_manager.set_sparse_checkout(self.sparse_pattern)
-
-    def get_relevant_config(self) -> dict[str, str]:
-        """
-        Get the relevant git config entries (to avoid logging sensitive information)
-        """
-        git_config = (
-            subprocess.check_output(
-                ["git", "config", "--list"],
-                cwd=self.repo_dir,
-                stderr=subprocess.PIPE,
-            )
-            .decode()
-            .strip()
-        )
-        git_config_dict = dict()
-        for line in git_config.split("\n"):
-            if not line:
-                continue
-            key, value = line.split("=")
-            if key in RELEVANT_GIT_CONFIG_KEYS:
-                git_config_dict[key] = value
-        return git_config_dict
-
-    def _set_config(self, key: str, value: str) -> None:
-        """
-        Set a git config option for the repo
-        """
-        try:
-            _run_command(["git", "config", key, value], cwd=self.repo_dir)
-        except subprocess.CalledProcessError as e:
-            raise FailedToSetGitConfigError from e
-
-
 def install_and_verify_dependencies(
     service: Service,
     force_update_dependencies: bool = False,
@@ -435,14 +338,7 @@ def install_dependency(dependency: RemoteConfig) -> set[InstalledRemoteDependenc
         DEVSERVICES_DEPENDENCIES_CACHE_DIR, f"{dependency.repo_name}.lock"
     )
     with lock(lock_path):
-        if (
-            os.path.exists(dependency_repo_dir)
-            and _is_valid_repo(dependency_repo_dir)
-            and _has_valid_config_file(dependency_repo_dir)
-        ):
-            _update_dependency(dependency, dependency_repo_dir)
-        else:
-            _checkout_dependency(dependency, dependency_repo_dir)
+        _fetch_dependency(dependency, dependency_repo_dir)
 
         if not verify_local_dependency(dependency):
             # TODO: what should we do if the local dependency isn't installed correctly?
@@ -503,185 +399,91 @@ def install_dependency(dependency: RemoteConfig) -> set[InstalledRemoteDependenc
     return installed_dependencies
 
 
-def _update_dependency(
+def _fetch_dependency(
     dependency: RemoteConfig,
     dependency_repo_dir: str,
 ) -> None:
     sentry_logger.info(
-        "Updating dependency",
+        "Fetching dependency",
         extra={
             "repo_name": dependency.repo_name,
             "repo_link": dependency.repo_link,
             "branch": dependency.branch,
         },
     )
-    git_config_manager = GitConfigManager(
-        dependency_repo_dir,
-        DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS,
-        f"{DEVSERVICES_DIR_NAME}/",
-    )
     try:
-        git_config_manager.ensure_config()
-    except FailedToSetGitConfigError as e:
+        repo_path = github.parse_repo_path(dependency.repo_link)
+    except ValueError as e:
         raise DependencyError(
             repo_name=dependency.repo_name,
             repo_link=dependency.repo_link,
             branch=dependency.branch,
         ) from e
 
-    try:
-        _run_command_with_retries(
-            [
-                "git",
-                "fetch",
-                "origin",
-                dependency.branch,
-                "--filter=blob:none",
-                "--no-recurse-submodules",  # Avoid fetching submodules
-            ],
-            cwd=dependency_repo_dir,
-        )
-    except subprocess.CalledProcessError as e:
-        # Try to set the git config context to help with debugging
-        _try_set_git_config_context(git_config_manager)
-        raise DependencyError(
-            repo_name=dependency.repo_name,
-            repo_link=dependency.repo_link,
-            branch=dependency.branch,
-            stderr=e.stderr,
-        ) from e
+    zip_url = github.zipball_url(repo_path, dependency.branch)
 
-    # Check if the local repo is up-to-date
-    try:
-        local_commit = _rev_parse(dependency_repo_dir, "HEAD")
-    except subprocess.CalledProcessError as e:
-        raise DependencyError(
-            repo_name=dependency.repo_name,
-            repo_link=dependency.repo_link,
-            branch=dependency.branch,
-            stderr=e.stderr,
-        ) from e
+    def _download() -> bytes:
+        req = github.build_api_request(zip_url)
+        with urllib.request.urlopen(req) as response:
+            return bytes(response.read())
 
     try:
-        remote_commit = _rev_parse(dependency_repo_dir, "FETCH_HEAD")
-    except subprocess.CalledProcessError as e:
+        zip_data = io.BytesIO(
+            retry(
+                _download,
+                exceptions=(OSError,),
+                should_retry=lambda e: not isinstance(e, urllib.error.HTTPError),
+            )
+        )
+    except (OSError, ValueError) as e:
         raise DependencyError(
             repo_name=dependency.repo_name,
             repo_link=dependency.repo_link,
             branch=dependency.branch,
-            stderr=e.stderr,
         ) from e
 
-    if local_commit == remote_commit:
-        # Already up-to-date, don't pull anything
-        logger = logging.getLogger(LOGGER_NAME)
-        logger.debug(
-            "Dependency %s is already up-to-date, not pulling anything",
-            dependency.repo_name,
-        )
-        return
-
-    # If it's not up-to-date, checkout the latest changes (forcibly)
     try:
-        _run_command(["git", "checkout", "-f", "FETCH_HEAD"], cwd=dependency_repo_dir)
-    except subprocess.CalledProcessError as e:
+        with zipfile.ZipFile(zip_data) as zf:
+            names = zf.namelist()
+            if not names:
+                raise DependencyError(
+                    repo_name=dependency.repo_name,
+                    repo_link=dependency.repo_link,
+                    branch=dependency.branch,
+                )
+            # GitHub zips always have a single top-level directory: "owner-repo-sha/"
+            prefix = names[0].split("/")[0] + "/"
+            devservices_prefix = f"{prefix}{DEVSERVICES_DIR_NAME}/"
+            to_extract = [
+                name
+                for name in names
+                if name.startswith(devservices_prefix) and not name.endswith("/")
+            ]
+            if not to_extract:
+                raise DependencyError(
+                    repo_name=dependency.repo_name,
+                    repo_link=dependency.repo_link,
+                    branch=dependency.branch,
+                )
+
+            if os.path.exists(dependency_repo_dir):
+                shutil.rmtree(dependency_repo_dir)
+            os.makedirs(dependency_repo_dir)
+
+            for name in to_extract:
+                relative_path = name[len(prefix) :]
+                target = os.path.join(dependency_repo_dir, relative_path)
+                os.makedirs(os.path.dirname(target), exist_ok=True)
+                with zf.open(name) as src, open(target, "wb") as dst:
+                    shutil.copyfileobj(src, dst)
+    except (zipfile.BadZipFile, OSError) as e:
         raise DependencyError(
             repo_name=dependency.repo_name,
             repo_link=dependency.repo_link,
             branch=dependency.branch,
-            stderr=e.stderr,
         ) from e
 
 
-def _checkout_dependency(
-    dependency: RemoteConfig,
-    dependency_repo_dir: str,
-) -> None:
-    sentry_logger.info(
-        "Checking out dependency",
-        extra={
-            "repo_name": dependency.repo_name,
-            "repo_link": dependency.repo_link,
-            "branch": dependency.branch,
-        },
-    )
-    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
-        try:
-            _run_command(
-                [
-                    "git",
-                    "clone",
-                    "--filter=blob:none",
-                    "--no-checkout",
-                    dependency.repo_link,
-                    temp_dir,
-                ],
-                cwd=temp_dir,
-            )
-        except subprocess.CalledProcessError as e:
-            raise UnableToCloneDependencyError(
-                repo_name=dependency.repo_name,
-                repo_link=dependency.repo_link,
-                branch=dependency.branch,
-                stderr=e.stderr,
-            ) from e
-
-        # Setup config for partial clone and sparse checkout
-        git_config_manager = GitConfigManager(
-            temp_dir,
-            DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS,
-            f"{DEVSERVICES_DIR_NAME}/",
-        )
-        try:
-            git_config_manager.ensure_config()
-        except FailedToSetGitConfigError as e:
-            raise DependencyError(
-                repo_name=dependency.repo_name,
-                repo_link=dependency.repo_link,
-                branch=dependency.branch,
-            ) from e
-
-        try:
-            _run_command(
-                ["git", "checkout", dependency.branch],
-                cwd=temp_dir,
-            )
-        except subprocess.CalledProcessError as e:
-            raise DependencyError(
-                repo_name=dependency.repo_name,
-                repo_link=dependency.repo_link,
-                branch=dependency.branch,
-                stderr=e.stderr,
-            ) from e
-
-        # Clean up the existing directory if it exists
-        if os.path.exists(dependency_repo_dir):
-            shutil.rmtree(dependency_repo_dir)
-        # Copy the cloned repo to the dependency cache directory
-        try:
-            shutil.copytree(temp_dir, dst=dependency_repo_dir)
-        except FileExistsError as e:
-            raise DependencyError(
-                repo_name=dependency.repo_name,
-                repo_link=dependency.repo_link,
-                branch=dependency.branch,
-            ) from e
-
-
-def _is_valid_repo(path: str) -> bool:
-    if not os.path.exists(os.path.join(path, ".git")):
-        return False
-    try:
-        _run_command(["git", "rev-parse", "--is-inside-work-tree"], cwd=path)
-        return True
-    except subprocess.CalledProcessError:
-        return False
-
-
-def _has_valid_config_file(path: str) -> bool:
-    return os.path.exists(os.path.join(path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME))
-
-
 def _get_remote_configs(dependencies: list[Dependency]) -> list[RemoteConfig]:
     return [
         dependency.remote
@@ -694,64 +496,6 @@ def _has_remote_config(remote_config: RemoteConfig | None) -> TypeGuard[RemoteCo
     return remote_config is not None
 
 
-def _rev_parse(repo_dir: str, ref: str) -> str:
-    logger = logging.getLogger(LOGGER_NAME)
-    logger.debug("Parsing revision for %s (%s)", ref, repo_dir)
-    rev = (
-        subprocess.check_output(
-            ["git", "rev-parse", ref], cwd=repo_dir, stderr=subprocess.PIPE
-        )
-        .strip()
-        .decode()
-    )
-    logger.debug("Parsed revision %s for %s (%s)", rev, ref, repo_dir)
-    return rev
-
-
-def _run_command(
-    cmd: list[str], cwd: str, stdout: int | TextIO | None = subprocess.DEVNULL
-) -> None:
-    logger = logging.getLogger(LOGGER_NAME)
-    logger.debug("Running command: %s in %s", " ".join(cmd), cwd)
-    subprocess.run(cmd, cwd=cwd, check=True, stdout=stdout, stderr=subprocess.PIPE)
-
-
-def _run_command_with_retries(
-    cmd: list[str],
-    cwd: str,
-    stdout: int | TextIO | None = subprocess.DEVNULL,
-    retries: int = 3,
-    backoff: int = 2,
-) -> None:
-    for i in range(retries):
-        try:
-            _run_command(cmd, cwd=cwd, stdout=stdout)
-            break
-        except subprocess.CalledProcessError as e:
-            logger = logging.getLogger(LOGGER_NAME)
-            logger.debug(
-                "Attempt %s of %s for %s failed: %s", i + 1, retries, cmd, e.stderr
-            )
-            capture_message(
-                f"Attempt {i + 1} of {retries} for {cmd} failed: {e.stderr}",
-                level="warning",
-            )
-            if i == retries - 1:
-                raise e
-            time.sleep(backoff**i)
-
-
-def _try_set_git_config_context(
-    git_config_manager: GitConfigManager,
-) -> None:
-    try:
-        git_config = git_config_manager.get_relevant_config()
-        set_context("git_config", git_config)
-    except subprocess.CalledProcessError as e:
-        logger = logging.getLogger(LOGGER_NAME)
-        logger.exception(e)
-
-
 def get_remote_dependency_config(remote_config: RemoteConfig) -> ServiceConfig:
     dependency_repo_dir = os.path.join(
         DEVSERVICES_DEPENDENCIES_CACHE_DIR,

devservices-1.4.1/devservices/utils/github.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+import threading
+import urllib.request
+
+from devservices.utils.console import Console
+
+GITHUB_API_ACCEPT = "application/vnd.github+json"
+
+_auth_warning_lock = threading.Lock()
+_auth_warned = False
+
+
+def parse_repo_path(repo_link: str) -> str:
+    """Extract the "owner/repo" path from a GitHub URL."""
+    url = repo_link.rstrip("/").removesuffix(".git")
+    if "github.com/" not in url:
+        raise ValueError(f"Not a GitHub URL: {repo_link}")
+    return url.split("github.com/", 1)[1]
+
+
+def zipball_url(repo_path: str, ref: str) -> str:
+    return f"https://api.github.com/repos/{repo_path}/zipball/{ref}"
+
+
+def build_api_request(url: str) -> urllib.request.Request:
+    """Build an authenticated GitHub API request, authenticating when possible.
+
+    Authenticated requests get the 5000 req/hr rate limit instead of the 60
+    req/hr unauthenticated limit (which CI runners, sharing egress IPs, exhaust
+    quickly and then receive HTTP 403s).
+    """
+    req = urllib.request.Request(url, headers={"Accept": GITHUB_API_ACCEPT})
+    token = _get_token()
+    if token:
+        # Use an unredirected header so the token is only sent to api.github.com
+        # and is NOT forwarded when the API 302-redirects the download to
+        # codeload.github.com (and S3-backed hosts for other endpoints), which
+        # reject or mishandle requests that carry an Authorization header.
+        req.add_unredirected_header("Authorization", f"Bearer {token}")
+    return req
+
+
+def _get_token() -> str | None:
+    # Prefer an explicit token from the environment (CI sets these), otherwise
+    # fall back to the locally authenticated `gh` CLI so local users are authed
+    # without extra setup.
+    token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+    if token:
+        return token
+    if shutil.which("gh") is not None:
+        try:
+            result = subprocess.run(
+                ["gh", "auth", "token"],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+        except (OSError, subprocess.SubprocessError):
+            result = None
+        if result is not None and result.returncode == 0:
+            token = result.stdout.strip()
+            if token:
+                return token
+    _warn_unauthenticated()
+    return None
+
+
+def _warn_unauthenticated() -> None:
+    # Fetches run concurrently across dependencies; only warn once per process so
+    # we don't spam the same message for every dependency.
+    global _auth_warned
+    with _auth_warning_lock:
+        if _auth_warned:
+            return
+        _auth_warned = True
+    if sys.platform == "darwin":
+        install_hint = "Install the GitHub CLI (brew install gh)"
+    else:
+        install_hint = "Install the GitHub CLI (https://github.com/cli/cli)"
+    Console().warning(
+        "Could not authenticate with GitHub; falling back to unauthenticated "
+        f"downloads, which are heavily rate-limited. {install_hint} and run "
+        "`gh auth login` to authenticate."
+    )

{devservices-1.3.2 → devservices-1.4.1}/devservices/utils/install_binary.py

@@ -3,12 +3,12 @@ from __future__ import annotations
 import os
 import shutil
 import tempfile
-import time
 from urllib.request import urlretrieve
 
 from devservices.constants import BINARY_PERMISSIONS
 from devservices.exceptions import BinaryInstallError
 from devservices.utils.console import Console
+from devservices.utils.retry import retry
 
 
 def install_binary(
@@ -21,26 +21,21 @@ def install_binary(
     with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
         temp_file = os.path.join(temp_dir, binary_name)
 
-        # Download the binary with retries
-        max_retries = 3
-        retry_delay_seconds = 1
         console.info(f"Downloading {binary_name} {version} from {url}...")
-        for attempt in range(max_retries):
-            try:
-                urlretrieve(url, temp_file)
-                break
-            except Exception as e:
-                if attempt < max_retries - 1:
-                    console.warning(
-                        f"Download failed. Retrying in {retry_delay_seconds} seconds... (Attempt {attempt + 1}/{max_retries - 1})"
-                    )
-                    time.sleep(retry_delay_seconds)
-                else:
-                    raise BinaryInstallError(
-                        f"Failed to download {binary_name} after {max_retries} attempts: {e}"
-                    ) from e
+        try:
+            retry(
+                lambda: urlretrieve(url, temp_file),
+                retries=3,
+                delay=1.0,
+                on_retry=lambda e, remaining: console.warning(
+                    f"Download failed. Retrying in 1 seconds... ({remaining} retries left)"
+                ),
+            )
+        except Exception as e:
+            raise BinaryInstallError(
+                f"Failed to download {binary_name} after 3 attempts: {e}"
+            ) from e
 
-        # Make the binary executable
         try:
             os.chmod(temp_file, BINARY_PERMISSIONS)
         except (PermissionError, FileNotFoundError) as e: