gitops 1.7.1__tar.gz → 1.8.0__tar.gz

{gitops-1.7.1 → gitops-1.8.0}/.github/workflows/test.yml

@@ -36,6 +36,12 @@ jobs:
           cache: true
           experimental: true
 
+      # Secrets in the cluster repo are git-crypt encrypted, and the app
+      # definition loader decrypts them. Without this the coverage of that path
+      # silently skips.
+      - name: Install git-crypt
+        run: sudo apt-get update && sudo apt-get install -y git-crypt
+
       - name: Run CI
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

{gitops-1.7.1 → gitops-1.8.0}/.release-please-manifest.json

@@ -1,5 +1,5 @@
 {
-    ".": "1.7.1",
+    ".": "1.8.0",
     "gitops_server": "0.15.1",
     "charts/gitops": "1.2.2"
 }

{gitops-1.7.1 → gitops-1.8.0}/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## [1.8.0](https://github.com/uptick/gitops/compare/gitops-v1.7.1...gitops-v1.8.0) (2026-08-21)
+
+
+### Features
+
+* **worker:** bound concurrent deployments ([956c630](https://github.com/uptick/gitops/commit/956c6309516ba831805aa1df90fd4fdb7314cb43))
+
+
+### Bug Fixes
+
+* **security:** stop embedding the github token in clone URLs ([fd88bc1](https://github.com/uptick/gitops/commit/fd88bc1176a51caf9dddca37aee9d4ecfc0b5c78))
+* **security:** stop embedding the github token in clone URLs ([68b7225](https://github.com/uptick/gitops/commit/68b722547d83e2ab1560ef7af0bc925d893b7580))
+
+
+### Performance Improvements
+
+* **deploy:** read app definitions from git instead of checking out the repo ([0b21790](https://github.com/uptick/gitops/commit/0b2179084e8920790de70363cdb369558342e895))
+* **deploy:** read app definitions from git instead of checking out the repo ([f90bc3a](https://github.com/uptick/gitops/commit/f90bc3a60141423313574813adeb1313406f7168))
+
 ## [1.7.1](https://github.com/uptick/gitops/compare/gitops-v1.7.0...gitops-v1.7.1) (2026-08-17)
 
 

{gitops-1.7.1 → gitops-1.8.0}/Dockerfile

@@ -56,7 +56,13 @@ COPY --link=true pyproject.toml uv.lock /app/
 RUN --mount=type=cache,target=/root/.cache/ \
     (uv sync --frozen --no-install-project --extra server || uv sync --frozen --no-install-project --extra server)
 # Install dependencies
-RUN git config --global advice.detachedHead false
+# Give git the token from the environment when it asks, so it is never embedded
+# in a clone URL, a command line, a log, or a repo's .git/config. The function
+# wrapper is the documented idiom -- git appends the operation (get/store/erase)
+# as an argument, which the wrapper discards. See gitcredentials(7).
+RUN git config --global advice.detachedHead false \
+    && git config --global credential.helper \
+       '!f() { printf "username=x-access-token\npassword=%s\n" "$GITHUB_OAUTH_TOKEN"; }; f'
 
 COPY cluster.key /app/
 COPY gitops /app/gitops/

{gitops-1.7.1 → gitops-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gitops
-Version: 1.7.1
+Version: 1.8.0
 Summary: Manage multiple apps across one or more k8s clusters.
 Author-email: Jarek Głowacki <jarekwg@gmail.com>
 License-File: LICENSE

{gitops-1.7.1 → gitops-1.8.0}/gitops/__init__.py

@@ -6,7 +6,7 @@ from gitops.utils.apps import App, get_app_details, get_apps  # type: ignore[att
 
 from .utils.cli import success, warning
 
-__version__ = "1.7.1"
+__version__ = "1.8.0"
 
 
 # Checking gitops version matches cluster repo version.

{gitops-1.7.1 → gitops-1.8.0}/gitops_server/settings.py

@@ -16,3 +16,8 @@ SLACK_ALERT_CHANNEL_ID = os.environ.get("SLACK_ALERT_CHANNEL_ID", "")
 
 # How many parallel deploys can be done at once
 GITOPS_MAX_PARALLEL_DEPLOYS = os.environ.get("GITOPS_MAX_PARALLEL_DEPLOYS", "5")
+
+# How many pushes can be processed at once. Each one loads app definitions for
+# two commits before it reaches GITOPS_MAX_PARALLEL_DEPLOYS, so without a bound
+# here a burst of pushes is only limited by how fast they arrive.
+GITOPS_MAX_CONCURRENT_DEPLOYMENTS = os.environ.get("GITOPS_MAX_CONCURRENT_DEPLOYMENTS", "20")

gitops-1.8.0/gitops_server/utils/git.py

@@ -0,0 +1,182 @@
+import asyncio
+import logging
+import os
+import re
+import tempfile
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from opentelemetry.trace import get_tracer
+
+from . import run
+
+tracer = get_tracer(__name__)
+logger = logging.getLogger("gitops")
+
+REPO_CACHE_DIR = Path("/tmp/gitops/repocache")
+REPO_CACHE: dict[str, Path] = {}
+repo_lock = asyncio.Semaphore(1)
+
+# git-crypt prefixes every encrypted blob with this magic.
+GIT_CRYPT_MAGIC = b"\0GITCRYPT\0"
+
+# git-crypt has no batch mode: one process per file. Bounded so concurrent
+# loads can't fork-storm.
+decrypt_semaphore = asyncio.Semaphore(16)
+
+
+async def clone_repo(git_repo_url: str, path: str, sha: str | None = None, branch: str | None = None) -> None:
+    """Shallow Clones a git repo url to path and git-crypt unlocks all encrypted files"""
+    logger.info(f'Cloning "{git_repo_url}".')
+
+    with tracer.start_as_current_span("tempo_repo.clone_repo"):
+        if sha:
+            await run(f"git clone --depth 100 {git_repo_url} {path}; cd {path}; git checkout {sha}")
+        elif branch:
+            await run(f"git clone --depth 1 {git_repo_url} {path} --branch {branch}")
+        else:
+            await run(f"git clone --depth 100 {git_repo_url} {path};")
+
+    if GIT_CRYPT_KEY_FILE := os.environ.get("GIT_CRYPT_KEY_FILE"):
+        with tracer.start_as_current_span("temp_repo.git_crypt_unlock"):
+            await run(f"cd {path}; git-crypt unlock {GIT_CRYPT_KEY_FILE}")
+
+
+def is_sha(sha_or_ref: str | None) -> bool:
+    """Check if the given string is a valid SHA-1 hash or a short SHA."""
+    if not sha_or_ref:
+        return False
+    return bool(re.fullmatch(r"[a-fA-F0-9]{4,40}", sha_or_ref))
+
+
+@asynccontextmanager
+async def temp_repo(git_repo_url: str, ref: str | None) -> AsyncGenerator[str, None]:
+    """Checks out a git_repo_url to a temporary folder location. Returns temporary folder location"""
+    # Someone passed us a branch name, not a sha
+
+    if not is_sha(ref):
+        async with temp_repo_branch(git_repo_url, branch=ref) as temp_repo_path:
+            yield temp_repo_path
+            return
+
+    assert ref is not None  # is_sha() returns False for None
+    with tracer.start_as_current_span("checkout_temp_repo"):
+        cache_path = await ensure_repo_cache(git_repo_url, ref)
+
+        # Copy the repo cache to a temporary folder
+        with tempfile.TemporaryDirectory() as temporary_folder_path:
+            await run(f"rm -rf {temporary_folder_path}", suppress_errors=True)
+            await run(f"cp -r {cache_path} {temporary_folder_path}")
+            # ref is already in the cache, so no per-copy fetch.
+            await run(f"cd {temporary_folder_path};git checkout {ref}")
+
+            yield temporary_folder_path
+
+
+async def _git(repo: str, *args: str, stdin: bytes | None = None) -> bytes:
+    """Run git in `repo`, returning raw stdout.
+
+    Not `run()`: blobs are binary, and `run()` decodes and logs every line.
+    """
+    proc = await asyncio.create_subprocess_exec(
+        "git",
+        "-C",
+        repo,
+        *args,
+        stdin=asyncio.subprocess.PIPE if stdin is not None else None,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate(stdin)
+    if proc.returncode != 0:
+        raise Exception(f"git {' '.join(args)} failed in {repo}: {stderr.decode(errors='replace')}")
+    return stdout
+
+
+async def ensure_repo_cache(git_repo_url: str, ref: str) -> str:
+    """Clone the shared cache if absent and ensure `ref` is in it."""
+    cache_path = REPO_CACHE_DIR / git_repo_url.split("/")[-1].split(".")[0]
+    async with repo_lock:
+        if not (cache_path / ".git").exists():
+            logger.info("Repo %s not in cache, cloning", git_repo_url)
+            if cache_path.exists():
+                await run(f"rm -rf {cache_path}", suppress_errors=True)
+            cache_path.mkdir(parents=True)
+            REPO_CACHE[git_repo_url] = cache_path
+            await clone_repo(git_repo_url, path=str(cache_path))
+
+        try:
+            await _git(str(cache_path), "cat-file", "-e", f"{ref}^{{commit}}")
+        except Exception:
+            logger.info("Ref %s not in cache, fetching", ref)
+            await _git(str(cache_path), "fetch", "--quiet", "origin")
+    return str(cache_path)
+
+
+async def list_tree_blobs(repo: str, sha: str, prefix: str) -> dict[str, str]:
+    """Map path -> blob oid for every file under `prefix` at `sha`."""
+    out = await _git(repo, "ls-tree", "-r", sha, "--", prefix)
+    blobs: dict[str, str] = {}
+    for line in out.decode().splitlines():
+        meta, _, path = line.partition("\t")
+        parts = meta.split()
+        if len(parts) == 3 and parts[1] == "blob":
+            blobs[path] = parts[2]
+    return blobs
+
+
+async def read_blobs(repo: str, oids: list[str]) -> dict[str, bytes]:
+    """Read many blobs in a single `git cat-file --batch` process."""
+    if not oids:
+        return {}
+    out = await _git(repo, "cat-file", "--batch", stdin=("\n".join(oids) + "\n").encode())
+
+    contents: dict[str, bytes] = {}
+    pos = 0
+    while pos < len(out):
+        eol = out.find(b"\n", pos)
+        if eol == -1:
+            break
+        header = out[pos:eol].decode()
+        pos = eol + 1
+        parts = header.split()
+        if len(parts) != 3:
+            # "<oid> missing" — nothing to consume for this entry.
+            continue
+        oid, _, size = parts[0], parts[1], int(parts[2])
+        contents[oid] = out[pos : pos + size]
+        pos += size + 1  # trailing newline
+    return contents
+
+
+async def decrypt_blob(repo: str, content: bytes) -> bytes:
+    """Decrypt a git-crypt blob; unencrypted content passes through.
+
+    Reading a blob bypasses the smudge filter, so apply it explicitly.
+    """
+    if not content.startswith(GIT_CRYPT_MAGIC):
+        return content
+    async with decrypt_semaphore:
+        proc = await asyncio.create_subprocess_exec(
+            "git-crypt",
+            "smudge",
+            cwd=repo,
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await proc.communicate(content)
+    if proc.returncode != 0:
+        raise Exception(f"git-crypt smudge failed in {repo}: {stderr.decode(errors='replace')}")
+    return stdout
+
+
+@asynccontextmanager
+async def temp_repo_branch(git_repo_url: str, branch: str | None) -> AsyncGenerator[str, None]:
+    """Checks out a git_repo_url to a temporary folder location. Returns temporary folder location"""
+    with tracer.start_as_current_span("checkout_temp_repo_branch"):
+        # Copy the repo cache to a temporary folder
+        with tempfile.TemporaryDirectory() as temporary_folder_path:
+            await clone_repo(git_repo_url, path=str(temporary_folder_path), branch=branch)
+            yield temporary_folder_path

gitops-1.8.0/gitops_server/version.py

@@ -0,0 +1 @@
+__version__ = "1.8.0"

{gitops-1.7.1 → gitops-1.8.0}/gitops_server/workers/deployer/deploy.py

@@ -1,19 +1,23 @@
 import asyncio
+import copy
 import json
 import logging
 import os
+import posixpath
 import tempfile
 import time
 import uuid
 from typing import Any
 
+import yaml
 from opentelemetry import trace
 
 from gitops.common.app import App
+from gitops.common.utils import deep_merge
 from gitops_server import settings
 from gitops_server.types import AppDefinitions, RunOutput, UpdateAppResult
 from gitops_server.utils import get_repo_name_from_url, github, run, slack
-from gitops_server.utils.git import temp_repo
+from gitops_server.utils.git import decrypt_blob, ensure_repo_cache, list_tree_blobs, read_blobs, temp_repo
 from gitops_server.workers.deployer.semaphore_manager import AppSemaphoreManager
 
 from .hooks import handle_failed_deploy, handle_successful_deploy
@@ -41,12 +45,75 @@ async def post_result_summary(source: str, results: list[UpdateAppResult]) -> No
     )
 
 
+APPS_PREFIX = "apps/"
+# The shared definition every app `extends`.
+ROOT_DEPLOYMENT = "apps/deployment.yml"
+
+
+def _resolve_extends(
+    values: dict[str, Any],
+    path: str,
+    definitions: dict[str, dict[str, Any]],
+    resolved: dict[str, dict[str, Any]],
+) -> dict[str, Any]:
+    """Resolve `extends:` against blobs, not the filesystem.
+
+    Mirrors resolve_values(), which reads a checkout we no longer produce.
+    """
+    if "extends" not in values:
+        return values
+    parent_path = posixpath.normpath(posixpath.join(posixpath.dirname(path), values["extends"]))
+    if parent_path not in resolved:
+        parent = definitions.get(parent_path, {})
+        resolved[parent_path] = _resolve_extends(parent, parent_path, definitions, resolved)
+    # deep_merge mutates its first arg, and every app shares this parent.
+    return deep_merge(copy.deepcopy(resolved[parent_path]), values)
+
+
 @tracer.start_as_current_span("load_app_definitions")
 async def load_app_definitions(url: str, sha: str) -> AppDefinitions:
+    """Build this cluster's app definitions from git blobs, with no checkout.
+
+    `temp_repo` copies ~38MB per call, twice per push, to read <1MB of YAML.
+    `deployment.yml` is plaintext in git, so other clusters' apps are dropped
+    before anything is decrypted.
+    """
     logger.info(f'Loading app definitions at "{sha}".')
-    async with temp_repo(url, ref=sha) as repo:
-        app_definitions = AppDefinitions(name=get_repo_name_from_url(url), path=repo)
-        return app_definitions
+    repo = await ensure_repo_cache(url, sha)
+    blobs = await list_tree_blobs(repo, sha, APPS_PREFIX)
+
+    deployment_paths = {
+        path: oid for path, oid in blobs.items() if path == ROOT_DEPLOYMENT or path.endswith("/deployment.yml")
+    }
+    raw = await read_blobs(repo, list(set(deployment_paths.values())))
+    definitions = {path: (yaml.safe_load(raw.get(oid, b"")) or {}) for path, oid in deployment_paths.items()}
+
+    resolved: dict[str, dict[str, Any]] = {}
+    keep: dict[str, dict[str, Any]] = {}
+    for path, values in definitions.items():
+        # apps/<name>/deployment.yml; anything else is the shared parent.
+        parts = path.split("/")
+        if path == ROOT_DEPLOYMENT or len(parts) != 3:
+            continue
+        merged = _resolve_extends(values, path, definitions, resolved)
+        if merged.get("cluster") != settings.CLUSTER_NAME or "suspended" in (merged.get("tags") or []):
+            continue
+        keep[parts[1]] = merged
+
+    # Only our own apps pay for decryption.
+    secret_oids = {name: blobs.get(f"{APPS_PREFIX}{name}/secrets.yml") for name in keep}
+    raw_secrets = await read_blobs(repo, [oid for oid in secret_oids.values() if oid])
+
+    async def build(name: str, merged: dict[str, Any]) -> tuple[str, App]:
+        secrets: dict[str, str] = {}
+        oid = secret_oids.get(name)
+        if oid:
+            plaintext = await decrypt_blob(repo, raw_secrets.get(oid, b""))
+            secrets = (yaml.safe_load(plaintext) or {}).get("secrets", {})
+        return name, App(name, deployments=merged, secrets=secrets, account_id=settings.ACCOUNT_ID)
+
+    built = await asyncio.gather(*(build(name, merged) for name, merged in keep.items()))
+    return AppDefinitions(name=get_repo_name_from_url(url), apps=dict(built))
 
 
 class Deployer:

{gitops-1.7.1 → gitops-1.8.0}/gitops_server/workers/deployer/worker.py

@@ -4,6 +4,7 @@ from typing import Any
 
 from opentelemetry import trace
 
+from ... import settings
 from .deploy import Deployer
 from .semaphore_manager import AppSemaphoreManager
 
@@ -33,6 +34,10 @@ class DeployQueueWorker:
         self.queue: asyncio.Queue[Any] = asyncio.Queue()
         self.semaphore_manager = AppSemaphoreManager()
         self.active_deployments: set[asyncio.Task[None]] = set()
+        # Bounds work in flight ahead of GITOPS_MAX_PARALLEL_DEPLOYS, which only
+        # gates the helm stage. Everything before it -- loading app definitions
+        # for two commits -- was previously limited only by webhook arrival rate.
+        self.deployment_semaphore = asyncio.Semaphore(int(settings.GITOPS_MAX_CONCURRENT_DEPLOYMENTS))
 
     async def enqueue(self, work: Any) -> None:
         """Enqueue an item of work for future processing.
@@ -56,9 +61,17 @@ class DeployQueueWorker:
                 # Clean up completed tasks
                 self.active_deployments = {task for task in self.active_deployments if not task.done()}
 
-                # Process new work
-                work = await self.queue.get()
-                deployment_task = asyncio.create_task(self.process_deployment(work))
+                # Process new work. Acquire before dequeuing so a burst backs up
+                # in the queue rather than as in-flight tasks. Once the task
+                # exists it owns the release; until then this frame does, or the
+                # slot is lost for the life of the process.
+                await self.deployment_semaphore.acquire()
+                try:
+                    work = await self.queue.get()
+                    deployment_task = asyncio.create_task(self.process_deployment(work))
+                except BaseException:
+                    self.deployment_semaphore.release()
+                    raise
                 self.active_deployments.add(deployment_task)
 
                 # Log current status
@@ -79,6 +92,12 @@ class DeployQueueWorker:
         ref = work.get("ref")
         logger.info(f'Processing deployment for push to "{ref}".')
 
+        try:
+            await self._process_deployment(work, ref)
+        finally:
+            self.deployment_semaphore.release()
+
+    async def _process_deployment(self, work: Any, ref: str | None) -> None:
         if ref == "refs/heads/master":
             with tracer.start_as_current_span("gitops_process_webhook") as current_span:
                 try:

{gitops-1.7.1 → gitops-1.8.0}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "gitops"
-version = "1.7.1"
+version = "1.8.0"
 description = "Manage multiple apps across one or more k8s clusters."
 requires-python = ">=3.12"
 readme = "README.md"

gitops-1.8.0/tests/test_clone_credentials.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+import gitops_server.utils.git as git
+
+TOKEN = "ghp_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"  # noqa: S105
+URL = "https://github.com/uptick/uptick-cluster.git"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kwargs", [{}, {"sha": "a" * 40}, {"branch": "master"}])
+async def test_clone_command_never_contains_the_token(monkeypatch, kwargs):
+    """Credentials come from the git credential helper configured in the image,
+    so nothing run() logs, raises or returns can carry the token."""
+    monkeypatch.setenv("GITHUB_OAUTH_TOKEN", TOKEN)
+    monkeypatch.delenv("GIT_CRYPT_KEY_FILE", raising=False)
+    with patch.object(git, "run", new=AsyncMock()) as run_mock:
+        await git.clone_repo(URL, path="/tmp/x", **kwargs)
+    commands = " ".join(str(c) for c in run_mock.call_args_list)
+    assert TOKEN not in commands
+    assert URL in commands
+
+
+def test_image_configures_a_helper_holding_no_secret():
+    """The Dockerfile is what makes the clean URLs above authenticate."""
+    dockerfile = (Path(__file__).parent.parent / "Dockerfile").read_text()
+    assert "credential.helper" in dockerfile
+    # A reference to the env var, never a value baked into the image.
+    assert "$GITHUB_OAUTH_TOKEN" in dockerfile
+    assert "ghp_" not in dockerfile

gitops-1.8.0/tests/test_load_app_definitions.py

@@ -0,0 +1,145 @@
+import os
+import shutil
+import subprocess
+
+import pytest
+import yaml
+
+import gitops_server.settings as settings
+import gitops_server.utils.git as git_utils
+from gitops_server.workers.deployer.deploy import load_app_definitions
+
+URL = "https://github.com/uptick/mock-repo.git"
+
+APPS = {
+    "ours": {"cluster": "test-cluster", "tags": ["a"]},
+    "ours-too": {"cluster": "test-cluster", "tags": []},
+    "theirs": {"cluster": "other-cluster", "tags": []},
+    "suspended": {"cluster": "test-cluster", "tags": ["suspended"]},
+}
+
+
+def _git(repo, *args) -> str:
+    """Run git against the fixture repo and return stdout."""
+    result = subprocess.run(  # noqa: S603
+        ["git", "-C", str(repo), *args],  # noqa: S607
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout.strip()
+
+
+def _write(repo, path, data):
+    full = repo / path
+    full.parent.mkdir(parents=True, exist_ok=True)
+    full.write_text(yaml.dump(data))
+
+
+def _make_repo(cache_root, encrypt=False):
+    """One flat apps/ directory holding several clusters' apps, as in the real repo."""
+    repo = cache_root / "mock-repo"
+    repo.mkdir()
+
+    _git(repo, "init", "-q", "-b", "main")
+    _git(repo, "config", "user.email", "test@example.com")
+    _git(repo, "config", "user.name", "test")
+
+    if encrypt:
+        # Must precede `git add` so the clean filter encrypts on the way in.
+        _git(repo, "crypt", "init")
+        (repo / ".gitattributes").write_text("**/secrets.yml filter=git-crypt diff=git-crypt\n")
+
+    # Shared parent that every app extends.
+    _write(repo, "apps/deployment.yml", {"chart": "https://github.com/some/chart", "namespace": "default"})
+    for name, values in APPS.items():
+        _write(repo, f"apps/{name}/deployment.yml", {"extends": "../deployment.yml", **values})
+        _write(repo, f"apps/{name}/secrets.yml", {"secrets": {"TOKEN": f"{name}-token"}})
+
+    _git(repo, "add", "-A")
+    _git(repo, "commit", "-q", "-m", "init")
+    return repo, _git(repo, "rev-parse", "HEAD")
+
+
+@pytest.fixture
+def repo(tmp_path, monkeypatch):
+    cache_root = tmp_path / "repocache"
+    cache_root.mkdir()
+    repo, sha = _make_repo(cache_root)
+    monkeypatch.setattr(git_utils, "REPO_CACHE_DIR", cache_root)
+    monkeypatch.setattr(settings, "CLUSTER_NAME", "test-cluster")
+    return repo, sha, cache_root
+
+
+@pytest.fixture
+def encrypted_repo(tmp_path, monkeypatch):
+    cache_root = tmp_path / "repocache"
+    cache_root.mkdir()
+    repo, sha = _make_repo(cache_root, encrypt=True)
+    monkeypatch.setattr(git_utils, "REPO_CACHE_DIR", cache_root)
+    monkeypatch.setattr(settings, "CLUSTER_NAME", "test-cluster")
+    return repo, sha, cache_root
+
+
+@pytest.mark.asyncio
+async def test_returns_only_this_clusters_unsuspended_apps(repo):
+    _, sha, _ = repo
+    definitions = await load_app_definitions(URL, sha)
+    assert sorted(definitions.apps) == ["ours", "ours-too"]
+
+
+@pytest.mark.asyncio
+async def test_resolves_extends_from_the_parent_definition(repo):
+    _, sha, _ = repo
+    definitions = await load_app_definitions(URL, sha)
+    # `namespace` and `chart` only exist on the parent.
+    assert definitions.apps["ours"].namespace == "default"
+    assert definitions.apps["ours"].values["tags"] == ["a"]
+
+
+@pytest.mark.asyncio
+async def test_loads_secrets_for_kept_apps(repo):
+    _, sha, _ = repo
+    definitions = await load_app_definitions(URL, sha)
+    # App base64-encodes secrets by default.
+    assert definitions.apps["ours"].secrets == {"TOKEN": "b3Vycy10b2tlbg=="}
+
+
+@pytest.mark.asyncio
+async def test_reads_a_past_sha_not_the_working_tree(repo):
+    """Blobs must resolve at the requested commit, not the checked-out one."""
+    repo_path, first_sha, _ = repo
+    _write(repo_path, "apps/ours/deployment.yml", {"extends": "../deployment.yml", "cluster": "other-cluster"})
+    _git(repo_path, "add", "-A")
+    _git(repo_path, "commit", "-q", "-m", "move ours to another cluster")
+    second_sha = _git(repo_path, "rev-parse", "HEAD")
+
+    assert sorted((await load_app_definitions(URL, first_sha)).apps) == ["ours", "ours-too"]
+    assert sorted((await load_app_definitions(URL, second_sha)).apps) == ["ours-too"]
+
+
+@pytest.mark.asyncio
+async def test_does_not_materialise_a_working_copy(repo):
+    """The point of reading blobs: no per-load copy on disk."""
+    _, sha, cache_root = repo
+    before = set(os.listdir(cache_root))
+    await load_app_definitions(URL, sha)
+    assert set(os.listdir(cache_root)) == before
+
+
+@pytest.mark.skipif(shutil.which("git-crypt") is None, reason="git-crypt is not installed")
+@pytest.mark.asyncio
+async def test_decrypts_git_crypt_secrets(encrypted_repo):
+    """Encrypted secrets must be run back through git-crypt."""
+    repo_path, sha, _ = encrypted_repo
+
+    # Prove the fixture is encrypted at rest, else this passes for the wrong reason.
+    raw = subprocess.run(  # noqa: S603
+        ["git", "-C", str(repo_path), "cat-file", "blob", f"{sha}:apps/ours/secrets.yml"],  # noqa: S607
+        check=True,
+        capture_output=True,
+    ).stdout
+    assert raw.startswith(git_utils.GIT_CRYPT_MAGIC)
+
+    definitions = await load_app_definitions(URL, sha)
+    assert definitions.apps["ours"].secrets == {"TOKEN": "b3Vycy10b2tlbg=="}

gitops-1.8.0/tests/test_worker_concurrency.py

@@ -0,0 +1,76 @@
+import asyncio
+from unittest import mock
+
+import pytest
+
+from gitops_server.workers.deployer.worker import DeployQueueWorker
+
+
+@pytest.mark.asyncio
+async def test_concurrent_deployments_are_bounded():
+    """A burst of pushes must not put more than the limit in flight at once.
+
+    Everything before `helm_parallel_semaphore` -- loading app definitions for
+    two commits -- runs here, so an unbounded worker scales that with the
+    webhook arrival rate.
+    """
+    limit = 3
+    in_flight = 0
+    peak = 0
+    release = asyncio.Event()
+
+    async def slow_deployment(work, ref):
+        nonlocal in_flight, peak
+        in_flight += 1
+        peak = max(peak, in_flight)
+        try:
+            await release.wait()
+        finally:
+            in_flight -= 1
+
+    with mock.patch("gitops_server.settings.GITOPS_MAX_CONCURRENT_DEPLOYMENTS", str(limit)):
+        worker = DeployQueueWorker()
+    worker._process_deployment = slow_deployment  # type: ignore[method-assign]
+
+    runner = asyncio.create_task(worker.run())
+    for _ in range(10):
+        await worker.enqueue({"ref": "refs/heads/master"})
+
+    # Let the worker take as much as it is willing to take.
+    for _ in range(20):
+        await asyncio.sleep(0)
+
+    assert peak == limit, f"expected at most {limit} in flight, saw {peak}"
+    assert worker.queue.qsize() > 0, "excess work should wait in the queue"
+
+    release.set()
+    for _ in range(50):
+        await asyncio.sleep(0)
+
+    assert peak == limit
+    runner.cancel()
+
+
+@pytest.mark.asyncio
+async def test_slot_is_released_when_a_deployment_fails():
+    """A failing deployment must not permanently consume a slot."""
+    limit = 2
+    started = 0
+
+    async def failing_deployment(work, ref):
+        nonlocal started
+        started += 1
+        raise RuntimeError("boom")
+
+    with mock.patch("gitops_server.settings.GITOPS_MAX_CONCURRENT_DEPLOYMENTS", str(limit)):
+        worker = DeployQueueWorker()
+    worker._process_deployment = failing_deployment  # type: ignore[method-assign]
+
+    runner = asyncio.create_task(worker.run())
+    for _ in range(6):
+        await worker.enqueue({"ref": "refs/heads/master"})
+    for _ in range(60):
+        await asyncio.sleep(0)
+
+    assert started == 6, f"slots leaked after failures: only {started} of 6 ran"
+    runner.cancel()

{gitops-1.7.1 → gitops-1.8.0}/uv.lock

@@ -360,7 +360,7 @@ wheels = [
 
 [[package]]
 name = "gitops"
-version = "1.7.1"
+version = "1.8.0"
 source = { editable = "." }
 dependencies = [
     { name = "boto3" },

gitops-1.7.1/gitops_server/utils/git.py

@@ -1,88 +0,0 @@
-import asyncio
-import logging
-import os
-import re
-import tempfile
-from collections.abc import AsyncGenerator
-from contextlib import asynccontextmanager
-from pathlib import Path
-
-from opentelemetry.trace import get_tracer
-
-from . import run
-
-tracer = get_tracer(__name__)
-logger = logging.getLogger("gitops")
-
-REPO_CACHE_DIR = Path("/tmp/gitops/repocache")
-REPO_CACHE: dict[str, Path] = {}
-repo_lock = asyncio.Semaphore(1)
-
-
-async def clone_repo(git_repo_url: str, path: str, sha: str | None = None, branch: str | None = None) -> None:
-    """Shallow Clones a git repo url to path and git-crypt unlocks all encrypted files"""
-    logger.info(f'Cloning "{git_repo_url}".')
-
-    url_with_oauth_token = git_repo_url.replace("://", f"://{os.environ['GITHUB_OAUTH_TOKEN'].strip()}@")
-
-    with tracer.start_as_current_span("tempo_repo.clone_repo"):
-        if sha:
-            await run(f"git clone --depth 100 {url_with_oauth_token} {path}; cd {path}; git checkout {sha}")
-        elif branch:
-            await run(f"git clone --depth 1 {url_with_oauth_token} {path} --branch {branch}")
-        else:
-            await run(f"git clone --depth 100 {url_with_oauth_token} {path};")
-
-    if GIT_CRYPT_KEY_FILE := os.environ.get("GIT_CRYPT_KEY_FILE"):
-        with tracer.start_as_current_span("temp_repo.git_crypt_unlock"):
-            await run(f"cd {path}; git-crypt unlock {GIT_CRYPT_KEY_FILE}")
-
-
-def is_sha(sha_or_ref: str | None) -> bool:
-    """Check if the given string is a valid SHA-1 hash or a short SHA."""
-    if not sha_or_ref:
-        return False
-    return bool(re.fullmatch(r"[a-fA-F0-9]{4,40}", sha_or_ref))
-
-
-@asynccontextmanager
-async def temp_repo(git_repo_url: str, ref: str | None) -> AsyncGenerator[str, None]:
-    """Checks out a git_repo_url to a temporary folder location. Returns temporary folder location"""
-    # Someone passed us a branch name, not a sha
-
-    if not is_sha(ref):
-        async with temp_repo_branch(git_repo_url, branch=ref) as temp_repo_path:
-            yield temp_repo_path
-            return
-
-    with tracer.start_as_current_span("checkout_temp_repo"):
-        cache_path = REPO_CACHE_DIR / git_repo_url.split("/")[-1].split(".")[0]
-
-        # Prep the repo cache
-        if not (cache_path / ".git").exists():
-            logger.info("Repo %s not in cache, cloning", git_repo_url)
-            async with repo_lock:
-                if cache_path.exists():
-                    await run(f"rm -rf {cache_path}", suppress_errors=True)
-                if not cache_path.exists():
-                    cache_path.mkdir(parents=True)
-                REPO_CACHE[git_repo_url] = cache_path
-                await clone_repo(git_repo_url, path=str(cache_path))
-
-        # Copy the repo cache to a temporary folder
-        with tempfile.TemporaryDirectory() as temporary_folder_path:
-            await run(f"rm -rf {temporary_folder_path}", suppress_errors=True)
-            await run(f"cp -r {cache_path} {temporary_folder_path}")
-            await run(f"cd {temporary_folder_path};git fetch; git checkout {ref}")
-
-            yield temporary_folder_path
-
-
-@asynccontextmanager
-async def temp_repo_branch(git_repo_url: str, branch: str | None) -> AsyncGenerator[str, None]:
-    """Checks out a git_repo_url to a temporary folder location. Returns temporary folder location"""
-    with tracer.start_as_current_span("checkout_temp_repo_branch"):
-        # Copy the repo cache to a temporary folder
-        with tempfile.TemporaryDirectory() as temporary_folder_path:
-            await clone_repo(git_repo_url, path=str(temporary_folder_path), branch=branch)
-            yield temporary_folder_path

gitops-1.7.1/gitops_server/version.py

@@ -1 +0,0 @@
-__version__ = "1.7.1"