gitops 1.10.1__tar.gz → 1.11.0__tar.gz

{gitops-1.10.1 → gitops-1.11.0}/.github/workflows/release-please.yaml

@@ -84,7 +84,7 @@ jobs:
         shell: bash
 
       - name: Publish Pypi Package
-        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # ratchet:pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # ratchet:pypa/gh-action-pypi-publish@release/v1
 
   publish_docker_image:
     name: Build and Push Docker Image

{gitops-1.10.1 → gitops-1.11.0}/.mise.toml

@@ -1,6 +1,6 @@
 [tools]
 python = { version = "3.12.13", venv = ".venv" }
-uv = { version = "0.11.26"}
+uv = { version = "0.13.4"}
 
 [env]
 _.python.venv = { path = "{{config_root}}/.venv", create = true }

gitops-1.11.0/.release-please-manifest.json

@@ -0,0 +1,5 @@
+{
+    ".": "1.11.0",
+    "gitops_server": "0.15.1",
+    "charts/gitops": "1.5.0"
+}

{gitops-1.10.1 → gitops-1.11.0}/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## [1.11.0](https://github.com/uptick/gitops/compare/gitops-v1.10.2...gitops-v1.11.0) (2026-10-27)
+
+
+### Features
+
+* **server:** hold karpenter.sh/do-not-disrupt while deploying ([b977e5f](https://github.com/uptick/gitops/commit/b977e5f949b237c5b458f6b9e3029f02592e55a8))
+* **server:** hold karpenter.sh/do-not-disrupt while deploying ([c14359f](https://github.com/uptick/gitops/commit/c14359f39ba354432378bdb2ad9963e442fc9f66))
+
+
+### Bug Fixes
+
+* **server:** clear a leftover do-not-disrupt annotation on startup ([7c53772](https://github.com/uptick/gitops/commit/7c53772e40aa94a22c9268d926a2ca8f9122f607))
+
+## [1.10.2](https://github.com/uptick/gitops/compare/gitops-v1.10.1...gitops-v1.10.2) (2026-10-22)
+
+
+### Bug Fixes
+
+* **ci:** upgrade pypa/gh-action-pypi-publish to v1.14.3 ([384631c](https://github.com/uptick/gitops/commit/384631cc0f931dcb82056969fcc9512b329f08a4))
+* **deps:** upgrade uv to 0.13.4 ([1f63116](https://github.com/uptick/gitops/commit/1f631162fc7b40b55bbdef6486a25d364666b637))
+* **deps:** upgrade uv to 0.13.4 and the PyPI publish action to v1.14.3 ([f20a90b](https://github.com/uptick/gitops/commit/f20a90bec048a682e730f393afd686493a5d91fc))
+
 ## [1.10.1](https://github.com/uptick/gitops/compare/gitops-v1.10.0...gitops-v1.10.1) (2026-10-21)
 
 

{gitops-1.10.1 → gitops-1.11.0}/Dockerfile

@@ -2,7 +2,7 @@
 # This is a multi-stage build to use uv for dependency management and is necessary to ensure the the image version we pull is for the right architecture.
 # This is automatically set by the buildx builder. when using --platoform linux/amd64,linux/arm64 argument and then
 # using FROM in the Dockerfile.
-FROM ghcr.io/astral-sh/uv:0.11.26 AS uv_extractor
+FROM ghcr.io/astral-sh/uv:0.13.4 AS uv_extractor
 
 
 # Stage 2: The main application build stage

{gitops-1.10.1 → gitops-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gitops
-Version: 1.10.1
+Version: 1.11.0
 Summary: Manage multiple apps across one or more k8s clusters.
 Author-email: Jarek Głowacki <jarekwg@gmail.com>
 License-File: LICENSE
@@ -106,6 +106,8 @@ subjects:
   namespace: gitops
 ```
 
+It also needs `patch` on pods in its own namespace, to set `karpenter.sh/do-not-disrupt` on itself while deploying. The `admin` or `edit` ClusterRole covers this.
+
 ## Contribution
 We're using [releaseplease](https://github.com/googleapis/release-please), to publish a new version do the following:
 1. Checkout a feature branch and make the changes

{gitops-1.10.1 → gitops-1.11.0}/README.md

@@ -73,6 +73,8 @@ subjects:
   namespace: gitops
 ```
 
+It also needs `patch` on pods in its own namespace, to set `karpenter.sh/do-not-disrupt` on itself while deploying. The `admin` or `edit` ClusterRole covers this.
+
 ## Contribution
 We're using [releaseplease](https://github.com/googleapis/release-please), to publish a new version do the following:
 1. Checkout a feature branch and make the changes

{gitops-1.10.1 → gitops-1.11.0}/charts/gitops/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [1.5.0](https://github.com/uptick/gitops/compare/helm-v1.4.1...helm-v1.5.0) (2026-10-27)
+
+
+### Features
+
+* **server:** hold karpenter.sh/do-not-disrupt while deploying ([b977e5f](https://github.com/uptick/gitops/commit/b977e5f949b237c5b458f6b9e3029f02592e55a8))
+* **server:** hold karpenter.sh/do-not-disrupt while deploying ([c14359f](https://github.com/uptick/gitops/commit/c14359f39ba354432378bdb2ad9963e442fc9f66))
+
 ## [1.4.1](https://github.com/uptick/gitops/compare/helm-v1.4.0...helm-v1.4.1) (2026-10-21)
 
 

{gitops-1.10.1 → gitops-1.11.0}/charts/gitops/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v1
 appVersion: "1.0"
 description: GitOps Server Helm chart.
 name: gitops
-version: 1.4.1
+version: 1.5.0

{gitops-1.10.1 → gitops-1.11.0}/charts/gitops/templates/deployment.yml

@@ -78,6 +78,15 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: status.hostIP
+          # The server annotates its own pod while deploying.
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: OTEL_EXPORTER_OTLP_ENDPOINT
             value: "http://$(NODE_IP):4317"
           - name: OTEL_PYTHON_LOG_CORRELATION

{gitops-1.10.1 → gitops-1.11.0}/gitops/__init__.py

@@ -6,7 +6,7 @@ from gitops.utils.apps import App, get_app_details, get_apps  # type: ignore[att
 
 from .utils.cli import success, warning
 
-__version__ = "1.10.1"
+__version__ = "1.11.0"
 
 
 # Checking gitops version matches cluster repo version.

{gitops-1.10.1 → gitops-1.11.0}/gitops_server/app.py

@@ -18,6 +18,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     """Manage application lifespan - start and stop background workers."""
     from .utils.github_auth import credentials, validate_credentials
     from .workers import DeployQueueWorker
+    from .workers.deployer.disruption import build_disruption_guard
 
     # Fail here rather than on the first push, where hooks.py only logs a warning.
     validate_credentials()
@@ -25,13 +26,17 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
 
     loop = asyncio.get_running_loop()
     deploy_queue_worker = DeployQueueWorker.get_worker()
-    deploy_queue_worker_task = asyncio.ensure_future(deploy_queue_worker.run(), loop=loop)
+    tasks = [asyncio.ensure_future(deploy_queue_worker.run(), loop=loop)]
+    if guard := build_disruption_guard(deploy_queue_worker.is_busy):
+        deploy_queue_worker.on_activity = guard.poke
+        tasks.append(asyncio.ensure_future(guard.run(), loop=loop))
     yield
-    deploy_queue_worker_task.cancel()
-    try:
-        await deploy_queue_worker_task
-    except asyncio.CancelledError:
-        pass
+    for task in tasks:
+        task.cancel()
+        try:
+            await task
+        except asyncio.CancelledError:
+            pass
 
 
 app = FastAPI(lifespan=lifespan)

{gitops-1.10.1 → gitops-1.11.0}/gitops_server/settings.py

@@ -22,6 +22,14 @@ GITOPS_MAX_PARALLEL_DEPLOYS = os.environ.get("GITOPS_MAX_PARALLEL_DEPLOYS", "5")
 # here a burst of pushes is only limited by how fast they arrive.
 GITOPS_MAX_CONCURRENT_DEPLOYMENTS = os.environ.get("GITOPS_MAX_CONCURRENT_DEPLOYMENTS", "20")
 
+# Set by the chart from the downward API. Empty outside Kubernetes, which turns
+# off managing karpenter.sh/do-not-disrupt.
+POD_NAME = os.environ.get("POD_NAME", "")
+POD_NAMESPACE = os.environ.get("POD_NAMESPACE", "")
+
+# Report to Sentry once karpenter.sh/do-not-disrupt has been held this long without a break.
+GITOPS_DO_NOT_DISRUPT_WARN_SECONDS = os.environ.get("GITOPS_DO_NOT_DISRUPT_WARN_SECONDS", "10800")
+
 # Pass --disable-openapi-validation to helm. Off by default: helm keeps checking
 # manifests against the cluster schema before applying them, which costs each
 # helm process a download and parse of that schema. The API server still

gitops-1.11.0/gitops_server/version.py

@@ -0,0 +1 @@
+__version__ = "1.11.0"

gitops-1.11.0/gitops_server/workers/deployer/disruption.py

@@ -0,0 +1,97 @@
+import asyncio
+import logging
+import time
+from collections.abc import Awaitable, Callable
+
+from kubernetes_asyncio import client, config  # type: ignore[import-untyped]
+
+from gitops_server import settings
+
+logger = logging.getLogger("gitops_worker")
+
+ANNOTATION = "karpenter.sh/do-not-disrupt"
+RECONCILE_INTERVAL_SECONDS = 30
+
+Patcher = Callable[[bool], Awaitable[None]]
+
+
+class DisruptionGuard:
+    """Holds `karpenter.sh/do-not-disrupt` on this pod while deploys are in flight.
+
+    Karpenter won't drain the node for consolidation or drift while the annotation is
+    set, so a release isn't cut off mid helm upgrade.
+    """
+
+    def __init__(self, is_busy: Callable[[], bool], patch: Patcher, interval: float, warn_after: float) -> None:
+        self.is_busy = is_busy
+        self.patch = patch
+        self.interval = interval
+        self.warn_after = warn_after
+        # Unknown until the first patch: a container restart keeps the pod and any annotation on it.
+        self.applied: bool | None = None
+        self.busy_since: float | None = None
+        self.warned = False
+        self._changed = asyncio.Event()
+
+    def poke(self) -> None:
+        """Reconcile now instead of at the next interval."""
+        self._changed.set()
+
+    def check_hold_time(self, now: float) -> None:
+        if not self.is_busy():
+            self.busy_since = None
+            self.warned = False
+            return
+        if self.busy_since is None:
+            self.busy_since = now
+        # Error level so Sentry raises it: a stuck annotation pins the node.
+        if not self.warned and now - self.busy_since >= self.warn_after:
+            logger.error(f"{ANNOTATION} has been held for over {self.warn_after:.0f}s of continuous deploys.")
+            self.warned = True
+
+    async def reconcile(self) -> None:
+        self.check_hold_time(time.monotonic())
+        want = self.is_busy()
+        if want == self.applied:
+            return
+        try:
+            await self.patch(want)
+        except Exception:
+            # `applied` stays as it was, so the next pass retries.
+            logger.exception(f"Failed to {'set' if want else 'clear'} {ANNOTATION}.")
+            return
+        self.applied = want
+
+    async def run(self) -> None:
+        while True:
+            self._changed.clear()
+            await self.reconcile()
+            try:
+                await asyncio.wait_for(self._changed.wait(), timeout=self.interval)
+            except TimeoutError:
+                pass
+
+
+def pod_annotation_patcher(pod_name: str, namespace: str) -> Patcher:
+    config.load_incluster_config()
+    api = client.CoreV1Api(client.ApiClient())
+
+    async def patch(hold: bool) -> None:
+        # In a merge patch, null deletes the key.
+        body = {"metadata": {"annotations": {ANNOTATION: "true" if hold else None}}}
+        await api.patch_namespaced_pod(pod_name, namespace, body, _content_type="application/merge-patch+json")
+        logger.info(f"{'Set' if hold else 'Cleared'} {ANNOTATION} on pod {pod_name!r}.")
+
+    return patch
+
+
+def build_disruption_guard(is_busy: Callable[[], bool]) -> DisruptionGuard | None:
+    if not (settings.POD_NAME and settings.POD_NAMESPACE):
+        logger.info(f"POD_NAME or POD_NAMESPACE unset; not managing {ANNOTATION}.")
+        return None
+    return DisruptionGuard(
+        is_busy,
+        pod_annotation_patcher(settings.POD_NAME, settings.POD_NAMESPACE),
+        interval=RECONCILE_INTERVAL_SECONDS,
+        warn_after=float(settings.GITOPS_DO_NOT_DISRUPT_WARN_SECONDS),
+    )

{gitops-1.10.1 → gitops-1.11.0}/gitops_server/workers/deployer/worker.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+from collections.abc import Callable
 from typing import Any
 
 from opentelemetry import trace
@@ -38,6 +39,12 @@ class DeployQueueWorker:
         # gates the helm stage. Everything before it -- loading app definitions
         # for two commits -- was previously limited only by webhook arrival rate.
         self.deployment_semaphore = asyncio.Semaphore(int(settings.GITOPS_MAX_CONCURRENT_DEPLOYMENTS))
+        # Called when work is queued or a deployment finishes.
+        self.on_activity: Callable[[], None] = lambda: None
+
+    def is_busy(self) -> bool:
+        # active_deployments is only pruned on the next dequeue, so it can hold finished tasks.
+        return self.queue.qsize() > 0 or any(not task.done() for task in self.active_deployments)
 
     async def enqueue(self, work: Any) -> None:
         """Enqueue an item of work for future processing.
@@ -46,6 +53,7 @@ class DeployQueueWorker:
         """
         logger.info(f"Enqueued work, {self.queue.qsize() + 1} items in the queue.")
         await self.queue.put(work)
+        self.on_activity()
 
     async def run(self) -> None:
         """Run the worker.
@@ -96,6 +104,7 @@ class DeployQueueWorker:
             await self._process_deployment(work, ref)
         finally:
             self.deployment_semaphore.release()
+            self.on_activity()
 
     async def _process_deployment(self, work: Any, ref: str | None) -> None:
         if ref == "refs/heads/master":

{gitops-1.10.1 → gitops-1.11.0}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "gitops"
-version = "1.10.1"
+version = "1.11.0"
 description = "Manage multiple apps across one or more k8s clusters."
 requires-python = ">=3.12"
 readme = "README.md"

gitops-1.11.0/tests/test_disruption_guard.py

@@ -0,0 +1,114 @@
+import asyncio
+from unittest import mock
+
+import pytest
+
+from gitops_server.workers.deployer.disruption import DisruptionGuard
+from gitops_server.workers.deployer.worker import DeployQueueWorker
+
+
+class FakePatcher:
+    def __init__(self) -> None:
+        self.calls: list[bool] = []
+        self.fail_next = False
+
+    async def __call__(self, hold: bool) -> None:
+        if self.fail_next:
+            self.fail_next = False
+            raise RuntimeError("apiserver unavailable")
+        self.calls.append(hold)
+
+
+def make_guard(busy: list[bool], warn_after: float = 3600) -> tuple[DisruptionGuard, FakePatcher]:
+    patcher = FakePatcher()
+    guard = DisruptionGuard(lambda: busy[0], patcher, interval=30, warn_after=warn_after)
+    return guard, patcher
+
+
+@pytest.mark.asyncio
+async def test_annotation_follows_busy_state():
+    busy = [False]
+    guard, patcher = make_guard(busy)
+
+    await guard.reconcile()
+    await guard.reconcile()
+    assert patcher.calls == [False], "first pass clears anything left by a previous process"
+
+    busy[0] = True
+    await guard.reconcile()
+    await guard.reconcile()
+    busy[0] = False
+    await guard.reconcile()
+
+    assert patcher.calls == [False, True, False]
+
+
+@pytest.mark.asyncio
+async def test_failed_patch_is_retried():
+    busy = [True]
+    guard, patcher = make_guard(busy)
+    patcher.fail_next = True
+
+    await guard.reconcile()
+    assert guard.applied is None
+    await guard.reconcile()
+
+    assert patcher.calls == [True]
+    assert guard.applied
+
+
+@pytest.mark.asyncio
+async def test_long_hold_reported_once_per_run(caplog):
+    busy = [True]
+    guard, patcher = make_guard(busy, warn_after=60)
+    clock = "gitops_server.workers.deployer.disruption.time.monotonic"
+
+    def errors() -> int:
+        return sum(r.levelname == "ERROR" for r in caplog.records)
+
+    with mock.patch(clock, return_value=0):
+        await guard.reconcile()
+    with mock.patch(clock, return_value=61):
+        await guard.reconcile()
+        await guard.reconcile()
+    assert errors() == 1
+    assert patcher.calls == [True], "a long hold is reported, not released"
+
+    busy[0] = False
+    with mock.patch(clock, return_value=62):
+        await guard.reconcile()
+    busy[0] = True
+    with mock.patch(clock, return_value=63):
+        await guard.reconcile()
+    with mock.patch(clock, return_value=124):
+        await guard.reconcile()
+    assert errors() == 2, "a fresh run of deploys can report again"
+
+
+@pytest.mark.asyncio
+async def test_worker_busy_until_deployment_finishes():
+    release = asyncio.Event()
+
+    async def slow_deployment(work, ref):
+        await release.wait()
+
+    worker = DeployQueueWorker()
+    worker._process_deployment = slow_deployment  # type: ignore[method-assign]
+    activity = mock.Mock()
+    worker.on_activity = activity
+
+    assert not worker.is_busy()
+    runner = asyncio.create_task(worker.run())
+    await worker.enqueue({"ref": "refs/heads/master"})
+    for _ in range(5):
+        await asyncio.sleep(0)
+    assert worker.is_busy()
+
+    release.set()
+    for _ in range(5):
+        await asyncio.sleep(0)
+    # The finished task is still in active_deployments until the next dequeue.
+    assert worker.active_deployments
+    assert not worker.is_busy()
+    assert activity.call_count == 2, "once on enqueue, once on finish"
+    runner.cancel()

{gitops-1.10.1 → gitops-1.11.0}/uv.lock

@@ -536,7 +536,7 @@ wheels = [
 
 [[package]]
 name = "gitops"
-version = "1.10.1"
+version = "1.11.0"
 source = { editable = "." }
 dependencies = [
     { name = "boto3" },

gitops-1.10.1/.release-please-manifest.json

@@ -1,5 +0,0 @@
-{
-    ".": "1.10.1",
-    "gitops_server": "0.15.1",
-    "charts/gitops": "1.4.1"
-}

gitops-1.10.1/gitops_server/version.py

@@ -1 +0,0 @@
-__version__ = "1.10.1"