gitops 1.11.0__tar.gz → 1.12.0__tar.gz

gitops-1.12.0/.release-please-manifest.json

@@ -0,0 +1,5 @@
+{
+    ".": "1.12.0",
+    "gitops_server": "0.15.1",
+    "charts/gitops": "1.6.0"
+}

{gitops-1.11.0 → gitops-1.12.0}/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [1.12.0](https://github.com/uptick/gitops/compare/gitops-v1.11.0...gitops-v1.12.0) (2026-11-02)
+
+
+### Features
+
+* **server:** finish running helm upgrades on shutdown ([92a7952](https://github.com/uptick/gitops/commit/92a7952f91b90543ed573d8c96d8853ff8f7caf5))
+* **server:** finish running helm upgrades on shutdown ([ef16fd2](https://github.com/uptick/gitops/commit/ef16fd21ae53cba6987aecc4455ffdf6b254f4e1))
+* **server:** report work dropped at shutdown, at ERROR by default ([eb2717b](https://github.com/uptick/gitops/commit/eb2717b3ead89abdcffe37e0b14cfd9688ed9994))
+
+
+### Bug Fixes
+
+* **deps:** upgrade botocore to 1.43.117 ([7d1e151](https://github.com/uptick/gitops/commit/7d1e1517770c5f7ffa66a883a6e860852405c011))
+* **deps:** upgrade botocore to 1.43.117 ([77e7bcf](https://github.com/uptick/gitops/commit/77e7bcfbc1d7f46124d486f67cb35a9d3ac18467))
+
 ## [1.11.0](https://github.com/uptick/gitops/compare/gitops-v1.10.2...gitops-v1.11.0) (2026-10-27)
 
 

{gitops-1.11.0 → gitops-1.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gitops
-Version: 1.11.0
+Version: 1.12.0
 Summary: Manage multiple apps across one or more k8s clusters.
 Author-email: Jarek Głowacki <jarekwg@gmail.com>
 License-File: LICENSE

{gitops-1.11.0 → gitops-1.12.0}/charts/gitops/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [1.6.0](https://github.com/uptick/gitops/compare/helm-v1.5.0...helm-v1.6.0) (2026-11-02)
+
+
+### Features
+
+* **server:** finish running helm upgrades on shutdown ([92a7952](https://github.com/uptick/gitops/commit/92a7952f91b90543ed573d8c96d8853ff8f7caf5))
+* **server:** finish running helm upgrades on shutdown ([ef16fd2](https://github.com/uptick/gitops/commit/ef16fd21ae53cba6987aecc4455ffdf6b254f4e1))
+
 ## [1.5.0](https://github.com/uptick/gitops/compare/helm-v1.4.1...helm-v1.5.0) (2026-10-27)
 
 

{gitops-1.11.0 → gitops-1.12.0}/charts/gitops/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v1
 appVersion: "1.0"
 description: GitOps Server Helm chart.
 name: gitops
-version: 1.5.0
+version: 1.6.0

{gitops-1.11.0 → gitops-1.12.0}/charts/gitops/templates/deployment.yml

@@ -24,6 +24,8 @@ spec:
     spec:
       serviceAccountName: {{ include "uptick.serviceAccountName" . }}
       priorityClassName: system-node-critical
+      # Must exceed GITOPS_SHUTDOWN_TIMEOUT_SECONDS, or running helm upgrades are killed.
+      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
       containers:
       - name: gitops
         image: {{ .Values.image }}

{gitops-1.11.0 → gitops-1.12.0}/charts/gitops/values.yaml

@@ -44,6 +44,8 @@ podDisruptionBudget: {}
 resources:
   requests:
     memory: 750Mi
+
+terminationGracePeriodSeconds: 720
 serviceAccount:
   # Specifies whether a service account should be created
   create: true

{gitops-1.11.0 → gitops-1.12.0}/gitops/__init__.py

@@ -6,7 +6,7 @@ from gitops.utils.apps import App, get_app_details, get_apps  # type: ignore[att
 
 from .utils.cli import success, warning
 
-__version__ = "1.11.0"
+__version__ = "1.12.0"
 
 
 # Checking gitops version matches cluster repo version.

{gitops-1.11.0 → gitops-1.12.0}/gitops_server/app.py

@@ -26,17 +26,24 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
 
     loop = asyncio.get_running_loop()
     deploy_queue_worker = DeployQueueWorker.get_worker()
-    tasks = [asyncio.ensure_future(deploy_queue_worker.run(), loop=loop)]
+    worker_task = asyncio.ensure_future(deploy_queue_worker.run(), loop=loop)
+    guard_task = None
     if guard := build_disruption_guard(deploy_queue_worker.is_busy):
         deploy_queue_worker.on_activity = guard.poke
-        tasks.append(asyncio.ensure_future(guard.run(), loop=loop))
+        guard_task = asyncio.ensure_future(guard.run(), loop=loop)
     yield
-    for task in tasks:
-        task.cancel()
-        try:
-            await task
-        except asyncio.CancelledError:
-            pass
+    await _cancel(worker_task)
+    await deploy_queue_worker.drain(float(settings.GITOPS_SHUTDOWN_TIMEOUT_SECONDS))
+    if guard_task:
+        await _cancel(guard_task)
+
+
+async def _cancel(task: asyncio.Future[None]) -> None:
+    task.cancel()
+    try:
+        await task
+    except asyncio.CancelledError:
+        pass
 
 
 app = FastAPI(lifespan=lifespan)

{gitops-1.11.0 → gitops-1.12.0}/gitops_server/settings.py

@@ -30,6 +30,14 @@ POD_NAMESPACE = os.environ.get("POD_NAMESPACE", "")
 # Report to Sentry once karpenter.sh/do-not-disrupt has been held this long without a break.
 GITOPS_DO_NOT_DISRUPT_WARN_SECONDS = os.environ.get("GITOPS_DO_NOT_DISRUPT_WARN_SECONDS", "10800")
 
+# How long shutdown waits for running helm upgrades. Covers helm's 600s --timeout;
+# keep it below the chart's terminationGracePeriodSeconds.
+GITOPS_SHUTDOWN_TIMEOUT_SECONDS = os.environ.get("GITOPS_SHUTDOWN_TIMEOUT_SECONDS", "660")
+
+# Log level for work shutdown drops: skipped apps, queued pushes, upgrades still running.
+# ERROR and above reach Sentry.
+GITOPS_SHUTDOWN_DROPPED_LOG_LEVEL = os.environ.get("GITOPS_SHUTDOWN_DROPPED_LOG_LEVEL", "ERROR")
+
 # Pass --disable-openapi-validation to helm. Off by default: helm keeps checking
 # manifests against the cluster schema before applying them, which costs each
 # helm process a download and parse of that schema. The API server still

gitops-1.12.0/gitops_server/version.py

@@ -0,0 +1 @@
+__version__ = "1.12.0"

{gitops-1.11.0 → gitops-1.12.0}/gitops_server/workers/deployer/deploy.py

@@ -33,6 +33,11 @@ MAX_HELM_HISTORY = 3
 # Kube api may rate limit otherwise
 helm_parallel_semaphore = asyncio.Semaphore(int(settings.GITOPS_MAX_PARALLEL_DEPLOYS))
 
+# Set on SIGTERM. Apps that haven't started their helm upgrade are skipped, so the
+# drain waits for at most one upgrade rather than a whole push.
+shutting_down = asyncio.Event()
+DROPPED_LOG_LEVEL = logging.getLevelNamesMapping()[settings.GITOPS_SHUTDOWN_DROPPED_LOG_LEVEL.upper()]
+
 
 @tracer.start_as_current_span("post_result_summary")
 async def post_result_summary(source: str, results: list[UpdateAppResult]) -> None:
@@ -138,6 +143,7 @@ class Deployer:
         # Track which ones we've deployed and which ones failed
         self.successful_apps: set[str] = set()
         self.failed_apps: set[str] = set()
+        self.skipped_on_shutdown: list[str] = []
 
         self.post_ts: str | None = None
         self.skip_deploy = "--skip-deploy" in commit_message
@@ -195,6 +201,11 @@ class Deployer:
             *[self.uninstall_app(self.previous_app_definitions.apps[app_name]) for app_name in self.removed_apps]
         )
         without_nones = [r for r in update_results + uninstall_results if r is not None]
+        if self.skipped_on_shutdown:
+            # Not posting the result summary: skipped apps would be missing from the counts.
+            # Args, not an f-string, so Sentry groups these into one issue.
+            logger.log(DROPPED_LOG_LEVEL, "Shutdown skipped apps: %s", sorted(self.skipped_on_shutdown))
+            return
         await post_result_summary(self.current_app_definitions.name, without_nones)
 
     async def uninstall_app(self, app: App) -> UpdateAppResult:
@@ -232,6 +243,9 @@ class Deployer:
         from_timestamp = time.time()
         async with self.semaphore_manager.app_semaphore(app.name):
             async with helm_parallel_semaphore:
+                if shutting_down.is_set():
+                    self.skipped_on_shutdown.append(app.name)
+                    return None
                 try:
                     return await self._update_app_deployment(app)
                 except Exception as e:

{gitops-1.11.0 → gitops-1.12.0}/gitops_server/workers/deployer/worker.py

@@ -6,7 +6,7 @@ from typing import Any
 from opentelemetry import trace
 
 from ... import settings
-from .deploy import Deployer
+from .deploy import DROPPED_LOG_LEVEL, Deployer, shutting_down
 from .semaphore_manager import AppSemaphoreManager
 
 logger = logging.getLogger("gitops_worker")
@@ -91,6 +91,22 @@ class DeployQueueWorker:
             except Exception as e:
                 logger.error(str(e), exc_info=True)
 
+    async def drain(self, timeout: float) -> None:
+        """Wait up to `timeout` for running deployments, skipping apps not yet started.
+
+        Cancel `run()` first, or it keeps dequeuing.
+        """
+        shutting_down.set()
+        if queued := self.queue.qsize():
+            logger.log(DROPPED_LOG_LEVEL, "Shutting down with %d pushes still queued.", queued)
+        running = [task for task in self.active_deployments if not task.done()]
+        if not running:
+            return
+        logger.info(f"Waiting up to {timeout:.0f}s for {len(running)} deployments to finish.")
+        _, pending = await asyncio.wait(running, timeout=timeout)
+        if pending:
+            logger.log(DROPPED_LOG_LEVEL, "%d deployments still running at shutdown.", len(pending))
+
     async def process_deployment(self, work: Any) -> None:
         """Process a single deployment in a separate task.
 

{gitops-1.11.0 → gitops-1.12.0}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "gitops"
-version = "1.11.0"
+version = "1.12.0"
 description = "Manage multiple apps across one or more k8s clusters."
 requires-python = ">=3.12"
 readme = "README.md"

gitops-1.12.0/tests/test_shutdown.py

@@ -0,0 +1,78 @@
+import asyncio
+from unittest.mock import patch
+
+import pytest
+
+from gitops_server.workers.deployer import Deployer
+from gitops_server.workers.deployer.deploy import shutting_down
+from gitops_server.workers.deployer.semaphore_manager import AppSemaphoreManager
+from gitops_server.workers.deployer.worker import DeployQueueWorker
+
+from .sample_data import SAMPLE_GITHUB_PAYLOAD
+from .utils import mock_load_app_definitions
+
+
+@pytest.fixture(autouse=True)
+def reset_shutdown():
+    AppSemaphoreManager._instance = None
+    yield
+    shutting_down.clear()
+
+
+async def start_worker(deployment) -> tuple[DeployQueueWorker, asyncio.Task]:
+    worker = DeployQueueWorker()
+    worker._process_deployment = deployment  # type: ignore[method-assign]
+    runner = asyncio.create_task(worker.run())
+    await worker.enqueue({"ref": "refs/heads/master"})
+    for _ in range(5):
+        await asyncio.sleep(0)
+    return worker, runner
+
+
+@pytest.mark.asyncio
+async def test_drain_waits_for_running_deployment():
+    finished = asyncio.Event()
+
+    async def deployment(work, ref):
+        await asyncio.sleep(0.05)
+        finished.set()
+
+    worker, runner = await start_worker(deployment)
+    runner.cancel()
+    await worker.drain(timeout=5)
+
+    assert finished.is_set()
+    assert shutting_down.is_set()
+
+
+@pytest.mark.asyncio
+async def test_drain_gives_up_after_timeout():
+    async def deployment(work, ref):
+        await asyncio.Event().wait()
+
+    worker, runner = await start_worker(deployment)
+    runner.cancel()
+    await asyncio.wait_for(worker.drain(timeout=0.05), timeout=1)
+
+    assert worker.is_busy(), "the stuck deployment is left running"
+
+
+@pytest.mark.asyncio
+@patch("gitops_server.workers.deployer.deploy.run")
+@patch("gitops_server.workers.deployer.deploy.post_result_summary")
+@patch("gitops_server.utils.slack.post")
+@patch("gitops_server.workers.deployer.deploy.load_app_definitions", mock_load_app_definitions)
+@patch("gitops_server.workers.deployer.deploy.temp_repo")
+async def test_apps_not_started_are_skipped_once_shutting_down(
+    temp_repo_mock, post_mock, summary_mock, run_mock, caplog
+):
+    deployer = await Deployer.from_push_event(SAMPLE_GITHUB_PAYLOAD, AppSemaphoreManager())
+    shutting_down.set()
+
+    await deployer.deploy()
+
+    run_mock.assert_not_called()
+    summary_mock.assert_not_called()
+    [record] = [r for r in caplog.records if r.levelname == "ERROR"]
+    assert record.args == (sorted(deployer.skipped_on_shutdown),)
+    assert deployer.skipped_on_shutdown, "the error names the apps to redeploy"

{gitops-1.11.0 → gitops-1.12.0}/uv.lock

@@ -222,16 +222,16 @@ wheels = [
 
 [[package]]
 name = "boto3"
-version = "1.35.4"
+version = "1.43.117"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "botocore" },
     { name = "jmespath" },
     { name = "s3transfer" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/ca/59/baeef26b73a6fd71602370b6e1ba3eb79d0d43dff6efcf552065548c49a0/boto3-1.35.4.tar.gz", hash = "sha256:d997b82c468bd5c2d5cd29810d47079b66b178d2b5ae021aebe262c4d78d4c94", size = 108591, upload-time = "2024-08-22T20:02:16.884Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/94/a7/61f0295f79dfdcebb5aa01f86c614fac7e1294faabbd66259d3609c50539/boto3-1.43.117.tar.gz", hash = "sha256:f3d314908d866b577ee6b74ed432c21e565c4c4dbd640ef3fd5e0da30fa13a98", size = 112677, upload-time = "2026-10-21T19:30:03.849Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/76/88/73608e1f1c4f4276213a0d8658cfe799a3b2d6e4189543676b8b852a788e/boto3-1.35.4-py3-none-any.whl", hash = "sha256:96c39593afb7b55ebb74d08c8e3201041d105b557c8c8536c9054c9f13da5f2a", size = 139142, upload-time = "2024-08-22T20:02:13.231Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/3b/22b05a545fb95983733d84681364c6b0a36afb0710d832f77732d1fc4c0b/boto3-1.43.117-py3-none-any.whl", hash = "sha256:a5b6cd7e67653987fcf0e212187c16f7b58af2ae192e368f7212ddab7c05b25d", size = 140043, upload-time = "2026-10-21T19:30:02.114Z" },
 ]
 
 [[package]]
@@ -249,16 +249,16 @@ wheels = [
 
 [[package]]
 name = "botocore"
-version = "1.35.99"
+version = "1.43.117"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "jmespath" },
     { name = "python-dateutil" },
     { name = "urllib3" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/7c/9c/1df6deceee17c88f7170bad8325aa91452529d683486273928eecfd946d8/botocore-1.35.99.tar.gz", hash = "sha256:1eab44e969c39c5f3d9a3104a0836c24715579a455f12b3979a31d7cde51b3c3", size = 13490969, upload-time = "2025-01-14T20:20:11.419Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/6e/a0/d0a5c44ad6a01f195681ddcb4ce203f06c712811daf1d15f8f8785e8a9f8/botocore-1.43.117.tar.gz", hash = "sha256:71993702832cc37c2495048b9945f2929d306fccb3fb767e9964e437e637fa6c", size = 16366888, upload-time = "2026-10-21T19:29:59.274Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/fc/dd/d87e2a145fad9e08d0ec6edcf9d71f838ccc7acdd919acc4c0d4a93515f8/botocore-1.35.99-py3-none-any.whl", hash = "sha256:b22d27b6b617fc2d7342090d6129000af2efd20174215948c0d7ae2da0fab445", size = 13293216, upload-time = "2025-01-14T20:20:06.427Z" },
+    { url = "https://files.pythonhosted.org/packages/10/94/050e35c9febf6dd2fce4a76be587ae4a323964865700467d9d7b082f7133/botocore-1.43.117-py3-none-any.whl", hash = "sha256:5920ef9b3002e1361846549078549d9154a6e0004c22853c01ce7b279d05d495", size = 16061765, upload-time = "2026-10-21T19:29:56.001Z" },
 ]
 
 [[package]]
@@ -536,7 +536,7 @@ wheels = [
 
 [[package]]
 name = "gitops"
-version = "1.11.0"
+version = "1.12.0"
 source = { editable = "." }
 dependencies = [
     { name = "boto3" },
@@ -1532,14 +1532,14 @@ wheels = [
 
 [[package]]
 name = "s3transfer"
-version = "0.10.4"
+version = "0.19.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "botocore" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/c0/0a/1cdbabf9edd0ea7747efdf6c9ab4e7061b085aa7f9bfc36bb1601563b069/s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7", size = 145287, upload-time = "2024-11-20T21:06:05.981Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/66/05/7957af15543b8c9799209506df4660cba7afc4cf94bfb60513827e96bed6/s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e", size = 83175, upload-time = "2024-11-20T21:06:03.961Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" },
 ]
 
 [[package]]

gitops-1.11.0/.release-please-manifest.json

@@ -1,5 +0,0 @@
-{
-    ".": "1.11.0",
-    "gitops_server": "0.15.1",
-    "charts/gitops": "1.5.0"
-}

gitops-1.11.0/gitops_server/version.py

@@ -1 +0,0 @@
-__version__ = "1.11.0"