gitops 1.4.1__tar.gz → 1.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitops-1.4.1 → gitops-1.5.1}/.release-please-manifest.json +1 -1
- {gitops-1.4.1 → gitops-1.5.1}/CHANGELOG.md +14 -0
- gitops-1.5.1/CLAUDE.md +101 -0
- {gitops-1.4.1 → gitops-1.5.1}/PKG-INFO +2 -2
- {gitops-1.4.1 → gitops-1.5.1}/gitops/__init__.py +1 -1
- {gitops-1.4.1 → gitops-1.5.1}/gitops/common/app.py +1 -1
- {gitops-1.4.1 → gitops-1.5.1}/gitops/core.py +1 -1
- {gitops-1.4.1 → gitops-1.5.1}/gitops/utils/async_runner.py +1 -1
- {gitops-1.4.1 → gitops-1.5.1}/gitops/utils/cli.py +1 -1
- {gitops-1.4.1 → gitops-1.5.1}/gitops/utils/kube.py +1 -1
- gitops-1.5.1/gitops_server/version.py +1 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/workers/deployer/deploy.py +79 -68
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/workers/deployer/hooks.py +1 -1
- gitops-1.5.1/gitops_server/workers/deployer/semaphore_manager.py +52 -0
- gitops-1.5.1/gitops_server/workers/deployer/worker.py +98 -0
- {gitops-1.4.1 → gitops-1.5.1}/pyproject.toml +2 -2
- {gitops-1.4.1 → gitops-1.5.1}/tests/test_deploy.py +13 -4
- gitops-1.5.1/tests/test_deploy_concurrent.py +255 -0
- {gitops-1.4.1 → gitops-1.5.1}/uv.lock +2 -2
- gitops-1.4.1/gitops_server/version.py +0 -1
- gitops-1.4.1/gitops_server/workers/deployer/worker.py +0 -67
- {gitops-1.4.1 → gitops-1.5.1}/.dockerignore +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/.github/workflows/release-please.yaml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/.github/workflows/test.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/.gitignore +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/.mise.toml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/.pre-commit-config.yaml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/Dockerfile +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/LICENSE +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/README.md +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/.helmignore +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/CHANGELOG.md +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/Chart.yaml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/README.md +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/_helpers.tpl +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/certificate.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/deployment.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/environment.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/ingress.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/issuer.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/pdb.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/secrets.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/service.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/templates/serviceaccount.yaml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/charts/gitops/values.yaml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/docker-compose.yml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/common/__init__.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/common/utils.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/main.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/py.typed +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/settings.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/shorthands.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/utils/__init__.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/utils/apps.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/utils/exceptions.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/utils/images.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/utils/tags.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops/utils/yaml.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/CHANGELOG.md +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/README.md +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/__init__.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/app.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/main.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/settings.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/types.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/utils/__init__.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/utils/git.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/utils/github.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/utils/slack.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/workers/__init__.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/gitops_server/workers/deployer/__init__.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/release-please-config.json +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/release.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/ruff.toml +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/secrets.example.env +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/setup.cfg +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/__init__.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/conftest.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/sample_data.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/test_app.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/test_core.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/test_git.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/test_gitops_server.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/test_utils.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/utils.py +0 -0
- {gitops-1.4.1 → gitops-1.5.1}/tests/webhook_sample_data.py +0 -0
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.5.1](https://github.com/uptick/gitops/compare/gitops-v1.5.0...gitops-v1.5.1) (2025-08-04)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* relaxing boto3 requirements ([75f82bb](https://github.com/uptick/gitops/commit/75f82bb8254546a037ea1055b8ebf404a919de45))
|
|
9
|
+
|
|
10
|
+
## [1.5.0](https://github.com/uptick/gitops/compare/gitops-v1.4.1...gitops-v1.5.0) (2025-07-23)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### Features
|
|
14
|
+
|
|
15
|
+
* implement concurrent deployments with per-app serialization ([6d3cc28](https://github.com/uptick/gitops/commit/6d3cc2827c75791aa10d6e12cb60192ed26129d4))
|
|
16
|
+
|
|
3
17
|
## [1.4.1](https://github.com/uptick/gitops/compare/gitops-v1.4.0...gitops-v1.4.1) (2025-06-04)
|
|
4
18
|
|
|
5
19
|
|
gitops-1.5.1/CLAUDE.md
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
+
|
|
5
|
+
## Project Overview
|
|
6
|
+
|
|
7
|
+
This is a GitOps tool for managing multiple applications across Kubernetes clusters. It consists of two main components:
|
|
8
|
+
|
|
9
|
+
1. **GitOps CLI** (`gitops/`) - Command-line tool for managing app deployments, built using the `invoke` framework
|
|
10
|
+
2. **GitOps Server** (`gitops_server/`) - FastAPI-based webhook server that handles deployment automation
|
|
11
|
+
|
|
12
|
+
## Development Commands
|
|
13
|
+
|
|
14
|
+
### Setup
|
|
15
|
+
```bash
|
|
16
|
+
# Install dependencies using uv
|
|
17
|
+
uv sync
|
|
18
|
+
|
|
19
|
+
# Install the CLI tool for development
|
|
20
|
+
uv tool install -e .
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
### Testing
|
|
24
|
+
```bash
|
|
25
|
+
# Run all tests
|
|
26
|
+
uv run pytest
|
|
27
|
+
|
|
28
|
+
# Run specific test file
|
|
29
|
+
uv run pytest tests/test_core.py
|
|
30
|
+
|
|
31
|
+
# Run with coverage
|
|
32
|
+
uv run pytest --cov=gitops --cov=gitops_server
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### Linting and Formatting
|
|
36
|
+
```bash
|
|
37
|
+
# Format code with ruff
|
|
38
|
+
uv run ruff format .
|
|
39
|
+
|
|
40
|
+
# Lint code
|
|
41
|
+
uv run ruff check .
|
|
42
|
+
|
|
43
|
+
# Fix auto-fixable linting issues
|
|
44
|
+
uv run ruff check --fix .
|
|
45
|
+
|
|
46
|
+
# Type checking with mypy
|
|
47
|
+
uv run mypy gitops/ gitops_server/
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### Running the CLI
|
|
51
|
+
```bash
|
|
52
|
+
# Run gitops commands during development
|
|
53
|
+
uv run python -m gitops.main --help
|
|
54
|
+
|
|
55
|
+
# Or if installed globally
|
|
56
|
+
gitops --help
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Running the Server
|
|
60
|
+
```bash
|
|
61
|
+
# Run the gitops server locally
|
|
62
|
+
uv run uvicorn gitops_server.main:app --reload
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Architecture
|
|
66
|
+
|
|
67
|
+
### CLI Component (`gitops/`)
|
|
68
|
+
- **main.py**: Entry point using invoke's Program framework
|
|
69
|
+
- **core.py**: Main CLI commands (bump, summary, etc.)
|
|
70
|
+
- **shorthands.py**: Shorthand command aliases
|
|
71
|
+
- **settings.py**: Configuration management
|
|
72
|
+
- **utils/**: Helper modules for apps, images, kubernetes operations, CLI formatting, etc.
|
|
73
|
+
|
|
74
|
+
### Server Component (`gitops_server/`)
|
|
75
|
+
- **main.py**: FastAPI application entry point with webhook endpoint
|
|
76
|
+
- **app.py**: FastAPI app configuration
|
|
77
|
+
- **workers/**: Background workers for deployment operations
|
|
78
|
+
- **workers/deploy.py**: Main deployment loop code
|
|
79
|
+
- **workers/worker.py**: Entry point for recieving webhook
|
|
80
|
+
- **utils/**: Git, GitHub, and Slack integration utilities
|
|
81
|
+
|
|
82
|
+
### Key Patterns
|
|
83
|
+
- Uses `invoke` framework for CLI task definitions
|
|
84
|
+
- Async operations with progress bars for CLI commands
|
|
85
|
+
- FastAPI with webhook validation for server component
|
|
86
|
+
- Kubernetes operations through the `kubernetes_asyncio` library
|
|
87
|
+
- Git operations for managing cluster repository state
|
|
88
|
+
|
|
89
|
+
## Configuration
|
|
90
|
+
|
|
91
|
+
- **Environment Variables**: Set `GITOPS_APPS_DIRECTORY` to specify the apps directory location
|
|
92
|
+
- **Secrets**: Server configuration goes in `secrets.env` (see `secrets.example.env`)
|
|
93
|
+
- **Ruff Config**: Located in `ruff.toml` with specific rules for this codebase
|
|
94
|
+
- **Dependencies**: Managed through `pyproject.toml` with separate server optional dependencies
|
|
95
|
+
|
|
96
|
+
## Testing Structure
|
|
97
|
+
|
|
98
|
+
- Tests are in the `tests/` directory
|
|
99
|
+
- Uses pytest with async support (`pytest-asyncio`)
|
|
100
|
+
- Sample data and webhook fixtures in `sample_data.py` and `webhook_sample_data.py`
|
|
101
|
+
- Conftest provides shared test configuration
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitops
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.5.1
|
|
4
4
|
Summary: Manage multiple apps across one or more k8s clusters.
|
|
5
5
|
Author-email: Jarek Głowacki <jarekwg@gmail.com>
|
|
6
6
|
License-File: LICENSE
|
|
7
7
|
Requires-Python: >=3.12
|
|
8
|
-
Requires-Dist: boto3
|
|
8
|
+
Requires-Dist: boto3>=1.35.4
|
|
9
9
|
Requires-Dist: colorama>=0.4.4
|
|
10
10
|
Requires-Dist: dsnparse==0.2.1
|
|
11
11
|
Requires-Dist: humanize>=3.5.0
|
|
@@ -107,7 +107,7 @@ class App:
|
|
|
107
107
|
"""[610829907584.dkr.ecr.ap-southeast-2.amazonaws.com/uptick:yoink-9f03ac80f3]"""
|
|
108
108
|
image = self.values.get("image", "")
|
|
109
109
|
if isinstance(image, dict):
|
|
110
|
-
return f"{image['repository']}:{image.get('tag','latest')}"
|
|
110
|
+
return f"{image['repository']}:{image.get('tag', 'latest')}"
|
|
111
111
|
else:
|
|
112
112
|
return image
|
|
113
113
|
|
|
@@ -107,7 +107,7 @@ def bump( # noqa: C901
|
|
|
107
107
|
print(f"Redeploying {colourise(app_name, Fore.LIGHTGREEN_EX)}")
|
|
108
108
|
update_app(app_name, **{"bump": str(uuid.uuid4())})
|
|
109
109
|
else:
|
|
110
|
-
print(f"Skipping {colourise(app_name, Fore.LIGHTGREEN_EX)}: already on
|
|
110
|
+
print(f"Skipping {colourise(app_name, Fore.LIGHTGREEN_EX)}: already on {colour_image(new_image_tag)}")
|
|
111
111
|
if redeploy:
|
|
112
112
|
commit_message = f"Redeploying {filter}"
|
|
113
113
|
else:
|
|
@@ -62,7 +62,7 @@ async def run_tasks_async_with_progress(tasks, max_concurrency=10):
|
|
|
62
62
|
async def print_async_complete(task, x, y, status_y_offset, win_info, sem):
|
|
63
63
|
cor, name = task
|
|
64
64
|
addstr(win_info, x, y, name)
|
|
65
|
-
output = f'
|
|
65
|
+
output = f"{'-' * 20}\n{progress(name)}\n{'-' * 20}\n"
|
|
66
66
|
try:
|
|
67
67
|
await sem.acquire()
|
|
68
68
|
output += await cor
|
|
@@ -38,7 +38,7 @@ def confirm(message: str = "") -> bool:
|
|
|
38
38
|
def confirm_dangerous_command() -> None:
|
|
39
39
|
message = (
|
|
40
40
|
"You are about to execute a dangerous command against a"
|
|
41
|
-
f" {colourise('production'
|
|
41
|
+
f" {colourise('production', Fore.RED)} environment. Please ensure you are pairing with"
|
|
42
42
|
" someone else."
|
|
43
43
|
)
|
|
44
44
|
# TODO. Include an actual multi person MFA to proceed.
|
|
@@ -290,7 +290,7 @@ async def _run_job(
|
|
|
290
290
|
|
|
291
291
|
async def wait_for_pod(context: str, namespace: str, pod: str) -> str | None:
|
|
292
292
|
while True:
|
|
293
|
-
cmd =
|
|
293
|
+
cmd = f'kubectl get pod --context {context} -n {namespace} -o jsonpath="{{.status.phase}}" {pod}'
|
|
294
294
|
stdout, _, _ = await async_run(cmd)
|
|
295
295
|
output = stdout.decode().lower()
|
|
296
296
|
if output != "pending":
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.5.1"
|
|
@@ -14,6 +14,7 @@ from gitops_server import settings
|
|
|
14
14
|
from gitops_server.types import AppDefinitions, RunOutput, UpdateAppResult
|
|
15
15
|
from gitops_server.utils import get_repo_name_from_url, github, run, slack
|
|
16
16
|
from gitops_server.utils.git import temp_repo
|
|
17
|
+
from gitops_server.workers.deployer.semaphore_manager import AppSemaphoreManager
|
|
17
18
|
|
|
18
19
|
from .hooks import handle_failed_deploy, handle_successful_deploy
|
|
19
20
|
|
|
@@ -24,9 +25,13 @@ ROLE_ARN = f"arn:aws:iam::{settings.ACCOUNT_ID}:role/GitopsAccess"
|
|
|
24
25
|
logger = logging.getLogger("gitops")
|
|
25
26
|
MAX_HELM_HISTORY = 3
|
|
26
27
|
|
|
28
|
+
# Max parallel helm installs at a time
|
|
29
|
+
# Kube api may rate limit otherwise
|
|
30
|
+
helm_parallel_semaphore = asyncio.Semaphore(int(settings.GITOPS_MAX_PARALLEL_DEPLOYS))
|
|
31
|
+
|
|
27
32
|
|
|
28
33
|
@tracer.start_as_current_span("post_result_summary")
|
|
29
|
-
async def post_result_summary(source: str, results: list[UpdateAppResult]):
|
|
34
|
+
async def post_result_summary(source: str, results: list[UpdateAppResult]) -> None:
|
|
30
35
|
n_success = sum([r["exit_code"] == 0 for r in results])
|
|
31
36
|
n_failed = sum([r["exit_code"] != 0 for r in results])
|
|
32
37
|
await slack.post(
|
|
@@ -52,6 +57,7 @@ class Deployer:
|
|
|
52
57
|
commit_message: str,
|
|
53
58
|
current_app_definitions: AppDefinitions,
|
|
54
59
|
previous_app_definitions: AppDefinitions,
|
|
60
|
+
semaphore_manager: AppSemaphoreManager,
|
|
55
61
|
skip_migrations: bool = False,
|
|
56
62
|
):
|
|
57
63
|
self.author_name = author_name
|
|
@@ -61,9 +67,6 @@ class Deployer:
|
|
|
61
67
|
self.previous_app_definitions = previous_app_definitions
|
|
62
68
|
self.deploy_id = str(uuid.uuid4())
|
|
63
69
|
self.skip_migrations = skip_migrations
|
|
64
|
-
# Max parallel helm installs at a time
|
|
65
|
-
# Kube api may rate limit otherwise
|
|
66
|
-
self.semaphore = asyncio.Semaphore(int(settings.GITOPS_MAX_PARALLEL_DEPLOYS))
|
|
67
70
|
|
|
68
71
|
# Track which ones we've deployed and which ones failed
|
|
69
72
|
self.successful_apps: set[str] = set()
|
|
@@ -71,9 +74,10 @@ class Deployer:
|
|
|
71
74
|
|
|
72
75
|
self.post_ts: str | None = None
|
|
73
76
|
self.skip_deploy = "--skip-deploy" in commit_message
|
|
77
|
+
self.semaphore_manager = semaphore_manager
|
|
74
78
|
|
|
75
79
|
@classmethod
|
|
76
|
-
async def from_push_event(cls, push_event):
|
|
80
|
+
async def from_push_event(cls, push_event: dict[str, Any], semaphore_manager: AppSemaphoreManager) -> "Deployer":
|
|
77
81
|
url = push_event["repository"]["clone_url"]
|
|
78
82
|
author_name = push_event.get("head_commit", {}).get("author", {}).get("name")
|
|
79
83
|
author_email = push_event.get("head_commit", {}).get("author", {}).get("email")
|
|
@@ -91,10 +95,11 @@ class Deployer:
|
|
|
91
95
|
commit_message,
|
|
92
96
|
current_app_definitions,
|
|
93
97
|
previous_app_definitions,
|
|
98
|
+
semaphore_manager,
|
|
94
99
|
skip_migrations,
|
|
95
100
|
)
|
|
96
101
|
|
|
97
|
-
async def deploy(self):
|
|
102
|
+
async def deploy(self) -> None:
|
|
98
103
|
if self.skip_deploy:
|
|
99
104
|
logger.info("Skipping deploy due to `--skip-deploy` flag.")
|
|
100
105
|
return
|
|
@@ -122,20 +127,22 @@ class Deployer:
|
|
|
122
127
|
uninstall_results = await asyncio.gather(
|
|
123
128
|
*[self.uninstall_app(self.previous_app_definitions.apps[app_name]) for app_name in self.removed_apps]
|
|
124
129
|
)
|
|
125
|
-
|
|
130
|
+
without_nones = [r for r in update_results + uninstall_results if r is not None]
|
|
131
|
+
await post_result_summary(self.current_app_definitions.name, without_nones)
|
|
126
132
|
|
|
127
133
|
async def uninstall_app(self, app: App) -> UpdateAppResult:
|
|
128
134
|
with tracer.start_as_current_span("uninstall_app", attributes={"app": app.name}):
|
|
129
|
-
async with self.
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
135
|
+
async with self.semaphore_manager.app_semaphore(app.name):
|
|
136
|
+
async with helm_parallel_semaphore:
|
|
137
|
+
logger.info(f"Uninstalling app {app.name!r}.")
|
|
138
|
+
result = await run(f"helm uninstall {app.name} -n {app.namespace}", suppress_errors=True)
|
|
139
|
+
if result:
|
|
140
|
+
update_result = UpdateAppResult(app_name=app.name, slack_message="", **result)
|
|
141
|
+
await self.post_result(
|
|
142
|
+
app=app,
|
|
143
|
+
result=update_result,
|
|
144
|
+
deployer=self,
|
|
145
|
+
)
|
|
139
146
|
return update_result
|
|
140
147
|
|
|
141
148
|
async def rollback_deployment(self, app: App) -> None:
|
|
@@ -150,61 +157,33 @@ class Deployer:
|
|
|
150
157
|
)
|
|
151
158
|
|
|
152
159
|
async def update_app_deployment(self, app: App) -> UpdateAppResult | None:
|
|
160
|
+
async with self.semaphore_manager.app_semaphore(app.name):
|
|
161
|
+
async with helm_parallel_semaphore:
|
|
162
|
+
return await self._update_app_deployment(app)
|
|
163
|
+
|
|
164
|
+
async def _update_app_deployment(self, app: App) -> UpdateAppResult | None:
|
|
153
165
|
app.set_value("deployment.labels.gitops/deploy_id", self.deploy_id)
|
|
154
166
|
app.set_value("deployment.labels.gitops/status", github.STATUSES.in_progress)
|
|
155
167
|
if github_deployment_url := app.values.get("github/deployment_url"):
|
|
156
168
|
app.set_value("deployment.annotations.github/deployment_url", github_deployment_url)
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
await run(f"cd {chart_folder_path}; helm dependency build")
|
|
167
|
-
|
|
168
|
-
with tempfile.NamedTemporaryFile(suffix=".yml") as cfg:
|
|
169
|
-
cfg.write(json.dumps(app.values).encode())
|
|
170
|
-
cfg.flush()
|
|
171
|
-
os.fsync(cfg.fileno())
|
|
172
|
-
|
|
173
|
-
with tracer.start_as_current_span("helm_upgrade"):
|
|
174
|
-
|
|
175
|
-
async def upgrade_helm_git() -> RunOutput:
|
|
176
|
-
result = await run(
|
|
177
|
-
"helm secrets upgrade --create-namespace"
|
|
178
|
-
f" --history-max {MAX_HELM_HISTORY}"
|
|
179
|
-
" --install"
|
|
180
|
-
" --timeout=600s"
|
|
181
|
-
f"{' --set skip_migrations=true' if self.skip_migrations else ''}"
|
|
182
|
-
f" -f {cfg.name}"
|
|
183
|
-
f" --namespace={app.namespace}"
|
|
184
|
-
f" {app.name}"
|
|
185
|
-
f" {chart_folder_path}",
|
|
186
|
-
suppress_errors=True,
|
|
187
|
-
)
|
|
188
|
-
return result
|
|
169
|
+
with tracer.start_as_current_span("update_app_deployment", attributes={"app": app.name}) as span:
|
|
170
|
+
logger.info(f"Deploying app {app.name!r}.")
|
|
171
|
+
from_timestamp = time.time()
|
|
172
|
+
if app.chart.type == "git":
|
|
173
|
+
span.set_attribute("gitops.chart.type", "git")
|
|
174
|
+
assert app.chart.git_repo_url
|
|
175
|
+
async with temp_repo(app.chart.git_repo_url, ref=app.chart.git_sha) as chart_folder_path:
|
|
176
|
+
with tracer.start_as_current_span("helm_dependency_build"):
|
|
177
|
+
await run(f"cd {chart_folder_path}; helm dependency build")
|
|
189
178
|
|
|
190
|
-
result = await upgrade_helm_git()
|
|
191
|
-
if result["exit_code"] != 0 and "is in progress" in result["output"]:
|
|
192
|
-
await self.rollback_deployment(app)
|
|
193
|
-
result = await upgrade_helm_git()
|
|
194
|
-
|
|
195
|
-
elif app.chart.type == "helm":
|
|
196
|
-
span.set_attribute("gitops.chart.type", "helm")
|
|
197
179
|
with tempfile.NamedTemporaryFile(suffix=".yml") as cfg:
|
|
198
180
|
cfg.write(json.dumps(app.values).encode())
|
|
199
181
|
cfg.flush()
|
|
200
182
|
os.fsync(cfg.fileno())
|
|
201
|
-
chart_version_arguments = f" --version={app.chart.version}" if app.chart.version else ""
|
|
202
|
-
with tracer.start_as_current_span("helm_repo_add"):
|
|
203
|
-
await run(f"helm repo add {app.chart.helm_repo} {app.chart.helm_repo_url}")
|
|
204
183
|
|
|
205
184
|
with tracer.start_as_current_span("helm_upgrade"):
|
|
206
185
|
|
|
207
|
-
async def
|
|
186
|
+
async def upgrade_helm_git() -> RunOutput:
|
|
208
187
|
result = await run(
|
|
209
188
|
"helm secrets upgrade --create-namespace"
|
|
210
189
|
f" --history-max {MAX_HELM_HISTORY}"
|
|
@@ -214,22 +193,54 @@ class Deployer:
|
|
|
214
193
|
f" -f {cfg.name}"
|
|
215
194
|
f" --namespace={app.namespace}"
|
|
216
195
|
f" {app.name}"
|
|
217
|
-
f" {
|
|
196
|
+
f" {chart_folder_path}",
|
|
218
197
|
suppress_errors=True,
|
|
219
198
|
)
|
|
220
199
|
return result
|
|
221
200
|
|
|
222
|
-
result = await
|
|
201
|
+
result = await upgrade_helm_git()
|
|
223
202
|
if result["exit_code"] != 0 and "is in progress" in result["output"]:
|
|
224
203
|
await self.rollback_deployment(app)
|
|
225
|
-
result = await
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
204
|
+
result = await upgrade_helm_git()
|
|
205
|
+
|
|
206
|
+
elif app.chart.type == "helm":
|
|
207
|
+
span.set_attribute("gitops.chart.type", "helm")
|
|
208
|
+
with tempfile.NamedTemporaryFile(suffix=".yml") as cfg:
|
|
209
|
+
cfg.write(json.dumps(app.values).encode())
|
|
210
|
+
cfg.flush()
|
|
211
|
+
os.fsync(cfg.fileno())
|
|
212
|
+
chart_version_arguments = f" --version={app.chart.version}" if app.chart.version else ""
|
|
213
|
+
with tracer.start_as_current_span("helm_repo_add"):
|
|
214
|
+
await run(f"helm repo add {app.chart.helm_repo} {app.chart.helm_repo_url}")
|
|
215
|
+
|
|
216
|
+
with tracer.start_as_current_span("helm_upgrade"):
|
|
217
|
+
|
|
218
|
+
async def upgrade_helm_chart() -> RunOutput:
|
|
219
|
+
result = await run(
|
|
220
|
+
"helm secrets upgrade --create-namespace"
|
|
221
|
+
f" --history-max {MAX_HELM_HISTORY}"
|
|
222
|
+
" --install"
|
|
223
|
+
" --timeout=600s"
|
|
224
|
+
f"{' --set skip_migrations=true' if self.skip_migrations else ''}"
|
|
225
|
+
f" -f {cfg.name}"
|
|
226
|
+
f" --namespace={app.namespace}"
|
|
227
|
+
f" {app.name}"
|
|
228
|
+
f" {app.chart.helm_chart} {chart_version_arguments}",
|
|
229
|
+
suppress_errors=True,
|
|
230
|
+
)
|
|
231
|
+
return result
|
|
232
|
+
|
|
233
|
+
result = await upgrade_helm_chart()
|
|
234
|
+
if result["exit_code"] != 0 and "is in progress" in result["output"]:
|
|
235
|
+
await self.rollback_deployment(app)
|
|
236
|
+
result = await upgrade_helm_chart()
|
|
237
|
+
else:
|
|
238
|
+
logger.warning("Local is not implemented yet")
|
|
239
|
+
return None
|
|
229
240
|
|
|
230
|
-
|
|
241
|
+
update_result = UpdateAppResult(app_name=app.name, slack_message="", **result)
|
|
231
242
|
|
|
232
|
-
|
|
243
|
+
await self.post_result(app=app, result=update_result, deployer=self, from_timestamp=from_timestamp)
|
|
233
244
|
return update_result
|
|
234
245
|
|
|
235
246
|
def calculate_app_deltas(self) -> tuple[set[str], set[str], set[str]]:
|
|
@@ -103,7 +103,7 @@ async def handle_failed_deploy(app: App, result: UpdateAppResult, deployer, **kw
|
|
|
103
103
|
await find_commiter_slack_user(name=deployer.author_name, email=deployer.author_email) or DEFAULT_USER_GROUP
|
|
104
104
|
)
|
|
105
105
|
slack_user_msg = f" {slack_user} " if slack_user else ""
|
|
106
|
-
log_msg = f"<{get_dashboard_url(workspace_name=app.name, from_timestamp=kwargs.get(
|
|
106
|
+
log_msg = f"<{get_dashboard_url(workspace_name=app.name, from_timestamp=kwargs.get('from_timestamp'), to_timestamp=time.time())}|(Deployment Logs)>"
|
|
107
107
|
result["slack_message"] = (
|
|
108
108
|
f"Failed to deploy app `{result['app_name']}` for cluster"
|
|
109
109
|
f" `{settings.CLUSTER_NAME}` :rotating_light:"
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import logging
|
|
3
|
+
from contextlib import asynccontextmanager
|
|
4
|
+
from collections.abc import AsyncGenerator
|
|
5
|
+
|
|
6
|
+
logger = logging.getLogger("gitops.semaphore")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class AppSemaphoreManager:
|
|
10
|
+
"""Manages per-app semaphores to ensure sequential updates per app.
|
|
11
|
+
|
|
12
|
+
This ensures that while multiple deployments can run concurrently,
|
|
13
|
+
any given app is only updated by one deployment at a time.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
_instance: "AppSemaphoreManager | None" = None
|
|
17
|
+
|
|
18
|
+
def __init__(self) -> None:
|
|
19
|
+
self._app_semaphores: dict[str, asyncio.Semaphore] = {}
|
|
20
|
+
self._semaphore_lock = asyncio.Lock()
|
|
21
|
+
self._initialized: bool = True
|
|
22
|
+
|
|
23
|
+
async def get_app_semaphore(self, app_name: str) -> asyncio.Semaphore:
|
|
24
|
+
"""Get the semaphore for an app.
|
|
25
|
+
|
|
26
|
+
If the semaphore does not exist, create it.
|
|
27
|
+
"""
|
|
28
|
+
async with self._semaphore_lock:
|
|
29
|
+
if app_name not in self._app_semaphores:
|
|
30
|
+
logger.debug(f"Creating new semaphore for app: {app_name}")
|
|
31
|
+
self._app_semaphores[app_name] = asyncio.Semaphore(1)
|
|
32
|
+
semaphore = self._app_semaphores[app_name]
|
|
33
|
+
return semaphore
|
|
34
|
+
|
|
35
|
+
@asynccontextmanager
|
|
36
|
+
async def app_semaphore(self, app_name: str) -> AsyncGenerator[None, None]:
|
|
37
|
+
"""Async context manager to acquire and release a semaphore for the given app.
|
|
38
|
+
|
|
39
|
+
Each app gets exactly one semaphore with a limit of 1,
|
|
40
|
+
ensuring sequential updates per app.
|
|
41
|
+
"""
|
|
42
|
+
semaphore = await self.get_app_semaphore(app_name)
|
|
43
|
+
async with semaphore:
|
|
44
|
+
yield
|
|
45
|
+
|
|
46
|
+
def get_locked_apps(self) -> set[str]:
|
|
47
|
+
"""Return a set of apps that currently have active locks."""
|
|
48
|
+
locked_apps = set()
|
|
49
|
+
for app_name, semaphore in self._app_semaphores.items():
|
|
50
|
+
if semaphore.locked():
|
|
51
|
+
locked_apps.add(app_name)
|
|
52
|
+
return locked_apps
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import logging
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from opentelemetry import trace
|
|
6
|
+
|
|
7
|
+
from .deploy import Deployer
|
|
8
|
+
from .semaphore_manager import AppSemaphoreManager
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger("gitops_worker")
|
|
11
|
+
|
|
12
|
+
tracer = trace.get_tracer(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class DeployQueueWorker:
|
|
16
|
+
"""Concurrent deployment worker with per-app serialization.
|
|
17
|
+
|
|
18
|
+
Multiple deployments can run simultaneously, but apps are updated
|
|
19
|
+
sequentially using per-app semaphores. This ensures cluster consistency
|
|
20
|
+
while allowing parallel deployments of different apps.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
# Worker singleton
|
|
24
|
+
_worker: "DeployQueueWorker | None" = None
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def get_worker(cls) -> "DeployQueueWorker":
|
|
28
|
+
if not cls._worker:
|
|
29
|
+
cls._worker = cls()
|
|
30
|
+
return cls._worker
|
|
31
|
+
|
|
32
|
+
def __init__(self) -> None:
|
|
33
|
+
self.queue: asyncio.Queue[Any] = asyncio.Queue()
|
|
34
|
+
self.semaphore_manager = AppSemaphoreManager()
|
|
35
|
+
self.active_deployments: set[asyncio.Task[None]] = set()
|
|
36
|
+
|
|
37
|
+
async def enqueue(self, work: Any) -> None:
|
|
38
|
+
"""Enqueue an item of work for future processing.
|
|
39
|
+
|
|
40
|
+
The `work` argument is the body of an incoming GitHub push webhook.
|
|
41
|
+
"""
|
|
42
|
+
logger.info(f"Enqueued work, {self.queue.qsize() + 1} items in the queue.")
|
|
43
|
+
await self.queue.put(work)
|
|
44
|
+
|
|
45
|
+
async def run(self) -> None:
|
|
46
|
+
"""Run the worker.
|
|
47
|
+
|
|
48
|
+
Manages concurrent deployments while maintaining per-app serialization.
|
|
49
|
+
Each deployment runs as a separate task, allowing multiple deployments
|
|
50
|
+
to proceed simultaneously.
|
|
51
|
+
# TODO: Need to gracefully handle termination.
|
|
52
|
+
"""
|
|
53
|
+
logger.info("Starting up concurrent deployer worker loop")
|
|
54
|
+
while True:
|
|
55
|
+
try:
|
|
56
|
+
# Clean up completed tasks
|
|
57
|
+
self.active_deployments = {task for task in self.active_deployments if not task.done()}
|
|
58
|
+
|
|
59
|
+
# Process new work
|
|
60
|
+
work = await self.queue.get()
|
|
61
|
+
deployment_task = asyncio.create_task(self.process_deployment(work))
|
|
62
|
+
self.active_deployments.add(deployment_task)
|
|
63
|
+
|
|
64
|
+
# Log current status
|
|
65
|
+
locked_apps = self.semaphore_manager.get_locked_apps()
|
|
66
|
+
logger.info(
|
|
67
|
+
f"Active deployments: {len(self.active_deployments)}, " + f"Locked apps: {sorted(locked_apps)}"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
except Exception as e:
|
|
71
|
+
logger.error(str(e), exc_info=True)
|
|
72
|
+
|
|
73
|
+
async def process_deployment(self, work: Any) -> None:
|
|
74
|
+
"""Process a single deployment in a separate task.
|
|
75
|
+
|
|
76
|
+
This allows multiple deployments to run concurrently while
|
|
77
|
+
maintaining per-app serialization through semaphores.
|
|
78
|
+
"""
|
|
79
|
+
ref = work.get("ref")
|
|
80
|
+
logger.info(f'Processing deployment for push to "{ref}".')
|
|
81
|
+
|
|
82
|
+
if ref == "refs/heads/master":
|
|
83
|
+
with tracer.start_as_current_span("gitops_process_webhook") as current_span:
|
|
84
|
+
try:
|
|
85
|
+
deployer = await Deployer.from_push_event(work, self.semaphore_manager)
|
|
86
|
+
current_span.set_attribute("gitops.ref", ref)
|
|
87
|
+
current_span.set_attribute("gitops.after", work.get("after"))
|
|
88
|
+
current_span.set_attribute("gitops.before", work.get("before"))
|
|
89
|
+
current_span.set_attribute("gitops.author_email", deployer.author_email)
|
|
90
|
+
current_span.set_attribute("gitops.author_name", deployer.author_name)
|
|
91
|
+
current_span.set_attribute("gitops.commit_message", deployer.commit_message)
|
|
92
|
+
await deployer.deploy()
|
|
93
|
+
|
|
94
|
+
except Exception as e:
|
|
95
|
+
logger.error(f"Deployment failed: {e}", exc_info=True)
|
|
96
|
+
raise
|
|
97
|
+
else:
|
|
98
|
+
logger.info(f'Ignoring push to "{ref}" (not master branch).')
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
[project]
|
|
3
3
|
name = "gitops"
|
|
4
|
-
version = "1.
|
|
4
|
+
version = "1.5.1"
|
|
5
5
|
description = "Manage multiple apps across one or more k8s clusters."
|
|
6
6
|
requires-python = ">=3.12"
|
|
7
7
|
readme = "README.md"
|
|
@@ -11,7 +11,7 @@ dependencies = [
|
|
|
11
11
|
"pyyaml>=6.0.2",
|
|
12
12
|
"dsnparse==0.2.1",
|
|
13
13
|
"colorama>=0.4.4",
|
|
14
|
-
"boto3
|
|
14
|
+
"boto3>=1.35.4",
|
|
15
15
|
"humanize>=3.5.0",
|
|
16
16
|
"tabulate>=0.8.9",
|
|
17
17
|
"packaging>=24.1",
|