gitops 1.7.1__tar.gz → 1.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitops-1.7.1 → gitops-1.9.0}/.github/workflows/test.yml +6 -0
- gitops-1.9.0/.release-please-manifest.json +5 -0
- {gitops-1.7.1 → gitops-1.9.0}/CHANGELOG.md +37 -0
- {gitops-1.7.1 → gitops-1.9.0}/PKG-INFO +2 -1
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/CHANGELOG.md +13 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/Chart.yaml +1 -1
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/deployment.yml +5 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/secrets.yml +2 -1
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/values.yaml +5 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/__init__.py +1 -1
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/app.py +8 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/settings.py +5 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/utils/__init__.py +15 -2
- gitops-1.9.0/gitops_server/utils/git.py +197 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/utils/github.py +5 -7
- gitops-1.9.0/gitops_server/utils/github_auth.py +133 -0
- gitops-1.9.0/gitops_server/version.py +1 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/workers/deployer/deploy.py +71 -4
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/workers/deployer/hooks.py +1 -1
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/workers/deployer/worker.py +22 -3
- {gitops-1.7.1 → gitops-1.9.0}/pyproject.toml +3 -1
- gitops-1.9.0/tests/test_clone_credentials.py +109 -0
- gitops-1.9.0/tests/test_github_auth.py +106 -0
- gitops-1.9.0/tests/test_load_app_definitions.py +145 -0
- gitops-1.9.0/tests/test_worker_concurrency.py +76 -0
- {gitops-1.7.1 → gitops-1.9.0}/uv.lock +162 -2
- gitops-1.7.1/.release-please-manifest.json +0 -5
- gitops-1.7.1/gitops_server/utils/git.py +0 -88
- gitops-1.7.1/gitops_server/version.py +0 -1
- {gitops-1.7.1 → gitops-1.9.0}/.dockerignore +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/.github/CODEOWNERS +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/.github/workflows/release-please.yaml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/.gitignore +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/.mise.toml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/.pre-commit-config.yaml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/CLAUDE.md +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/Dockerfile +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/LICENSE +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/README.md +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/.helmignore +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/README.md +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/_helpers.tpl +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/certificate.yml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/environment.yml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/ingress.yml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/issuer.yml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/pdb.yml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/service.yml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/charts/gitops/templates/serviceaccount.yaml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/docker-compose.yml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/docs/plans/2026-03-13-kms-encryption-design.md +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/common/__init__.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/common/app.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/common/utils.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/core.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/main.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/py.typed +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/settings.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/shorthands.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/utils/__init__.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/utils/apps.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/utils/async_runner.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/utils/cli.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/utils/exceptions.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/utils/images.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/utils/kube.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/utils/tags.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops/utils/yaml.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/CHANGELOG.md +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/README.md +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/__init__.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/main.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/types.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/utils/slack.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/workers/__init__.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/workers/deployer/__init__.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/gitops_server/workers/deployer/semaphore_manager.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/release-please-config.json +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/release.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/renovate.json +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/ruff.toml +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/secrets.example.env +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/setup.cfg +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/__init__.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/conftest.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/sample_data.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/test_app.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/test_core.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/test_deploy.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/test_deploy_concurrent.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/test_git.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/test_gitops_server.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/test_utils.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/utils.py +0 -0
- {gitops-1.7.1 → gitops-1.9.0}/tests/webhook_sample_data.py +0 -0
|
@@ -36,6 +36,12 @@ jobs:
|
|
|
36
36
|
cache: true
|
|
37
37
|
experimental: true
|
|
38
38
|
|
|
39
|
+
# Secrets in the cluster repo are git-crypt encrypted, and the app
|
|
40
|
+
# definition loader decrypts them. Without this the coverage of that path
|
|
41
|
+
# silently skips.
|
|
42
|
+
- name: Install git-crypt
|
|
43
|
+
run: sudo apt-get update && sudo apt-get install -y git-crypt
|
|
44
|
+
|
|
39
45
|
- name: Run CI
|
|
40
46
|
env:
|
|
41
47
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.9.0](https://github.com/uptick/gitops/compare/gitops-v1.8.0...gitops-v1.9.0) (2026-09-09)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* authenticate to GitHub as an App ([51ae678](https://github.com/uptick/gitops/commit/51ae678a874cf205b0f612404426c7444611f96c))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* authenticate the repo-cache fetch, and tighten the mint ([3c1100b](https://github.com/uptick/gitops/commit/3c1100b936a3fb83eef28d3ed2732d0afaf8d607))
|
|
14
|
+
* scope the credential helper to github.com ([7068133](https://github.com/uptick/gitops/commit/7068133b1a3796ffa85de191549097d2ad724f6d))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
### Documentation
|
|
18
|
+
|
|
19
|
+
* cut comments back to what is not derivable from the code ([4d41784](https://github.com/uptick/gitops/commit/4d4178478da4efa37b924e3cab3da430e8ec68bd))
|
|
20
|
+
|
|
21
|
+
## [1.8.0](https://github.com/uptick/gitops/compare/gitops-v1.7.1...gitops-v1.8.0) (2026-08-21)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
### Features
|
|
25
|
+
|
|
26
|
+
* **worker:** bound concurrent deployments ([956c630](https://github.com/uptick/gitops/commit/956c6309516ba831805aa1df90fd4fdb7314cb43))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
### Bug Fixes
|
|
30
|
+
|
|
31
|
+
* **security:** stop embedding the github token in clone URLs ([fd88bc1](https://github.com/uptick/gitops/commit/fd88bc1176a51caf9dddca37aee9d4ecfc0b5c78))
|
|
32
|
+
* **security:** stop embedding the github token in clone URLs ([68b7225](https://github.com/uptick/gitops/commit/68b722547d83e2ab1560ef7af0bc925d893b7580))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
### Performance Improvements
|
|
36
|
+
|
|
37
|
+
* **deploy:** read app definitions from git instead of checking out the repo ([0b21790](https://github.com/uptick/gitops/commit/0b2179084e8920790de70363cdb369558342e895))
|
|
38
|
+
* **deploy:** read app definitions from git instead of checking out the repo ([f90bc3a](https://github.com/uptick/gitops/commit/f90bc3a60141423313574813adeb1313406f7168))
|
|
39
|
+
|
|
3
40
|
## [1.7.1](https://github.com/uptick/gitops/compare/gitops-v1.7.0...gitops-v1.7.1) (2026-08-17)
|
|
4
41
|
|
|
5
42
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: gitops
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.9.0
|
|
4
4
|
Summary: Manage multiple apps across one or more k8s clusters.
|
|
5
5
|
Author-email: Jarek Głowacki <jarekwg@gmail.com>
|
|
6
6
|
License-File: LICENSE
|
|
@@ -22,6 +22,7 @@ Requires-Dist: httpx>=0.27.0; extra == 'server'
|
|
|
22
22
|
Requires-Dist: idna>=3.15; extra == 'server'
|
|
23
23
|
Requires-Dist: kubernetes-asyncio>=25.1.2; extra == 'server'
|
|
24
24
|
Requires-Dist: protobuf>=5.29.6; extra == 'server'
|
|
25
|
+
Requires-Dist: pyjwt[crypto]>=2.10.0; extra == 'server'
|
|
25
26
|
Requires-Dist: requests>=2.33.0; extra == 'server'
|
|
26
27
|
Requires-Dist: sentry-sdk>=1.3.0; extra == 'server'
|
|
27
28
|
Requires-Dist: slack-sdk>=3.33.5; extra == 'server'
|
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.3.0](https://github.com/uptick/gitops/compare/helm-v1.2.2...helm-v1.3.0) (2026-09-09)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* authenticate to GitHub as an App ([51ae678](https://github.com/uptick/gitops/commit/51ae678a874cf205b0f612404426c7444611f96c))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* authenticate the repo-cache fetch, and tighten the mint ([3c1100b](https://github.com/uptick/gitops/commit/3c1100b936a3fb83eef28d3ed2732d0afaf8d607))
|
|
14
|
+
* scope the credential helper to github.com ([7068133](https://github.com/uptick/gitops/commit/7068133b1a3796ffa85de191549097d2ad724f6d))
|
|
15
|
+
|
|
3
16
|
## [1.2.2](https://github.com/uptick/gitops/compare/helm-v1.2.1...helm-v1.2.2) (2026-05-28)
|
|
4
17
|
|
|
5
18
|
|
|
@@ -65,6 +65,11 @@ spec:
|
|
|
65
65
|
secretKeyRef:
|
|
66
66
|
name: {{ .Release.Name }}-secret
|
|
67
67
|
key: GITHUB_OAUTH_TOKEN
|
|
68
|
+
- name: GITHUB_APP_PRIVATE_KEY
|
|
69
|
+
valueFrom:
|
|
70
|
+
secretKeyRef:
|
|
71
|
+
name: {{ .Release.Name }}-secret
|
|
72
|
+
key: GITHUB_APP_PRIVATE_KEY
|
|
68
73
|
- name: GITHUB_WEBHOOK_KEY
|
|
69
74
|
valueFrom:
|
|
70
75
|
secretKeyRef:
|
|
@@ -10,4 +10,5 @@ stringData:
|
|
|
10
10
|
SENTRY_DSN: {{ .Values.secrets.SENTRY_DSN| quote}}
|
|
11
11
|
SLACK_TOKEN: {{ .Values.secrets.SLACK_TOKEN| quote}}
|
|
12
12
|
data:
|
|
13
|
-
GIT_CRYPT_KEY: {{ .Values.secrets.GIT_CRYPT_KEY | quote}}
|
|
13
|
+
GIT_CRYPT_KEY: {{ .Values.secrets.GIT_CRYPT_KEY | quote}}
|
|
14
|
+
GITHUB_APP_PRIVATE_KEY: {{ .Values.secrets.GITHUB_APP_PRIVATE_KEY | quote}}
|
|
@@ -9,11 +9,16 @@ environment:
|
|
|
9
9
|
TZ: "UTC"
|
|
10
10
|
# The channel id to send slack messages to
|
|
11
11
|
SLACK_CHANNEL_ID: ""
|
|
12
|
+
# GitHub App identifiers. Empty when authenticating with a PAT.
|
|
13
|
+
GITHUB_APP_ID: ""
|
|
14
|
+
GITHUB_INSTALLATION_ID: ""
|
|
12
15
|
|
|
13
16
|
secrets:
|
|
14
17
|
ACCOUNT_ID: ""
|
|
15
18
|
SLACK_TOKEN: ""
|
|
16
19
|
GITHUB_OAUTH_TOKEN: ""
|
|
20
|
+
# base64-encoded PEM. Empty when authenticating with a PAT.
|
|
21
|
+
GITHUB_APP_PRIVATE_KEY: ""
|
|
17
22
|
GITHUB_WEBHOOK_KEY: ""
|
|
18
23
|
GIT_CRYPT_KEY: ""
|
|
19
24
|
SENTRY_DSN: ""
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
import logging
|
|
2
3
|
import os
|
|
3
4
|
from collections.abc import AsyncIterator
|
|
4
5
|
from contextlib import asynccontextmanager
|
|
@@ -9,12 +10,19 @@ from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
|
|
|
9
10
|
|
|
10
11
|
from . import settings
|
|
11
12
|
|
|
13
|
+
logger = logging.getLogger("gitops")
|
|
14
|
+
|
|
12
15
|
|
|
13
16
|
@asynccontextmanager
|
|
14
17
|
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
|
15
18
|
"""Manage application lifespan - start and stop background workers."""
|
|
19
|
+
from .utils.github_auth import credentials, validate_credentials
|
|
16
20
|
from .workers import DeployQueueWorker
|
|
17
21
|
|
|
22
|
+
# Fail here rather than on the first push, where hooks.py only logs a warning.
|
|
23
|
+
validate_credentials()
|
|
24
|
+
logger.info("GitHub authentication: %s", credentials.describe())
|
|
25
|
+
|
|
18
26
|
loop = asyncio.get_running_loop()
|
|
19
27
|
deploy_queue_worker = DeployQueueWorker.get_worker()
|
|
20
28
|
deploy_queue_worker_task = asyncio.ensure_future(deploy_queue_worker.run(), loop=loop)
|
|
@@ -16,3 +16,8 @@ SLACK_ALERT_CHANNEL_ID = os.environ.get("SLACK_ALERT_CHANNEL_ID", "")
|
|
|
16
16
|
|
|
17
17
|
# How many parallel deploys can be done at once
|
|
18
18
|
GITOPS_MAX_PARALLEL_DEPLOYS = os.environ.get("GITOPS_MAX_PARALLEL_DEPLOYS", "5")
|
|
19
|
+
|
|
20
|
+
# How many pushes can be processed at once. Each one loads app definitions for
|
|
21
|
+
# two commits before it reaches GITOPS_MAX_PARALLEL_DEPLOYS, so without a bound
|
|
22
|
+
# here a burst of pushes is only limited by how fast they arrive.
|
|
23
|
+
GITOPS_MAX_CONCURRENT_DEPLOYMENTS = os.environ.get("GITOPS_MAX_CONCURRENT_DEPLOYMENTS", "20")
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import logging
|
|
3
|
+
import os
|
|
3
4
|
from collections.abc import Callable
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
|
|
@@ -19,16 +20,28 @@ async def _read_stream(stream: asyncio.StreamReader | None, cb: Callable[[bytes]
|
|
|
19
20
|
break
|
|
20
21
|
|
|
21
22
|
|
|
22
|
-
async def run(
|
|
23
|
+
async def run(
|
|
24
|
+
command: str,
|
|
25
|
+
suppress_errors: bool = False,
|
|
26
|
+
cwd: str | Path | None = None,
|
|
27
|
+
env: dict[str, str] | None = None,
|
|
28
|
+
) -> RunOutput:
|
|
23
29
|
"""Run a shell command.
|
|
24
30
|
|
|
25
31
|
Runs the command in an asyncio executor to keep things async. Will
|
|
26
32
|
optionally prevent raising an exception on failure with `suppress_errors`.
|
|
33
|
+
|
|
34
|
+
`env` is overlaid on the current environment, so a caller can hand one command
|
|
35
|
+
a credential without putting it in argv.
|
|
27
36
|
"""
|
|
28
37
|
exit_code = 0
|
|
29
38
|
logger.info(f'Running "{command}".')
|
|
30
39
|
proc = await asyncio.create_subprocess_shell(
|
|
31
|
-
command,
|
|
40
|
+
command,
|
|
41
|
+
stdout=asyncio.subprocess.PIPE,
|
|
42
|
+
stderr=asyncio.subprocess.PIPE,
|
|
43
|
+
cwd=cwd,
|
|
44
|
+
env={**os.environ, **env} if env else None,
|
|
32
45
|
)
|
|
33
46
|
stdout, stderr = b"", b""
|
|
34
47
|
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import tempfile
|
|
6
|
+
from collections.abc import AsyncGenerator
|
|
7
|
+
from contextlib import asynccontextmanager
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from opentelemetry.trace import get_tracer
|
|
11
|
+
|
|
12
|
+
from . import run
|
|
13
|
+
from .github_auth import credentials
|
|
14
|
+
|
|
15
|
+
tracer = get_tracer(__name__)
|
|
16
|
+
logger = logging.getLogger("gitops")
|
|
17
|
+
|
|
18
|
+
REPO_CACHE_DIR = Path("/tmp/gitops/repocache")
|
|
19
|
+
REPO_CACHE: dict[str, Path] = {}
|
|
20
|
+
repo_lock = asyncio.Semaphore(1)
|
|
21
|
+
|
|
22
|
+
# git-crypt prefixes every encrypted blob with this magic.
|
|
23
|
+
GIT_CRYPT_MAGIC = b"\0GITCRYPT\0"
|
|
24
|
+
|
|
25
|
+
# git-crypt has no batch mode: one process per file. Bounded so concurrent
|
|
26
|
+
# loads can't fork-storm.
|
|
27
|
+
decrypt_semaphore = asyncio.Semaphore(16)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
async def credential_env(git_repo_url: str) -> dict[str, str] | None:
|
|
31
|
+
"""Credentials for a git subprocess, or None for a remote that needs none.
|
|
32
|
+
|
|
33
|
+
Local paths in tests are not GitHub, and asking would mint a token for nothing.
|
|
34
|
+
"""
|
|
35
|
+
if "://" not in git_repo_url:
|
|
36
|
+
return None
|
|
37
|
+
return await credentials.git_env()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def clone_repo(git_repo_url: str, path: str, sha: str | None = None, branch: str | None = None) -> None:
|
|
41
|
+
"""Shallow Clones a git repo url to path and git-crypt unlocks all encrypted files"""
|
|
42
|
+
logger.info(f'Cloning "{git_repo_url}".')
|
|
43
|
+
|
|
44
|
+
env = await credential_env(git_repo_url)
|
|
45
|
+
|
|
46
|
+
with tracer.start_as_current_span("tempo_repo.clone_repo"):
|
|
47
|
+
if sha:
|
|
48
|
+
await run(f"git clone --depth 100 {git_repo_url} {path}; cd {path}; git checkout {sha}", env=env)
|
|
49
|
+
elif branch:
|
|
50
|
+
await run(f"git clone --depth 1 {git_repo_url} {path} --branch {branch}", env=env)
|
|
51
|
+
else:
|
|
52
|
+
await run(f"git clone --depth 100 {git_repo_url} {path};", env=env)
|
|
53
|
+
|
|
54
|
+
if GIT_CRYPT_KEY_FILE := os.environ.get("GIT_CRYPT_KEY_FILE"):
|
|
55
|
+
with tracer.start_as_current_span("temp_repo.git_crypt_unlock"):
|
|
56
|
+
await run(f"cd {path}; git-crypt unlock {GIT_CRYPT_KEY_FILE}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def is_sha(sha_or_ref: str | None) -> bool:
|
|
60
|
+
"""Check if the given string is a valid SHA-1 hash or a short SHA."""
|
|
61
|
+
if not sha_or_ref:
|
|
62
|
+
return False
|
|
63
|
+
return bool(re.fullmatch(r"[a-fA-F0-9]{4,40}", sha_or_ref))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@asynccontextmanager
|
|
67
|
+
async def temp_repo(git_repo_url: str, ref: str | None) -> AsyncGenerator[str, None]:
|
|
68
|
+
"""Checks out a git_repo_url to a temporary folder location. Returns temporary folder location"""
|
|
69
|
+
# Someone passed us a branch name, not a sha
|
|
70
|
+
|
|
71
|
+
if not is_sha(ref):
|
|
72
|
+
async with temp_repo_branch(git_repo_url, branch=ref) as temp_repo_path:
|
|
73
|
+
yield temp_repo_path
|
|
74
|
+
return
|
|
75
|
+
|
|
76
|
+
assert ref is not None # is_sha() returns False for None
|
|
77
|
+
with tracer.start_as_current_span("checkout_temp_repo"):
|
|
78
|
+
cache_path = await ensure_repo_cache(git_repo_url, ref)
|
|
79
|
+
|
|
80
|
+
# Copy the repo cache to a temporary folder
|
|
81
|
+
with tempfile.TemporaryDirectory() as temporary_folder_path:
|
|
82
|
+
await run(f"rm -rf {temporary_folder_path}", suppress_errors=True)
|
|
83
|
+
await run(f"cp -r {cache_path} {temporary_folder_path}")
|
|
84
|
+
# ref is already in the cache, so no per-copy fetch.
|
|
85
|
+
await run(f"cd {temporary_folder_path};git checkout {ref}")
|
|
86
|
+
|
|
87
|
+
yield temporary_folder_path
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
async def _git(repo: str, *args: str, stdin: bytes | None = None, env: dict[str, str] | None = None) -> bytes:
|
|
91
|
+
"""Run git in `repo`, returning raw stdout.
|
|
92
|
+
|
|
93
|
+
Not `run()`: blobs are binary, and `run()` decodes and logs every line.
|
|
94
|
+
"""
|
|
95
|
+
proc = await asyncio.create_subprocess_exec(
|
|
96
|
+
"git",
|
|
97
|
+
"-C",
|
|
98
|
+
repo,
|
|
99
|
+
*args,
|
|
100
|
+
stdin=asyncio.subprocess.PIPE if stdin is not None else None,
|
|
101
|
+
stdout=asyncio.subprocess.PIPE,
|
|
102
|
+
stderr=asyncio.subprocess.PIPE,
|
|
103
|
+
env={**os.environ, **env} if env else None,
|
|
104
|
+
)
|
|
105
|
+
stdout, stderr = await proc.communicate(stdin)
|
|
106
|
+
if proc.returncode != 0:
|
|
107
|
+
raise Exception(f"git {' '.join(args)} failed in {repo}: {stderr.decode(errors='replace')}")
|
|
108
|
+
return stdout
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
async def ensure_repo_cache(git_repo_url: str, ref: str) -> str:
|
|
112
|
+
"""Clone the shared cache if absent and ensure `ref` is in it."""
|
|
113
|
+
cache_path = REPO_CACHE_DIR / git_repo_url.split("/")[-1].split(".")[0]
|
|
114
|
+
async with repo_lock:
|
|
115
|
+
if not (cache_path / ".git").exists():
|
|
116
|
+
logger.info("Repo %s not in cache, cloning", git_repo_url)
|
|
117
|
+
if cache_path.exists():
|
|
118
|
+
await run(f"rm -rf {cache_path}", suppress_errors=True)
|
|
119
|
+
cache_path.mkdir(parents=True)
|
|
120
|
+
REPO_CACHE[git_repo_url] = cache_path
|
|
121
|
+
await clone_repo(git_repo_url, path=str(cache_path))
|
|
122
|
+
|
|
123
|
+
try:
|
|
124
|
+
await _git(str(cache_path), "cat-file", "-e", f"{ref}^{{commit}}")
|
|
125
|
+
except Exception:
|
|
126
|
+
logger.info("Ref %s not in cache, fetching", ref)
|
|
127
|
+
env = await credential_env(git_repo_url)
|
|
128
|
+
await _git(str(cache_path), "fetch", "--quiet", "origin", env=env)
|
|
129
|
+
return str(cache_path)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
async def list_tree_blobs(repo: str, sha: str, prefix: str) -> dict[str, str]:
|
|
133
|
+
"""Map path -> blob oid for every file under `prefix` at `sha`."""
|
|
134
|
+
out = await _git(repo, "ls-tree", "-r", sha, "--", prefix)
|
|
135
|
+
blobs: dict[str, str] = {}
|
|
136
|
+
for line in out.decode().splitlines():
|
|
137
|
+
meta, _, path = line.partition("\t")
|
|
138
|
+
parts = meta.split()
|
|
139
|
+
if len(parts) == 3 and parts[1] == "blob":
|
|
140
|
+
blobs[path] = parts[2]
|
|
141
|
+
return blobs
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
async def read_blobs(repo: str, oids: list[str]) -> dict[str, bytes]:
|
|
145
|
+
"""Read many blobs in a single `git cat-file --batch` process."""
|
|
146
|
+
if not oids:
|
|
147
|
+
return {}
|
|
148
|
+
out = await _git(repo, "cat-file", "--batch", stdin=("\n".join(oids) + "\n").encode())
|
|
149
|
+
|
|
150
|
+
contents: dict[str, bytes] = {}
|
|
151
|
+
pos = 0
|
|
152
|
+
while pos < len(out):
|
|
153
|
+
eol = out.find(b"\n", pos)
|
|
154
|
+
if eol == -1:
|
|
155
|
+
break
|
|
156
|
+
header = out[pos:eol].decode()
|
|
157
|
+
pos = eol + 1
|
|
158
|
+
parts = header.split()
|
|
159
|
+
if len(parts) != 3:
|
|
160
|
+
# "<oid> missing" — nothing to consume for this entry.
|
|
161
|
+
continue
|
|
162
|
+
oid, _, size = parts[0], parts[1], int(parts[2])
|
|
163
|
+
contents[oid] = out[pos : pos + size]
|
|
164
|
+
pos += size + 1 # trailing newline
|
|
165
|
+
return contents
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
async def decrypt_blob(repo: str, content: bytes) -> bytes:
|
|
169
|
+
"""Decrypt a git-crypt blob; unencrypted content passes through.
|
|
170
|
+
|
|
171
|
+
Reading a blob bypasses the smudge filter, so apply it explicitly.
|
|
172
|
+
"""
|
|
173
|
+
if not content.startswith(GIT_CRYPT_MAGIC):
|
|
174
|
+
return content
|
|
175
|
+
async with decrypt_semaphore:
|
|
176
|
+
proc = await asyncio.create_subprocess_exec(
|
|
177
|
+
"git-crypt",
|
|
178
|
+
"smudge",
|
|
179
|
+
cwd=repo,
|
|
180
|
+
stdin=asyncio.subprocess.PIPE,
|
|
181
|
+
stdout=asyncio.subprocess.PIPE,
|
|
182
|
+
stderr=asyncio.subprocess.PIPE,
|
|
183
|
+
)
|
|
184
|
+
stdout, stderr = await proc.communicate(content)
|
|
185
|
+
if proc.returncode != 0:
|
|
186
|
+
raise Exception(f"git-crypt smudge failed in {repo}: {stderr.decode(errors='replace')}")
|
|
187
|
+
return stdout
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@asynccontextmanager
|
|
191
|
+
async def temp_repo_branch(git_repo_url: str, branch: str | None) -> AsyncGenerator[str, None]:
|
|
192
|
+
"""Checks out a git_repo_url to a temporary folder location. Returns temporary folder location"""
|
|
193
|
+
with tracer.start_as_current_span("checkout_temp_repo_branch"):
|
|
194
|
+
# Copy the repo cache to a temporary folder
|
|
195
|
+
with tempfile.TemporaryDirectory() as temporary_folder_path:
|
|
196
|
+
await clone_repo(git_repo_url, path=str(temporary_folder_path), branch=branch)
|
|
197
|
+
yield temporary_folder_path
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
import os
|
|
3
2
|
|
|
4
3
|
import httpx
|
|
5
4
|
|
|
6
|
-
|
|
5
|
+
from gitops_server.utils.github_auth import credentials
|
|
7
6
|
|
|
8
|
-
|
|
7
|
+
logger = logging.getLogger("github")
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
class IssueNotFound(Exception): ...
|
|
@@ -19,10 +18,9 @@ class STATUSES:
|
|
|
19
18
|
error = "error" # when error during helm deploy
|
|
20
19
|
|
|
21
20
|
|
|
22
|
-
def get_headers() -> dict[str, str]:
|
|
21
|
+
async def get_headers() -> dict[str, str]:
|
|
23
22
|
return {
|
|
24
|
-
|
|
25
|
-
"Content-Type": "application/json",
|
|
23
|
+
**await credentials.headers(),
|
|
26
24
|
"Accept": ("application/vnd.github.flash-preview+json, application/vnd.github.ant-man-preview+json"),
|
|
27
25
|
}
|
|
28
26
|
|
|
@@ -43,7 +41,7 @@ async def update_deployment(deployment_url: str, status: str, description: str,
|
|
|
43
41
|
response = await client.post(
|
|
44
42
|
status_url,
|
|
45
43
|
json=data,
|
|
46
|
-
headers=get_headers(),
|
|
44
|
+
headers=await get_headers(),
|
|
47
45
|
)
|
|
48
46
|
if response.status_code >= 300 and response.status_code != 404:
|
|
49
47
|
try:
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""GitHub credentials: a PAT in GITHUB_OAUTH_TOKEN when set, otherwise an App
|
|
2
|
+
installation token minted from the private key and cached until near expiry.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import time
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
|
|
11
|
+
import httpx
|
|
12
|
+
import jwt
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger("github_auth")
|
|
15
|
+
|
|
16
|
+
GITHUB_API = "https://api.github.com"
|
|
17
|
+
# Refresh with this much left, so a slow deploy never loses its token mid-flight.
|
|
18
|
+
MIN_REMAINING_SECONDS = 30 * 60
|
|
19
|
+
# GitHub rejects an App JWT that lives longer than this.
|
|
20
|
+
APP_JWT_TTL_SECONDS = 10 * 60
|
|
21
|
+
GIT_CREDENTIAL_ENV_VAR = "GITHUB_OAUTH_TOKEN"
|
|
22
|
+
|
|
23
|
+
# The credential helper travels with the token rather than living in the image, so
|
|
24
|
+
# it cannot drift from the code that supplies it. Scoped to github.com: an unscoped
|
|
25
|
+
# helper answers for every host git reaches, including across a redirect.
|
|
26
|
+
_CREDENTIAL_HELPER = f'!f() {{ printf "username=x-access-token\\npassword=%s\\n" "${GIT_CREDENTIAL_ENV_VAR}"; }}; f'
|
|
27
|
+
_HELPER_CONFIG = {
|
|
28
|
+
"GIT_CONFIG_COUNT": "2",
|
|
29
|
+
# An empty value resets the helper list. Without it git keeps whatever helper the
|
|
30
|
+
# environment already configures -- macOS ships credential.helper=osxkeychain in
|
|
31
|
+
# the Command Line Tools gitconfig -- and takes that answer instead of ours.
|
|
32
|
+
"GIT_CONFIG_KEY_0": "credential.helper",
|
|
33
|
+
"GIT_CONFIG_VALUE_0": "",
|
|
34
|
+
"GIT_CONFIG_KEY_1": "credential.https://github.com.helper",
|
|
35
|
+
"GIT_CONFIG_VALUE_1": _CREDENTIAL_HELPER,
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class GitHubCredentials:
|
|
40
|
+
"""Supplies a token for API calls and git clones, refreshing as needed.
|
|
41
|
+
|
|
42
|
+
Reads the environment on use, not at import, so a deployment can be swapped
|
|
43
|
+
between the PAT and the App without a code change or a restart.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(self) -> None:
|
|
47
|
+
self._token: str | None = None
|
|
48
|
+
self._expires_at: float = 0.0
|
|
49
|
+
# Without this two callers both find the cache stale and both mint.
|
|
50
|
+
self._lock = asyncio.Lock()
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def uses_app(self) -> bool:
|
|
54
|
+
return not os.environ.get("GITHUB_OAUTH_TOKEN", "").strip()
|
|
55
|
+
|
|
56
|
+
def describe(self) -> str:
|
|
57
|
+
if not self.uses_app:
|
|
58
|
+
return "GITHUB_OAUTH_TOKEN (personal access token)"
|
|
59
|
+
return (
|
|
60
|
+
f"GitHub App {os.environ.get('GITHUB_APP_ID', '')} "
|
|
61
|
+
f"installation {os.environ.get('GITHUB_INSTALLATION_ID', '')}"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def _app_jwt(self) -> str:
|
|
65
|
+
now = int(time.time())
|
|
66
|
+
return jwt.encode(
|
|
67
|
+
{"iat": now - 60, "exp": now + APP_JWT_TTL_SECONDS, "iss": os.environ.get("GITHUB_APP_ID", "")},
|
|
68
|
+
os.environ.get("GITHUB_APP_PRIVATE_KEY", ""),
|
|
69
|
+
algorithm="RS256",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
def _is_fresh(self) -> bool:
|
|
73
|
+
return bool(self._token) and time.time() < (self._expires_at - MIN_REMAINING_SECONDS)
|
|
74
|
+
|
|
75
|
+
async def token(self) -> str:
|
|
76
|
+
if not self.uses_app:
|
|
77
|
+
return os.environ["GITHUB_OAUTH_TOKEN"].strip()
|
|
78
|
+
if self._is_fresh():
|
|
79
|
+
assert self._token is not None
|
|
80
|
+
return self._token
|
|
81
|
+
|
|
82
|
+
async with self._lock:
|
|
83
|
+
# Another caller may have minted while we waited.
|
|
84
|
+
if self._is_fresh():
|
|
85
|
+
assert self._token is not None
|
|
86
|
+
return self._token
|
|
87
|
+
|
|
88
|
+
async with httpx.AsyncClient(timeout=30.0) as client:
|
|
89
|
+
response = await client.post(
|
|
90
|
+
f"{GITHUB_API}/app/installations/{os.environ.get('GITHUB_INSTALLATION_ID', '')}/access_tokens",
|
|
91
|
+
headers={
|
|
92
|
+
"Authorization": f"Bearer {self._app_jwt()}",
|
|
93
|
+
"Accept": "application/vnd.github+json",
|
|
94
|
+
},
|
|
95
|
+
)
|
|
96
|
+
response.raise_for_status()
|
|
97
|
+
data = response.json()
|
|
98
|
+
|
|
99
|
+
token = str(data["token"])
|
|
100
|
+
self._token = token
|
|
101
|
+
self._expires_at = datetime.fromisoformat(data["expires_at"].replace("Z", "+00:00")).timestamp()
|
|
102
|
+
logger.info("GitHub installation token refreshed, expires at %s", data["expires_at"])
|
|
103
|
+
return token
|
|
104
|
+
|
|
105
|
+
async def headers(self) -> dict[str, str]:
|
|
106
|
+
return {
|
|
107
|
+
"Authorization": f"Bearer {await self.token()}",
|
|
108
|
+
"Content-Type": "application/json",
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async def git_env(self) -> dict[str, str]:
|
|
112
|
+
"""Minted per command, so a long-lived process never pins an expiring token."""
|
|
113
|
+
return {GIT_CREDENTIAL_ENV_VAR: await self.token(), **_HELPER_CONFIG}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def validate_credentials() -> None:
|
|
117
|
+
if os.environ.get("GITHUB_OAUTH_TOKEN", "").strip():
|
|
118
|
+
return
|
|
119
|
+
required = ("GITHUB_APP_ID", "GITHUB_INSTALLATION_ID", "GITHUB_APP_PRIVATE_KEY")
|
|
120
|
+
if missing := [name for name in required if not os.environ.get(name)]:
|
|
121
|
+
raise RuntimeError(
|
|
122
|
+
"No GitHub credential configured: set GITHUB_OAUTH_TOKEN, or all of "
|
|
123
|
+
f"{', '.join(required)} (missing: {', '.join(missing)})"
|
|
124
|
+
)
|
|
125
|
+
# Signing parses the key, so an unusable one fails here rather than on the
|
|
126
|
+
# first deploy. A revoked key or a wrong id still only surfaces on first use.
|
|
127
|
+
try:
|
|
128
|
+
credentials._app_jwt()
|
|
129
|
+
except Exception as exc:
|
|
130
|
+
raise RuntimeError(f"GITHUB_APP_PRIVATE_KEY cannot sign a JWT: {exc}") from exc
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
credentials = GitHubCredentials()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.9.0"
|