generic-gitlab-cicd 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- generic_ci/__init__.py +3 -0
- generic_ci/__main__.py +3 -0
- generic_ci/cli.py +110 -0
- generic_ci/compiler.py +242 -0
- generic_ci/config.py +148 -0
- generic_ci/dependencies.py +186 -0
- generic_ci/models.py +189 -0
- generic_ci/runtime.py +482 -0
- generic_ci/sources.py +314 -0
- generic_ci/workflows/__init__.py +1 -0
- generic_ci/workflows/compiler.py +316 -0
- generic_ci/workflows/ecosystems.py +185 -0
- generic_ci/workflows/helm.py +168 -0
- generic_ci/workflows/models.py +222 -0
- generic_ci/workflows/publication.py +82 -0
- generic_ci/workflows/runtime.py +421 -0
- generic_gitlab_cicd-0.3.2.dist-info/METADATA +170 -0
- generic_gitlab_cicd-0.3.2.dist-info/RECORD +21 -0
- generic_gitlab_cicd-0.3.2.dist-info/WHEEL +5 -0
- generic_gitlab_cicd-0.3.2.dist-info/entry_points.txt +2 -0
- generic_gitlab_cicd-0.3.2.dist-info/top_level.txt +1 -0
generic_ci/runtime.py
ADDED
|
@@ -0,0 +1,482 @@
|
|
|
1
|
+
"""Job execution and receipts. No runtime dependency/tool downloads are bootstrapped."""
|
|
2
|
+
import base64
|
|
3
|
+
import contextlib
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import re
|
|
9
|
+
import shutil
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
import tempfile
|
|
13
|
+
import time
|
|
14
|
+
import tomllib
|
|
15
|
+
|
|
16
|
+
from packaging.version import Version
|
|
17
|
+
import yaml
|
|
18
|
+
|
|
19
|
+
from . import __version__
|
|
20
|
+
from .compiler import digest
|
|
21
|
+
from .config import affected, allowed_url
|
|
22
|
+
from .dependencies import prepared, resolve_candidates
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def write_json(path, value):
|
|
26
|
+
path = Path(path)
|
|
27
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
28
|
+
temporary = path.with_suffix(".tmp")
|
|
29
|
+
temporary.write_text(json.dumps(value, sort_keys=True, indent=2) + "\n")
|
|
30
|
+
temporary.replace(path)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def path_in(root, relative):
|
|
34
|
+
root = Path(root).resolve()
|
|
35
|
+
path = (root / relative).resolve()
|
|
36
|
+
if not path.is_relative_to(root):
|
|
37
|
+
raise ValueError(f"path escapes repository: {relative}")
|
|
38
|
+
return path
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def identity(config_hash):
|
|
42
|
+
return {"config": config_hash, "commit": os.environ.get("CI_COMMIT_SHA", "local"),
|
|
43
|
+
"pipeline": os.environ.get("CI_PIPELINE_ID", "local")}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def load_config(expected):
|
|
47
|
+
data = json.loads(base64.b64decode(os.environ["TOOLKIT_CONFIG_B64"], validate=True))
|
|
48
|
+
if digest(data) != expected:
|
|
49
|
+
raise ValueError("generated configuration fingerprint mismatch")
|
|
50
|
+
if data["version"] != __version__:
|
|
51
|
+
raise ValueError(f"runtime {__version__} does not match compiler {data['version']}; rebuild internal runtime image")
|
|
52
|
+
return data
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def make_plan(data, root, expected):
|
|
56
|
+
for path, sha in data["sources"].items():
|
|
57
|
+
if hashlib.sha256(path_in(root, path).read_bytes()).hexdigest() != sha:
|
|
58
|
+
raise ValueError(f"{path} changed: run generic-ci render and commit generated CI")
|
|
59
|
+
full = os.environ.get("CI_FULL_PIPELINE") == "true" or bool(os.environ.get("CI_COMMIT_TAG"))
|
|
60
|
+
full |= os.environ.get("CI_PIPELINE_SOURCE") in {"web", "api", "schedule", "trigger", "pipeline"}
|
|
61
|
+
changed = None
|
|
62
|
+
base = os.environ.get("CI_MERGE_REQUEST_DIFF_BASE_SHA") or os.environ.get("CI_COMMIT_BEFORE_SHA", "")
|
|
63
|
+
if not full and re.fullmatch(r"[0-9a-f]{40}", base) and set(base) != {"0"}:
|
|
64
|
+
result = subprocess.run(["git", "diff", "--name-only", "--no-renames", base, "HEAD"], cwd=root, capture_output=True, text=True)
|
|
65
|
+
if result.returncode == 0:
|
|
66
|
+
changed = result.stdout.splitlines()
|
|
67
|
+
if set(changed) & set(data["sources"]):
|
|
68
|
+
changed = None
|
|
69
|
+
candidates = resolve_candidates(os.environ, data["platform"]["allowed_hosts"], data["pipeline"]["projects"])
|
|
70
|
+
selected = affected(data["pipeline"], changed)
|
|
71
|
+
tag = os.environ.get("CI_COMMIT_TAG", "")
|
|
72
|
+
if tag:
|
|
73
|
+
release_projects = {name: p["release"] for name, p in data["pipeline"]["projects"].items() if p["release"]}
|
|
74
|
+
if release_projects:
|
|
75
|
+
selected = {name for name, r in release_projects.items() if re.fullmatch(re.escape(r["tag"]).replace(re.escape("{version}"), ".+"), tag)}
|
|
76
|
+
if not selected:
|
|
77
|
+
raise ValueError("tag does not match any declared project release")
|
|
78
|
+
for candidate in candidates:
|
|
79
|
+
selected.update(candidate["projects"])
|
|
80
|
+
# Required producer work must exist even when only a consumer changed.
|
|
81
|
+
# Shared deployments rebuild the entire declared image set, preserving workloads.
|
|
82
|
+
while True:
|
|
83
|
+
expanded = selected | {data["nodes"][upstream]["project"] for node in data["nodes"].values()
|
|
84
|
+
if node["project"] in selected for upstream in node["needs"]}
|
|
85
|
+
if expanded == selected:
|
|
86
|
+
break
|
|
87
|
+
selected = expanded
|
|
88
|
+
plan = {**identity(expected), "selected": sorted(selected), "candidates": candidates, "created": time.time()}
|
|
89
|
+
write_json(root / ".ci-out/plan.json", plan)
|
|
90
|
+
print(json.dumps({"selected": plan["selected"], "candidates": candidates}, indent=2))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def receipt_path(root, job):
|
|
94
|
+
return root / ".ci-out" / job / "receipt.json"
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def require_receipt(root, job, expected):
|
|
98
|
+
path = receipt_path(root, job)
|
|
99
|
+
if not path.is_file():
|
|
100
|
+
raise ValueError(f"required job {job} has no success receipt (skipped, failed, or expired artifacts)")
|
|
101
|
+
record = json.loads(path.read_text())
|
|
102
|
+
if any(record.get(key) != value for key, value in identity(expected).items()) or record.get("status") != "passed":
|
|
103
|
+
raise ValueError(f"required job {job} has stale or unsuccessful evidence")
|
|
104
|
+
for artifact in record.get("files", []):
|
|
105
|
+
source = path_in(root, artifact["artifact"])
|
|
106
|
+
if not source.is_file() or hashlib.sha256(source.read_bytes()).hexdigest() != artifact["sha256"]:
|
|
107
|
+
raise ValueError(f"artifact missing or modified: {source}")
|
|
108
|
+
return record
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def commands(lines, cwd, shell, environment=None):
|
|
112
|
+
environment = {**os.environ, **(environment or {})}
|
|
113
|
+
if not lines:
|
|
114
|
+
return
|
|
115
|
+
if shell == "powershell":
|
|
116
|
+
script = "$ErrorActionPreference='Stop'\n" + "\n".join(line + "\nif ($LASTEXITCODE) { exit $LASTEXITCODE }" for line in lines)
|
|
117
|
+
subprocess.run(["pwsh", "-NoProfile", "-NonInteractive", "-Command", script], cwd=cwd, env=environment, check=True)
|
|
118
|
+
else:
|
|
119
|
+
subprocess.run(["sh", "-eu", "-c", "\n".join(lines)], cwd=cwd, env=environment, check=True)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def collect(root, directory, paths, out):
|
|
123
|
+
files = []
|
|
124
|
+
for value in paths:
|
|
125
|
+
source = path_in(directory, value)
|
|
126
|
+
if not source.exists():
|
|
127
|
+
raise ValueError(f"declared output is missing: {source}")
|
|
128
|
+
for file in sorted(source.rglob("*")) if source.is_dir() else [source]:
|
|
129
|
+
if file.is_symlink():
|
|
130
|
+
raise ValueError("artifact symlinks are not supported")
|
|
131
|
+
if not file.is_file():
|
|
132
|
+
continue
|
|
133
|
+
relative = file.relative_to(root)
|
|
134
|
+
target = out / "files" / relative
|
|
135
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
136
|
+
shutil.copyfile(file, target)
|
|
137
|
+
files.append({"path": relative.as_posix(), "artifact": target.relative_to(root).as_posix(),
|
|
138
|
+
"sha256": hashlib.sha256(target.read_bytes()).hexdigest()})
|
|
139
|
+
return files
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def materialize(root, records):
|
|
143
|
+
written = {}
|
|
144
|
+
for record in records:
|
|
145
|
+
for file in record.get("files", []):
|
|
146
|
+
if file["path"] in written and written[file["path"]] != file["sha256"]:
|
|
147
|
+
raise ValueError(f"upstream output collision: {file['path']}")
|
|
148
|
+
written[file["path"]] = file["sha256"]
|
|
149
|
+
target = path_in(root, file["path"])
|
|
150
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
151
|
+
source = path_in(root, file["artifact"])
|
|
152
|
+
if source != target:
|
|
153
|
+
shutil.copyfile(source, target)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def version(directory, settings, *, check_bump=False):
|
|
157
|
+
path = path_in(directory, settings["version_file"])
|
|
158
|
+
metadata = tomllib.loads(path.read_text())["project"]
|
|
159
|
+
if "version" in metadata.get("dynamic", []) or "version" not in metadata:
|
|
160
|
+
raise ValueError("static project.version required; dynamic version strategy is not supported")
|
|
161
|
+
current = Version(metadata["version"])
|
|
162
|
+
if current.local:
|
|
163
|
+
raise ValueError("release version cannot have a local suffix")
|
|
164
|
+
tag = os.environ.get("CI_COMMIT_TAG", "")
|
|
165
|
+
if tag:
|
|
166
|
+
owns_tag = bool(re.fullmatch(re.escape(settings["tag"]).replace(re.escape("{version}"), ".+"), tag))
|
|
167
|
+
if (owns_tag or not check_bump) and settings["tag"].format(version=current) != tag:
|
|
168
|
+
raise ValueError(f"tag must match {settings['tag'].format(version=current)}")
|
|
169
|
+
elif check_bump and settings["bump"] and os.environ.get("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"):
|
|
170
|
+
target = os.environ["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"]
|
|
171
|
+
subprocess.run(["git", "fetch", "--no-tags", "origin", f"+refs/heads/{target}:refs/ci/version-base"], check=True)
|
|
172
|
+
root = Path(os.environ["CI_PROJECT_DIR"]).resolve()
|
|
173
|
+
previous = subprocess.run(["git", "show", f"refs/ci/version-base:{path.relative_to(root).as_posix()}"], capture_output=True, text=True)
|
|
174
|
+
if previous.returncode:
|
|
175
|
+
raise ValueError("cannot read baseline version; initialize a new package explicitly before release validation")
|
|
176
|
+
if current <= Version(tomllib.loads(previous.stdout)["project"]["version"]):
|
|
177
|
+
raise ValueError("bump project.version above target branch")
|
|
178
|
+
return str(current)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def bundle_dependencies(environment, directory, out, hosts):
|
|
182
|
+
"""Build an offline, hash-locked wheelhouse from the actual checked environment."""
|
|
183
|
+
name = tomllib.loads((directory / "pyproject.toml").read_text())["project"]["name"]
|
|
184
|
+
code = '''import importlib.metadata as m,json,sys
|
|
185
|
+
rows=[]
|
|
186
|
+
for d in m.distributions():
|
|
187
|
+
if d.metadata['Name'].lower().replace('_','-') == sys.argv[1].lower().replace('_','-'): continue
|
|
188
|
+
source=json.loads(d.read_text('direct_url.json') or '{}')
|
|
189
|
+
url=source.get('url')
|
|
190
|
+
if source.get('vcs_info'): url='git+'+url+'@'+source['vcs_info']['commit_id']
|
|
191
|
+
if source.get('subdirectory') and url:
|
|
192
|
+
from urllib.parse import quote
|
|
193
|
+
url += '#subdirectory='+quote(source['subdirectory'], safe='/')
|
|
194
|
+
rows.append(d.metadata['Name']+' @ '+url if url else d.metadata['Name']+'=='+d.version)
|
|
195
|
+
print('\\n'.join(rows))'''
|
|
196
|
+
requirements = subprocess.check_output([environment["interpreter"], "-c", code, name], text=True)
|
|
197
|
+
request = out / "resolved-requirements.txt"
|
|
198
|
+
request.write_text(requirements)
|
|
199
|
+
wheels = out / "wheels"
|
|
200
|
+
wheels.mkdir(parents=True, exist_ok=True)
|
|
201
|
+
subprocess.run([sys.executable, "-m", "pip", "wheel", "--no-deps", "-r", str(request), "--wheel-dir", str(wheels)], check=True)
|
|
202
|
+
from packaging.utils import parse_wheel_filename
|
|
203
|
+
locked = []
|
|
204
|
+
for wheel in sorted(wheels.glob("*.whl")):
|
|
205
|
+
package, ver, _, _ = parse_wheel_filename(wheel.name)
|
|
206
|
+
locked.append(f"{package}=={ver} --hash=sha256:{hashlib.sha256(wheel.read_bytes()).hexdigest()}")
|
|
207
|
+
(out / "requirements.txt").write_text("\n".join(locked) + "\n")
|
|
208
|
+
write_json(out / "provenance.json", {k: v for k, v in environment.items() if k != "interpreter"})
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def build_container(data, project_name, project, settings, root, directory, out, plan, records):
|
|
212
|
+
infra = data["platform"]
|
|
213
|
+
candidates = [c for c in plan["candidates"] if project_name in c["projects"]]
|
|
214
|
+
if candidates and not settings["dependency_bundle"]:
|
|
215
|
+
raise ValueError("candidate image requires container.dependency-bundle and the documented Dockerfile recipe")
|
|
216
|
+
is_preview = bool(os.environ.get("CI_MERGE_REQUEST_IID"))
|
|
217
|
+
protected = os.environ.get("CI_COMMIT_REF_PROTECTED") == "true"
|
|
218
|
+
same_project = os.environ.get("CI_MERGE_REQUEST_SOURCE_PROJECT_ID") == os.environ.get("CI_PROJECT_ID")
|
|
219
|
+
push = protected or (is_preview and same_project)
|
|
220
|
+
if candidates and not is_preview:
|
|
221
|
+
push = False
|
|
222
|
+
repository = settings["repository"] or infra["registry"].rstrip("/") + "/" + project_name
|
|
223
|
+
if is_preview:
|
|
224
|
+
repository = infra["preview_registry"].rstrip("/") + "/" + project_name
|
|
225
|
+
reference = repository + ":" + os.environ.get("CI_COMMIT_SHA", "local") + "-" + os.environ.get("CI_PIPELINE_ID", "local")
|
|
226
|
+
context = settings["context"]
|
|
227
|
+
context_path = path_in(root, context["repository"]) if isinstance(context, dict) else path_in(directory, context)
|
|
228
|
+
dockerfile = path_in(directory, settings["dockerfile"])
|
|
229
|
+
args = ["buildctl-daemonless.sh", "build", "--frontend", "dockerfile.v0", "--local", f"context={context_path}",
|
|
230
|
+
"--local", f"dockerfile={dockerfile.parent}", "--opt", f"filename={dockerfile.name}", "--opt", f"platform={settings['platform']}",
|
|
231
|
+
"--metadata-file", str(out / "image.json")]
|
|
232
|
+
for name, value in settings["build_args"].items():
|
|
233
|
+
args += ["--opt", f"build-arg:{name}={os.path.expandvars(value)}"]
|
|
234
|
+
if settings["target"]:
|
|
235
|
+
args += ["--opt", f"target={settings['target']}"]
|
|
236
|
+
for name, variable in settings["secrets"].items():
|
|
237
|
+
secret = os.environ.get(variable)
|
|
238
|
+
if not secret or not Path(secret).is_file():
|
|
239
|
+
raise ValueError(f"missing BuildKit secret file variable {variable}")
|
|
240
|
+
args += ["--secret", f"id={name},src={secret}"]
|
|
241
|
+
if settings["dependency_bundle"]:
|
|
242
|
+
bundle = out / "dependencies"
|
|
243
|
+
bundle.mkdir()
|
|
244
|
+
with prepared(directory, root, project["dependencies"], plan["candidates"], project_name, infra["allowed_hosts"], bundle) as environment:
|
|
245
|
+
if environment["manager"] != "uv":
|
|
246
|
+
raise ValueError("dependency-bundle requires uv")
|
|
247
|
+
# Check canonical source checks tested this exact universal lock.
|
|
248
|
+
for record in records:
|
|
249
|
+
previous = record.get("dependencies", {})
|
|
250
|
+
previous_policy = {k: v for k, v in previous.get("policy", {}).items() if k != "python"}
|
|
251
|
+
canonical_policy = {k: v for k, v in project["dependencies"].items() if k != "python"}
|
|
252
|
+
if previous_policy == canonical_policy and previous.get("lock_sha256") != environment["lock_sha256"]:
|
|
253
|
+
raise ValueError("container dependency resolution differs from required check; commit upgraded lock before building")
|
|
254
|
+
bundle_dependencies(environment, directory, bundle, infra["allowed_hosts"])
|
|
255
|
+
args += ["--local", f"ci-dependencies={bundle}", "--opt", "context:ci-dependencies=local:ci-dependencies"]
|
|
256
|
+
args += ["--output", f"type=image,name={reference},push=true" if push else f"type=oci,dest={out / 'image.oci.tar'}"]
|
|
257
|
+
with tempfile.TemporaryDirectory() as auth:
|
|
258
|
+
variable = "PREVIEW_REGISTRY_AUTH_FILE" if is_preview else "REGISTRY_AUTH_FILE"
|
|
259
|
+
if push and not os.environ.get(variable):
|
|
260
|
+
raise ValueError(f"set {variable} to an internal registry auth file")
|
|
261
|
+
if os.environ.get(variable):
|
|
262
|
+
shutil.copyfile(os.environ[variable], Path(auth) / "config.json")
|
|
263
|
+
flags = os.environ.get("BUILDKITD_FLAGS", "--oci-worker-no-process-sandbox")
|
|
264
|
+
if os.environ.get("BUILDKIT_CONFIG_FILE"):
|
|
265
|
+
flags += " --config " + os.environ["BUILDKIT_CONFIG_FILE"]
|
|
266
|
+
subprocess.run(args, cwd=root, env={**os.environ, "DOCKER_CONFIG": auth, "BUILDKITD_FLAGS": flags}, check=True)
|
|
267
|
+
image_digest = json.loads((out / "image.json").read_text()).get("containerimage.digest", "")
|
|
268
|
+
if not re.fullmatch(r"sha256:[a-f0-9]{64}", image_digest):
|
|
269
|
+
raise ValueError("BuildKit did not return an image digest")
|
|
270
|
+
return {"image": {"repository": repository, "digest": image_digest, "pushed": push}, "candidates": candidates}
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def set_value(values, path, value):
|
|
274
|
+
for part in path[:-1]:
|
|
275
|
+
if not part or not isinstance(values, dict):
|
|
276
|
+
raise ValueError("invalid chart image binding")
|
|
277
|
+
values = values.setdefault(part, {})
|
|
278
|
+
if path[-1] in values:
|
|
279
|
+
raise ValueError("overlapping chart image bindings")
|
|
280
|
+
values[path[-1]] = value
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def deploy(data, node, root, directory, out, plan, records, *, stop=False):
|
|
284
|
+
settings, infra = node["settings"], data["platform"]
|
|
285
|
+
target = infra["targets"][settings["target"]]
|
|
286
|
+
preview = settings["preview"]
|
|
287
|
+
if target["production"] and (os.environ.get("CI_COMMIT_REF_PROTECTED") != "true" or plan.get("candidates")):
|
|
288
|
+
raise ValueError("production requires a protected ref and an ordinary dependency run")
|
|
289
|
+
suffix = f"-mr-{os.environ.get('CI_PROJECT_ID', '')}-{os.environ.get('CI_MERGE_REQUEST_IID', '')}" if preview else ""
|
|
290
|
+
release_name = target["release_prefix"] + "-" + node["project"] + "-" + settings["name"] + suffix
|
|
291
|
+
if len(release_name) > 53:
|
|
292
|
+
release_name = release_name[:40].rstrip("-") + "-" + hashlib.sha256(release_name.encode()).hexdigest()[:12]
|
|
293
|
+
namespace = os.path.expandvars(target["namespace"])
|
|
294
|
+
environment = {"KUBECONFIG": os.environ[target["kubeconfig_variable"]], "TOOLKIT_DEPLOY_URL": os.path.expandvars(target["url"])}
|
|
295
|
+
if stop:
|
|
296
|
+
subprocess.run(["helm", "uninstall", release_name, "--namespace", namespace, "--ignore-not-found", "--wait"], env={**os.environ, **environment}, check=True)
|
|
297
|
+
return {"release": release_name, "namespace": namespace, "cleanup": "chart-owned resources only"}
|
|
298
|
+
images = {data["nodes"][job]["project"]: record["image"] for job, record in zip(node["needs"], records) if "image" in record}
|
|
299
|
+
overlay = {}
|
|
300
|
+
for project, binding in settings["images"].items():
|
|
301
|
+
if project not in images or not images[project]["pushed"]:
|
|
302
|
+
raise ValueError(f"no pushed image for {project}; deploy from a protected branch/tag or same-project MR")
|
|
303
|
+
set_value(overlay, binding["repository"], images[project]["repository"])
|
|
304
|
+
set_value(overlay, binding["digest"], images[project]["digest"])
|
|
305
|
+
write_json(out / "image-values.json", overlay)
|
|
306
|
+
chart = settings["chart"] or infra["chart"]
|
|
307
|
+
if not chart.startswith("oci://"):
|
|
308
|
+
chart = str(path_in(directory if settings["chart"] else root, chart))
|
|
309
|
+
args = [release_name, chart, "--namespace", namespace]
|
|
310
|
+
if chart.startswith("oci://"):
|
|
311
|
+
args += ["--version", settings["chart_version"] or infra["chart_version"]]
|
|
312
|
+
for file in settings["values"]:
|
|
313
|
+
args += ["--values", str(path_in(directory, file))]
|
|
314
|
+
args += ["--values", str(out / "image-values.json")]
|
|
315
|
+
# Render and ensure each promised image binding actually appears in the workload manifest.
|
|
316
|
+
rendered = subprocess.check_output(["helm", "template", *args], env={**os.environ, **environment}, text=True)
|
|
317
|
+
workload_images = set()
|
|
318
|
+
def scan(value):
|
|
319
|
+
if isinstance(value, dict):
|
|
320
|
+
for key, child in value.items():
|
|
321
|
+
if key in {"containers", "initContainers"} and isinstance(child, list):
|
|
322
|
+
workload_images.update(c.get("image") for c in child if isinstance(c, dict))
|
|
323
|
+
scan(child)
|
|
324
|
+
elif isinstance(value, list):
|
|
325
|
+
for child in value:
|
|
326
|
+
scan(child)
|
|
327
|
+
documents = list(yaml.safe_load_all(rendered))
|
|
328
|
+
for document in documents:
|
|
329
|
+
scan(document)
|
|
330
|
+
for image in images.values():
|
|
331
|
+
if image["repository"] + "@" + image["digest"] not in workload_images:
|
|
332
|
+
raise ValueError("custom chart did not consume a required digest-pinned image binding")
|
|
333
|
+
for document in documents:
|
|
334
|
+
if isinstance(document, dict) and document.get("kind") == "Secret":
|
|
335
|
+
for key in ("data", "stringData"):
|
|
336
|
+
if key in document:
|
|
337
|
+
document[key] = {name: "REDACTED" for name in document[key]}
|
|
338
|
+
(out / "rendered.yaml").write_text(yaml.safe_dump_all(documents))
|
|
339
|
+
commands(settings["before"], directory, node["shell"], environment)
|
|
340
|
+
major = subprocess.check_output(["helm", "version", "--template", "{{.Version}}"], text=True)
|
|
341
|
+
flag = "--atomic" if major.startswith("v3.") else "--rollback-on-failure" if major.startswith("v4.") else None
|
|
342
|
+
if not flag:
|
|
343
|
+
raise ValueError("unsupported Helm version")
|
|
344
|
+
subprocess.run(["helm", "upgrade", "--install", *args, "--create-namespace", "--wait", "--timeout", "10m", flag], env={**os.environ, **environment}, check=True)
|
|
345
|
+
# Explicitly not a database rollback: after-check failure is surfaced to GitLab.
|
|
346
|
+
commands(settings["after"], directory, node["shell"], environment)
|
|
347
|
+
return {"release": release_name, "namespace": namespace, "images": images}
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def publish(data, node, directory, root, out, plan, records):
|
|
351
|
+
if plan["candidates"] or os.environ.get("CI_COMMIT_REF_PROTECTED") != "true" or not os.environ.get("CI_COMMIT_TAG"):
|
|
352
|
+
raise ValueError("publication requires a protected tag without candidate overrides")
|
|
353
|
+
settings = node["settings"]
|
|
354
|
+
current = version(directory, settings)
|
|
355
|
+
project = data["pipeline"]["projects"][node["project"]]
|
|
356
|
+
result = {"version": current, "tag": os.environ["CI_COMMIT_TAG"], "publications": []}
|
|
357
|
+
package = project["package"]
|
|
358
|
+
if package is not None:
|
|
359
|
+
if not package["index"]:
|
|
360
|
+
raise ValueError("package.index must select a named tool.uv.index for publication")
|
|
361
|
+
package_directory = path_in(directory, package["directory"])
|
|
362
|
+
metadata = tomllib.loads((package_directory / "pyproject.toml").read_text())
|
|
363
|
+
indexes = [r for r in metadata.get("tool", {}).get("uv", {}).get("index", []) if r.get("name") == package["index"]]
|
|
364
|
+
if len(indexes) != 1 or not indexes[0].get("publish-url"):
|
|
365
|
+
raise ValueError("selected uv index must define exactly one publish-url")
|
|
366
|
+
for key in ("url", "publish-url"):
|
|
367
|
+
allowed_url(indexes[0][key], data["platform"]["allowed_hosts"])
|
|
368
|
+
package_records = [r for r in records if r.get("action") == "package" and r["project"] == node["project"]]
|
|
369
|
+
if len(package_records) != 1 or package_records[0]["package"]["version"] != current:
|
|
370
|
+
raise ValueError("built package version does not match release version")
|
|
371
|
+
files = [path_in(root, f["artifact"]) for r in package_records for f in r.get("files", []) if f["path"].endswith((".whl", ".tar.gz"))]
|
|
372
|
+
if not any(p.suffix == ".whl" for p in files) or not any(p.name.endswith(".tar.gz") for p in files):
|
|
373
|
+
raise ValueError("release requires built wheel and sdist artifacts")
|
|
374
|
+
subprocess.run(["uv", "publish", "--index", package["index"], *map(str, files)], cwd=package_directory, check=True)
|
|
375
|
+
result["publications"].append({"index": package["index"], "files": [{"name": p.name, "sha256": hashlib.sha256(p.read_bytes()).hexdigest()} for p in files]})
|
|
376
|
+
write_json(out / "publication.json", result)
|
|
377
|
+
if settings["gitlab_release"]:
|
|
378
|
+
subprocess.run(["glab", "release", "create", os.environ["CI_COMMIT_TAG"], "--notes-file", str(path_in(directory, settings["notes"]))], cwd=root, check=True)
|
|
379
|
+
return result
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def run_job(data, identifier, root, expected):
|
|
383
|
+
node = data["nodes"][identifier]
|
|
384
|
+
out = root / ".ci-out" / identifier
|
|
385
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
386
|
+
# Never retain a success marker from an earlier retry, including shell executors.
|
|
387
|
+
receipt_path(root, identifier).unlink(missing_ok=True)
|
|
388
|
+
if node["action"] == "stop":
|
|
389
|
+
return deploy(data, node, root, root, out, {}, [], stop=True)
|
|
390
|
+
plan = json.loads((root / ".ci-out/plan.json").read_text())
|
|
391
|
+
if any(plan.get(k) != v for k, v in identity(expected).items()):
|
|
392
|
+
raise ValueError("pipeline plan is stale")
|
|
393
|
+
project = data["pipeline"]["projects"][node["project"]]
|
|
394
|
+
if node["project"] not in plan["selected"]:
|
|
395
|
+
print("Project unaffected; no success evidence emitted")
|
|
396
|
+
return
|
|
397
|
+
records = [require_receipt(root, job, expected) for job in node["needs"]]
|
|
398
|
+
materialize(root, records)
|
|
399
|
+
directory = path_in(root, project["path"])
|
|
400
|
+
action, settings = node["action"], node["settings"]
|
|
401
|
+
result = {**identity(expected), "status": "passed", "project": node["project"], "action": action, "created": time.time(),
|
|
402
|
+
"candidates": plan["candidates"], "files": []}
|
|
403
|
+
if action in {"check", "step"}:
|
|
404
|
+
policy = settings.get("dependencies") or project["dependencies"]
|
|
405
|
+
# Matrix PYTHON_VERSION is a public convention and is checked, not just used in an image tag.
|
|
406
|
+
policy = {**policy}
|
|
407
|
+
if "PYTHON_VERSION" in node["variables"]:
|
|
408
|
+
policy["python"] = node["variables"]["PYTHON_VERSION"]
|
|
409
|
+
with prepared(directory, root, policy, plan["candidates"], node["project"], data["platform"]["allowed_hosts"], out) as environment:
|
|
410
|
+
result["dependencies"] = {k: v for k, v in environment.items() if k != "interpreter"}
|
|
411
|
+
image_env = {"TOOLKIT_IMAGE_" + r["project"].upper().replace("-", "_"): r["image"]["repository"] + "@" + r["image"]["digest"] for r in records if "image" in r}
|
|
412
|
+
try:
|
|
413
|
+
commands(settings["script"], directory, node["shell"], image_env)
|
|
414
|
+
finally:
|
|
415
|
+
reports = out / "reports"
|
|
416
|
+
reports.mkdir(exist_ok=True)
|
|
417
|
+
for index, pattern in enumerate(settings.get("junit", [])):
|
|
418
|
+
for j, file in enumerate(directory.glob(pattern)):
|
|
419
|
+
shutil.copyfile(file, reports / f"{index}-{j}.xml")
|
|
420
|
+
result["files"] = collect(root, directory, settings.get("outputs", {}).values(), out)
|
|
421
|
+
elif action == "package":
|
|
422
|
+
package_directory = path_in(directory, settings["directory"])
|
|
423
|
+
from .dependencies import validate_indexes
|
|
424
|
+
metadata = validate_indexes(package_directory, data["platform"]["allowed_hosts"])["project"]
|
|
425
|
+
if project["release"]:
|
|
426
|
+
version(directory, project["release"], check_bump=True)
|
|
427
|
+
dist = out / "dist"
|
|
428
|
+
subprocess.run(["uv", "build", "--no-sources", "--out-dir", str(dist)], cwd=package_directory, check=True)
|
|
429
|
+
files = sorted(dist.glob("*"))
|
|
430
|
+
from packaging.utils import parse_wheel_filename, parse_sdist_filename
|
|
431
|
+
for file in files:
|
|
432
|
+
if file.name.endswith(".whl"):
|
|
433
|
+
_, built_version, _, _ = parse_wheel_filename(file.name)
|
|
434
|
+
elif file.name.endswith(".tar.gz"):
|
|
435
|
+
_, built_version = parse_sdist_filename(file.name)
|
|
436
|
+
else:
|
|
437
|
+
raise ValueError(f"unexpected package build artifact {file.name}")
|
|
438
|
+
if built_version != Version(metadata["version"]):
|
|
439
|
+
raise ValueError("built distribution version does not match pyproject")
|
|
440
|
+
result["package"] = {"name": metadata["name"], "version": str(Version(metadata["version"]))}
|
|
441
|
+
subprocess.run([sys.executable, "-m", "twine", "check", *map(str, files)], check=True)
|
|
442
|
+
# Install the built wheel in an isolated environment; source-tree tests alone are insufficient.
|
|
443
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
444
|
+
subprocess.run(["uv", "venv", "--no-python-downloads", tmp], check=True)
|
|
445
|
+
interpreter = Path(tmp) / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
|
446
|
+
subprocess.run(["uv", "pip", "install", "--python", str(interpreter), *map(str, dist.glob("*.whl"))], check=True)
|
|
447
|
+
subprocess.run(["uv", "pip", "check", "--python", str(interpreter)], check=True)
|
|
448
|
+
result["files"] = [{"path": f.relative_to(root).as_posix(), "artifact": f.relative_to(root).as_posix(), "sha256": hashlib.sha256(f.read_bytes()).hexdigest()} for f in files]
|
|
449
|
+
elif action == "container":
|
|
450
|
+
if project["release"]:
|
|
451
|
+
version(directory, project["release"], check_bump=True)
|
|
452
|
+
result.update(build_container(data, node["project"], project, settings, root, directory, out, plan, records))
|
|
453
|
+
elif action == "deploy":
|
|
454
|
+
with prepared(directory, root, project["dependencies"], plan["candidates"], node["project"], data["platform"]["allowed_hosts"], out) as environment:
|
|
455
|
+
result["dependencies"] = {k: v for k, v in environment.items() if k != "interpreter"}
|
|
456
|
+
result.update(deploy(data, node, root, directory, out, plan, records))
|
|
457
|
+
elif action == "release":
|
|
458
|
+
result.update(publish(data, node, directory, root, out, plan, records))
|
|
459
|
+
else:
|
|
460
|
+
raise ValueError(f"unsupported action {action}")
|
|
461
|
+
write_json(receipt_path(root, identifier), result)
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def main():
|
|
465
|
+
try:
|
|
466
|
+
command = sys.argv[1]
|
|
467
|
+
expected = sys.argv[-1]
|
|
468
|
+
data = load_config(expected)
|
|
469
|
+
root = Path(os.environ.get("CI_PROJECT_DIR", ".")).resolve()
|
|
470
|
+
if command == "plan":
|
|
471
|
+
make_plan(data, root, expected)
|
|
472
|
+
elif command == "run":
|
|
473
|
+
run_job(data, sys.argv[2], root, expected)
|
|
474
|
+
else:
|
|
475
|
+
raise ValueError("expected plan or run")
|
|
476
|
+
except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as error:
|
|
477
|
+
print(f"generic-ci: {error}", file=sys.stderr)
|
|
478
|
+
raise SystemExit(1) from error
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
if __name__ == "__main__":
|
|
482
|
+
main()
|