generic-gitlab-cicd 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- generic_ci/__init__.py +3 -0
- generic_ci/__main__.py +3 -0
- generic_ci/cli.py +110 -0
- generic_ci/compiler.py +242 -0
- generic_ci/config.py +148 -0
- generic_ci/dependencies.py +186 -0
- generic_ci/models.py +189 -0
- generic_ci/runtime.py +482 -0
- generic_ci/sources.py +314 -0
- generic_ci/workflows/__init__.py +1 -0
- generic_ci/workflows/compiler.py +316 -0
- generic_ci/workflows/ecosystems.py +185 -0
- generic_ci/workflows/helm.py +168 -0
- generic_ci/workflows/models.py +222 -0
- generic_ci/workflows/publication.py +82 -0
- generic_ci/workflows/runtime.py +421 -0
- generic_gitlab_cicd-0.3.2.dist-info/METADATA +170 -0
- generic_gitlab_cicd-0.3.2.dist-info/RECORD +21 -0
- generic_gitlab_cicd-0.3.2.dist-info/WHEEL +5 -0
- generic_gitlab_cicd-0.3.2.dist-info/entry_points.txt +2 -0
- generic_gitlab_cicd-0.3.2.dist-info/top_level.txt +1 -0
generic_ci/__init__.py
ADDED
generic_ci/__main__.py
ADDED
generic_ci/cli.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Offline authoring CLI: schema, validate, explain, render and drift checking."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from pydantic import ValidationError
|
|
8
|
+
|
|
9
|
+
from .compiler import compile_pipeline, render, source_hashes
|
|
10
|
+
from .config import load
|
|
11
|
+
from .models import Pipeline, Platform
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main(argv=None):
|
|
15
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
16
|
+
if argv and argv[0] == 'source':
|
|
17
|
+
from .sources import source_main
|
|
18
|
+
try:
|
|
19
|
+
return source_main(argv[1:])
|
|
20
|
+
except (ValueError, OSError, ValidationError) as error:
|
|
21
|
+
print(f'generic-ci: {error}', file=sys.stderr)
|
|
22
|
+
return 1
|
|
23
|
+
parser = argparse.ArgumentParser(prog="generic-ci")
|
|
24
|
+
parser.add_argument("command", choices=["schema", "validate", "explain", "render", "init"])
|
|
25
|
+
parser.add_argument("--config")
|
|
26
|
+
parser.add_argument("--format", choices=["workflows", "legacy"], default="workflows")
|
|
27
|
+
parser.add_argument("--ecosystem", choices=["python", "npm", "pnpm", "bun"], default="python")
|
|
28
|
+
parser.add_argument("--platform")
|
|
29
|
+
parser.add_argument("--template")
|
|
30
|
+
parser.add_argument("--source")
|
|
31
|
+
parser.add_argument("--repo")
|
|
32
|
+
parser.add_argument("--ref")
|
|
33
|
+
parser.add_argument("--offline", action="store_true")
|
|
34
|
+
parser.add_argument("--root", default=".")
|
|
35
|
+
parser.add_argument("--output", "-o")
|
|
36
|
+
parser.add_argument("--check", action="store_true", help="Fail if output differs; never modify output")
|
|
37
|
+
parser.add_argument("--platform-schema", action="store_true")
|
|
38
|
+
options = parser.parse_args(argv)
|
|
39
|
+
try:
|
|
40
|
+
root = Path(options.root).resolve()
|
|
41
|
+
use_source = options.format == 'workflows' and (root / 'generic-ci.yml').exists()
|
|
42
|
+
from .sources import home
|
|
43
|
+
default_source = (home() / 'sources.json').exists()
|
|
44
|
+
if options.command == 'init' and (options.template or options.source or options.repo or default_source):
|
|
45
|
+
from .sources import initialize
|
|
46
|
+
initialize(root, options.template, options.source, options.repo, options.ref, options.offline, options.config)
|
|
47
|
+
return 0
|
|
48
|
+
config_path = root / (options.config or 'delivery.yml')
|
|
49
|
+
platform_path = root / (options.platform or 'ci-platform.yml')
|
|
50
|
+
if options.format == "workflows":
|
|
51
|
+
from .workflows import compiler as workflow_compiler
|
|
52
|
+
from .workflows.models import Pipeline as WorkflowPipeline, Platform as WorkflowPlatform
|
|
53
|
+
if options.command == "init":
|
|
54
|
+
import yaml
|
|
55
|
+
if config_path.exists():
|
|
56
|
+
raise ValueError("configuration exists; init will not overwrite it")
|
|
57
|
+
ecosystem = {"python": {}} if options.ecosystem == "python" else {"node": {"package-manager": options.ecosystem}}
|
|
58
|
+
command = "uv run --no-sync pytest" if options.ecosystem == "python" else options.ecosystem + " run test"
|
|
59
|
+
example = {"version": 1, "projects": {"app": {"path": ".", **ecosystem,
|
|
60
|
+
"checks": {"unit": {"script": [command]}},
|
|
61
|
+
"workflows": {"push": {"checks": ["unit"]}, "merge-request": {"checks": ["unit"]}}}}}
|
|
62
|
+
config_path.write_text(yaml.safe_dump(example, sort_keys=False))
|
|
63
|
+
print(f"Created {config_path}; provide an internal platform configuration with --platform")
|
|
64
|
+
return 0
|
|
65
|
+
if options.command == "schema":
|
|
66
|
+
model = (WorkflowPlatform if options.platform_schema else WorkflowPipeline) if options.format == "workflows" else (Platform if options.platform_schema else Pipeline)
|
|
67
|
+
result = json.dumps(model.model_json_schema(by_alias=True), indent=2) + "\n"
|
|
68
|
+
else:
|
|
69
|
+
if options.format == "workflows":
|
|
70
|
+
if use_source:
|
|
71
|
+
from .sources import load_project
|
|
72
|
+
pipeline, platform, origins, inputs = load_project(root, options.config, options.platform, options.offline)
|
|
73
|
+
else:
|
|
74
|
+
pipeline, platform = workflow_compiler.load(config_path, platform_path)
|
|
75
|
+
origins = {"checks": "developer-defined; no hidden suites"}
|
|
76
|
+
inputs = [config_path, platform_path]
|
|
77
|
+
else:
|
|
78
|
+
pipeline, platform, origins = load(config_path, platform_path)
|
|
79
|
+
inputs = [config_path, platform_path]
|
|
80
|
+
sources = source_hashes(root, inputs)
|
|
81
|
+
jobs, payload = (workflow_compiler.compile_pipeline(pipeline, platform, sources=sources) if options.format == "workflows" else compile_pipeline(pipeline, platform, sources=sources))
|
|
82
|
+
if options.command == "validate":
|
|
83
|
+
result = f"Valid: {len(pipeline.projects)} projects; {len(payload['nodes']) + 1} jobs. Target GitLab CI Lint is still required.\n"
|
|
84
|
+
elif options.command == "explain":
|
|
85
|
+
result = json.dumps({"generation": "committed top-level GitLab CI", "origins": origins,
|
|
86
|
+
"projects": payload["pipeline"]["projects"], "platform": payload["platform"], "jobs": payload["nodes"]}, indent=2) + "\n"
|
|
87
|
+
else:
|
|
88
|
+
if options.format == "workflows":
|
|
89
|
+
import yaml
|
|
90
|
+
result = "# Generated by generic-ci; edit delivery configuration and render again.\n" + yaml.safe_dump(jobs, sort_keys=False)
|
|
91
|
+
else:
|
|
92
|
+
result = render(pipeline, platform, sources=sources)
|
|
93
|
+
if options.check:
|
|
94
|
+
if not options.output or not Path(options.output).is_file() or Path(options.output).read_text() != result:
|
|
95
|
+
raise ValueError("generated output differs; render again and commit it")
|
|
96
|
+
return 0
|
|
97
|
+
if options.output:
|
|
98
|
+
target = Path(options.output)
|
|
99
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
100
|
+
target.write_text(result)
|
|
101
|
+
else:
|
|
102
|
+
print(result, end="")
|
|
103
|
+
return 0
|
|
104
|
+
except (ValueError, OSError, ValidationError) as error:
|
|
105
|
+
print(f"generic-ci: {error}", file=sys.stderr)
|
|
106
|
+
return 1
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
if __name__ == "__main__":
|
|
110
|
+
raise SystemExit(main())
|
generic_ci/compiler.py
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Deterministic compilation to a committed, top-level GitLab pipeline."""
|
|
2
|
+
import base64
|
|
3
|
+
import hashlib
|
|
4
|
+
import itertools
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import yaml
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def digest(value):
|
|
15
|
+
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def variants(native):
|
|
19
|
+
matrix = native.get("parallel")
|
|
20
|
+
if not matrix:
|
|
21
|
+
return [{}]
|
|
22
|
+
if set(matrix) != {"matrix"} or not isinstance(matrix["matrix"], list) or not matrix["matrix"]:
|
|
23
|
+
raise ValueError("gitlab.parallel must contain a nonempty native matrix list")
|
|
24
|
+
result = []
|
|
25
|
+
for row in matrix["matrix"]:
|
|
26
|
+
if not isinstance(row, dict) or not row:
|
|
27
|
+
raise ValueError("matrix rows must be nonempty mappings")
|
|
28
|
+
keys = list(row)
|
|
29
|
+
if any(not re.fullmatch(r"[A-Z][A-Z0-9_]*", k) or k.startswith(("CI_", "TOOLKIT_")) for k in keys):
|
|
30
|
+
raise ValueError("matrix keys must be uppercase and cannot use CI_ or TOOLKIT_ prefixes")
|
|
31
|
+
choices = [v if isinstance(v, list) else [v] for v in row.values()]
|
|
32
|
+
if any(not c or any(not isinstance(v, (str, int)) or isinstance(v, bool) for v in c) for c in choices):
|
|
33
|
+
raise ValueError("matrix values must be strings or integers (quote Python versions)")
|
|
34
|
+
for values in itertools.product(*choices):
|
|
35
|
+
result.append(dict(zip(keys, map(str, values))))
|
|
36
|
+
if len(result) > 200:
|
|
37
|
+
raise ValueError("matrix exceeds 200 combinations")
|
|
38
|
+
if len({json.dumps(v, sort_keys=True) for v in result}) != len(result):
|
|
39
|
+
raise ValueError("duplicate matrix combinations")
|
|
40
|
+
return result
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def compile_pipeline(pipeline, platform, *, sources=None):
|
|
44
|
+
data = pipeline.model_dump()
|
|
45
|
+
infra = platform.model_dump()
|
|
46
|
+
nodes = {}
|
|
47
|
+
refs = {}
|
|
48
|
+
|
|
49
|
+
def add(identifier, project, action, settings, *, native=None, variables=None):
|
|
50
|
+
if identifier in nodes:
|
|
51
|
+
raise ValueError(f"duplicate generated job {identifier}")
|
|
52
|
+
nodes[identifier] = {"project": project, "action": action, "settings": settings,
|
|
53
|
+
"native": native or {}, "variables": variables or {}, "needs": []}
|
|
54
|
+
return identifier
|
|
55
|
+
|
|
56
|
+
def check_refs(project, names):
|
|
57
|
+
p = data["projects"][project]
|
|
58
|
+
selected = names if names is not None else [k for k, v in p["checks"].items() if v]
|
|
59
|
+
for required in infra["mandatory_checks"]:
|
|
60
|
+
if required not in selected:
|
|
61
|
+
raise ValueError(f"{project}: missing mandatory check {required}")
|
|
62
|
+
return [f"{project}.{name}" for name in selected]
|
|
63
|
+
|
|
64
|
+
for name, project in data["projects"].items():
|
|
65
|
+
for category in ("steps", "checks"):
|
|
66
|
+
for key, settings in project[category].items():
|
|
67
|
+
if settings is False:
|
|
68
|
+
continue
|
|
69
|
+
public = f"{name}.{key}"
|
|
70
|
+
if public in refs or key in {"container", "package", "release"}:
|
|
71
|
+
raise ValueError(f"{public}: check/step name collides with a public output")
|
|
72
|
+
rows = variants(settings["gitlab"])
|
|
73
|
+
if category == "steps" and settings["outputs"] and len(rows) > 1:
|
|
74
|
+
raise ValueError(f"{public}: output-producing steps must have one variant; use separate named steps")
|
|
75
|
+
refs[public] = []
|
|
76
|
+
for index, variables in enumerate(rows):
|
|
77
|
+
identifier = f"{name}-{key}" + (f"-{index + 1}" if len(rows) > 1 else "")
|
|
78
|
+
refs[public].append(add(identifier, name, "step" if category == "steps" else "check", settings,
|
|
79
|
+
native=settings["gitlab"], variables=variables))
|
|
80
|
+
if project["package"] is not None:
|
|
81
|
+
refs[f"{name}.package"] = [add(f"{name}-package", name, "package", project["package"])]
|
|
82
|
+
if project["container"] is not None:
|
|
83
|
+
refs[f"{name}.container"] = [add(f"{name}-container", name, "container", project["container"])]
|
|
84
|
+
for key, deployment in project["deploy"].items():
|
|
85
|
+
add(f"{name}-deploy-{key}", name, "deploy", {**deployment, "name": key})
|
|
86
|
+
if deployment["preview"]:
|
|
87
|
+
add(f"{name}-stop-{key}", name, "stop", {**deployment, "name": key})
|
|
88
|
+
if project["release"]:
|
|
89
|
+
refs[f"{name}.release"] = [add(f"{name}-release", name, "release", project["release"])]
|
|
90
|
+
|
|
91
|
+
for identifier, node in nodes.items():
|
|
92
|
+
name = node["project"]
|
|
93
|
+
project = data["projects"][name]
|
|
94
|
+
settings = node["settings"]
|
|
95
|
+
requested = []
|
|
96
|
+
if node["action"] in {"step", "check", "container", "package"}:
|
|
97
|
+
requested += [r if "." in r else f"{name}.{r}" for r in settings.get("needs", [])]
|
|
98
|
+
if node["action"] in {"container", "package"}:
|
|
99
|
+
requested += check_refs(name, settings.get("checks"))
|
|
100
|
+
if node["action"] == "deploy":
|
|
101
|
+
requested += check_refs(name, settings["checks"])
|
|
102
|
+
images = settings["images"] or {name: {"repository": ["apps", name, "image", "repository"],
|
|
103
|
+
"digest": ["apps", name, "image", "digest"]}}
|
|
104
|
+
settings["images"] = images
|
|
105
|
+
requested += [f"{p}.container" for p in images]
|
|
106
|
+
if node["action"] == "release":
|
|
107
|
+
requested += [f"{p}.release" for p in settings["needs"]]
|
|
108
|
+
requested += [f"{name}.{kind}" for kind in ("package", "container") if project[kind] is not None]
|
|
109
|
+
requested += check_refs(name, None)
|
|
110
|
+
for ref in dict.fromkeys(requested):
|
|
111
|
+
if ref not in refs:
|
|
112
|
+
raise ValueError(f"{identifier}.needs: unknown/disabled reference {ref}")
|
|
113
|
+
for upstream in refs[ref]:
|
|
114
|
+
if nodes[upstream]["native"].get("allow_failure"):
|
|
115
|
+
raise ValueError(f"{identifier}: required check {ref} cannot allow failure")
|
|
116
|
+
node["needs"].append(upstream)
|
|
117
|
+
node["needs"] = sorted(set(node["needs"]))
|
|
118
|
+
if len(node["needs"]) >= 50:
|
|
119
|
+
raise ValueError(f"{identifier}: too many dependencies for supported GitLab needs limit")
|
|
120
|
+
|
|
121
|
+
visiting, visited = set(), set()
|
|
122
|
+
|
|
123
|
+
def visit(key):
|
|
124
|
+
if key in visiting:
|
|
125
|
+
raise ValueError(f"dependency cycle involving {key}")
|
|
126
|
+
if key in visited:
|
|
127
|
+
return
|
|
128
|
+
visiting.add(key)
|
|
129
|
+
for upstream in nodes[key]["needs"]:
|
|
130
|
+
visit(upstream)
|
|
131
|
+
visiting.remove(key)
|
|
132
|
+
visited.add(key)
|
|
133
|
+
|
|
134
|
+
for key in nodes:
|
|
135
|
+
visit(key)
|
|
136
|
+
# Selection follows actual cross-project graph edges as well as declared source dependencies.
|
|
137
|
+
for node in nodes.values():
|
|
138
|
+
if node["action"] == "release":
|
|
139
|
+
continue
|
|
140
|
+
project = data["projects"][node["project"]]
|
|
141
|
+
project["depends_on"] = sorted(set(project["depends_on"]) | {
|
|
142
|
+
nodes[k]["project"] for k in node["needs"] if nodes[k]["project"] != node["project"]})
|
|
143
|
+
if len(nodes) + 1 > platform.max_jobs:
|
|
144
|
+
raise ValueError(f"pipeline creates {len(nodes) + 1} jobs; platform limit is {platform.max_jobs}")
|
|
145
|
+
payload = {"version": __version__, "pipeline": data, "platform": infra, "nodes": nodes, "sources": sources or {}}
|
|
146
|
+
fingerprint = digest(payload)
|
|
147
|
+
encoded = base64.b64encode(json.dumps(payload, sort_keys=True).encode()).decode()
|
|
148
|
+
common_rules = [{"if": '$CI_PIPELINE_SOURCE == "merge_request_event"'}, {"if": "$CI_COMMIT_TAG"}, {"if": "$CI_COMMIT_BRANCH"}]
|
|
149
|
+
preview_rules = [{"if": '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == $CI_PROJECT_ID'}]
|
|
150
|
+
protected_rules = [{"if": '$CI_COMMIT_REF_PROTECTED == "true" && $CI_COMMIT_TAG'}]
|
|
151
|
+
default = infra["runtimes"]["default"]
|
|
152
|
+
output = {
|
|
153
|
+
"stages": ["prepare", "delivery"],
|
|
154
|
+
"workflow": {"auto_cancel": {"on_new_commit": "conservative"}, "rules": [
|
|
155
|
+
{"if": '$CI_PIPELINE_SOURCE == "merge_request_event"'}, {"if": "$CI_COMMIT_TAG"},
|
|
156
|
+
{"if": '$CI_PIPELINE_SOURCE == "push" && $CI_OPEN_MERGE_REQUESTS', "when": "never"},
|
|
157
|
+
{"if": "$CI_COMMIT_BRANCH"}, {"when": "never"}]},
|
|
158
|
+
"variables": {
|
|
159
|
+
"TOOLKIT_CONFIG_B64": encoded,
|
|
160
|
+
"CI_DEPENDENCY_OVERRIDES": {"value": "[]", "description": "Temporary JSON array of package/repository/ref/subdirectory overrides"},
|
|
161
|
+
"CI_DEPENDENCY_FILE": {"value": "", "description": "Optional override JSON file; exclusive with other override inputs"},
|
|
162
|
+
"CI_DEPENDENCY_REPO": {"value": "", "description": "Single candidate repository HTTPS URL"},
|
|
163
|
+
"CI_DEPENDENCY_REF": {"value": "", "description": "Single candidate branch, tag or SHA"},
|
|
164
|
+
"CI_DEPENDENCY_PACKAGE": {"value": "", "description": "Single candidate distribution name"},
|
|
165
|
+
"CI_DEPENDENCY_SUBDIRECTORY": {"value": "", "description": "Package path within candidate repository"},
|
|
166
|
+
"CI_FULL_PIPELINE": {"value": "false", "options": ["false", "true"], "description": "Run all projects regardless of changed paths"},
|
|
167
|
+
},
|
|
168
|
+
"toolkit-plan": {"stage": "prepare", "tags": default["tags"], "script": [f"python -m generic_ci.runtime plan {fingerprint}"],
|
|
169
|
+
"rules": common_rules, "artifacts": {"paths": [".ci-out/plan.json"], "expire_in": infra["artifact_retention"]},
|
|
170
|
+
"interruptible": True},
|
|
171
|
+
}
|
|
172
|
+
if default["image"]:
|
|
173
|
+
output["toolkit-plan"]["image"] = default["image"]
|
|
174
|
+
for identifier, node in nodes.items():
|
|
175
|
+
action = node["action"]
|
|
176
|
+
runtime_key = {"container": "build", "deploy": "helm", "stop": "helm", "release": "release"}.get(action, "default")
|
|
177
|
+
if action in {"check", "step"}:
|
|
178
|
+
runtime_key = node["settings"]["runtime"]
|
|
179
|
+
if runtime_key not in infra["runtimes"]:
|
|
180
|
+
raise ValueError(f"{identifier}.runtime: unknown runtime {runtime_key}")
|
|
181
|
+
runtime = infra["runtimes"].get(runtime_key, default)
|
|
182
|
+
native = node["native"]
|
|
183
|
+
node["shell"] = runtime["shell"] # Set before final fingerprint below.
|
|
184
|
+
job = {"stage": "delivery", "tags": runtime["tags"], "interruptible": action in {"check", "step", "package"},
|
|
185
|
+
"script": [f"python -m generic_ci.runtime run {identifier} FINGERPRINT"],
|
|
186
|
+
"needs": [{"job": "toolkit-plan", "artifacts": True}] + [{"job": k, "artifacts": True, "optional": True} for k in node["needs"]],
|
|
187
|
+
"rules": common_rules, "allow_failure": native.get("allow_failure", False), "retry": 0,
|
|
188
|
+
"variables": {**native.get("variables", {}), **node["variables"]},
|
|
189
|
+
"artifacts": {"when": "always", "expire_in": infra["artifact_retention"], "paths": [f".ci-out/{identifier}/"]}}
|
|
190
|
+
if runtime["image"]:
|
|
191
|
+
job["image"] = runtime["image"]
|
|
192
|
+
for key in ("image", "tags", "services", "timeout", "rules", "cache", "resource_group"):
|
|
193
|
+
if native.get(key) is not None:
|
|
194
|
+
job[key] = native[key]
|
|
195
|
+
if node["settings"].get("junit"):
|
|
196
|
+
job["artifacts"]["reports"] = {"junit": [f".ci-out/{identifier}/reports/*.xml"]}
|
|
197
|
+
if action in {"deploy", "stop"}:
|
|
198
|
+
settings = node["settings"]
|
|
199
|
+
target = infra["targets"][settings["target"]]
|
|
200
|
+
preview = settings["preview"]
|
|
201
|
+
env = f"review/{node['project']}/{settings['name']}/$CI_MERGE_REQUEST_IID" if preview else f"{node['project']}/{settings['target']}"
|
|
202
|
+
job["environment"] = {"name": env, "url": target["url"], "deployment_tier": "development" if preview else ("production" if target["production"] else "staging")}
|
|
203
|
+
job["resource_group"] = env
|
|
204
|
+
job["rules"] = preview_rules if preview else (protected_rules if target["production"] else common_rules)
|
|
205
|
+
job["when"] = "manual" if action == "stop" else settings["when"]
|
|
206
|
+
if preview:
|
|
207
|
+
job["environment"].update({"on_stop": identifier.replace("-deploy-", "-stop-"), "auto_stop_in": settings["auto_stop_in"]})
|
|
208
|
+
if action == "stop":
|
|
209
|
+
job["environment"].pop("on_stop", None)
|
|
210
|
+
job["environment"].pop("auto_stop_in", None)
|
|
211
|
+
job["environment"]["action"] = "stop"
|
|
212
|
+
job["variables"]["GIT_STRATEGY"] = "none"
|
|
213
|
+
job["needs"] = []
|
|
214
|
+
job["allow_failure"] = True
|
|
215
|
+
if action == "release":
|
|
216
|
+
tag = node["settings"]["tag"]
|
|
217
|
+
pattern = re.escape(tag).replace(re.escape("{version}"), ".+").replace("/", "\\/")
|
|
218
|
+
job["rules"] = [{"if": f'$CI_COMMIT_REF_PROTECTED == "true" && $CI_COMMIT_TAG =~ /^{pattern}$/'}]
|
|
219
|
+
job["when"] = "manual"
|
|
220
|
+
job["resource_group"] = f"release/{node['project']}"
|
|
221
|
+
job["environment"] = {"name": f"publish/{node['project']}", "deployment_tier": "other"}
|
|
222
|
+
output[identifier] = job
|
|
223
|
+
# Shell metadata is also signed by the expected configuration fingerprint.
|
|
224
|
+
fingerprint = digest(payload)
|
|
225
|
+
encoded = base64.b64encode(json.dumps(payload, sort_keys=True).encode()).decode()
|
|
226
|
+
if len(encoded) > 100_000:
|
|
227
|
+
raise ValueError("compiled execution configuration exceeds 100 KB; split the pipeline into smaller project groups")
|
|
228
|
+
output["variables"]["TOOLKIT_CONFIG_B64"] = encoded
|
|
229
|
+
output["toolkit-plan"]["script"] = [f"python -m generic_ci.runtime plan {fingerprint}"]
|
|
230
|
+
for identifier in nodes:
|
|
231
|
+
output[identifier]["script"] = [f"python -m generic_ci.runtime run {identifier} {fingerprint}"]
|
|
232
|
+
return output, payload
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def render(pipeline, platform, *, sources=None):
|
|
236
|
+
jobs, _ = compile_pipeline(pipeline, platform, sources=sources)
|
|
237
|
+
return "# Generated by generic-ci; edit the project configuration and render again.\n" + yaml.safe_dump(jobs, sort_keys=False, width=120)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def source_hashes(root, paths):
|
|
241
|
+
root = Path(root).resolve()
|
|
242
|
+
return {Path(path).resolve().relative_to(root).as_posix(): hashlib.sha256(Path(path).read_bytes()).hexdigest() for path in paths}
|
generic_ci/config.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Safe YAML loading, explicit preset expansion, and path/relationship validation."""
|
|
2
|
+
import copy
|
|
3
|
+
import fnmatch
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from urllib.parse import urlsplit
|
|
6
|
+
|
|
7
|
+
import yaml
|
|
8
|
+
|
|
9
|
+
from .models import Pipeline, Platform, relative
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class UniqueLoader(yaml.SafeLoader):
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def mapping(loader, node, deep=False):
|
|
17
|
+
result = {}
|
|
18
|
+
for key_node, value_node in node.value:
|
|
19
|
+
key = loader.construct_object(key_node, deep=deep)
|
|
20
|
+
if not isinstance(key, str):
|
|
21
|
+
raise ValueError(f"YAML mapping keys must be strings at line {key_node.start_mark.line + 1}")
|
|
22
|
+
if key in result:
|
|
23
|
+
raise ValueError(f"duplicate YAML key {key!r} at line {key_node.start_mark.line + 1}")
|
|
24
|
+
result[key] = loader.construct_object(value_node, deep=deep)
|
|
25
|
+
return result
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
UniqueLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, mapping)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def read_yaml(path):
|
|
32
|
+
text = Path(path).read_text()
|
|
33
|
+
if len(text) > 2_000_000:
|
|
34
|
+
raise ValueError("configuration exceeds 2 MB")
|
|
35
|
+
return yaml.load(text, Loader=UniqueLoader)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def allowed_url(value, hosts, *, registry=False):
|
|
39
|
+
parsed = urlsplit("https://" + value if registry else value.replace("oci://", "https://", 1))
|
|
40
|
+
if parsed.scheme != "https" or parsed.hostname not in hosts or parsed.username or parsed.password or parsed.query:
|
|
41
|
+
raise ValueError(f"endpoint must use an allowed credential-free HTTPS host (host: {parsed.hostname})")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def load(config_path, platform_path):
|
|
45
|
+
raw = read_yaml(config_path)
|
|
46
|
+
# Validate before merging so unknown keys cannot disappear through expansion.
|
|
47
|
+
declared = Pipeline.model_validate(raw)
|
|
48
|
+
effective = copy.deepcopy(raw)
|
|
49
|
+
origins = {}
|
|
50
|
+
for name, project in declared.projects.items():
|
|
51
|
+
row = effective["projects"][name]
|
|
52
|
+
if project.preset.startswith("python-"):
|
|
53
|
+
defaults = {"lint": {"script": ["uv run --no-sync ruff check ."]},
|
|
54
|
+
"unit": {"script": ["uv run --no-sync pytest --junitxml=reports/junit.xml"],
|
|
55
|
+
"junit": ["reports/junit.xml"]}}
|
|
56
|
+
explicit = row.get("checks", {})
|
|
57
|
+
merged = {}
|
|
58
|
+
for check, settings in defaults.items():
|
|
59
|
+
override = explicit.get(check, {})
|
|
60
|
+
merged[check] = False if override is False else {**settings, **override}
|
|
61
|
+
origins[f"projects.{name}.checks.{check}"] = "project" if check in explicit else project.preset
|
|
62
|
+
row["checks"] = {**merged, **{k: v for k, v in explicit.items() if k not in defaults}}
|
|
63
|
+
row.setdefault("dependencies", {"manager": "uv"})
|
|
64
|
+
if project.preset == "python-package":
|
|
65
|
+
row.setdefault("package", {})
|
|
66
|
+
else:
|
|
67
|
+
row.setdefault("container", {})
|
|
68
|
+
else:
|
|
69
|
+
row.setdefault("dependencies", {"manager": "none"})
|
|
70
|
+
pipeline = Pipeline.model_validate(effective)
|
|
71
|
+
platform = Platform.model_validate(read_yaml(platform_path))
|
|
72
|
+
for runtime in platform.runtimes.values():
|
|
73
|
+
if runtime.image:
|
|
74
|
+
allowed_url(runtime.image, platform.allowed_hosts, registry=True)
|
|
75
|
+
for registry in (platform.registry, platform.preview_registry):
|
|
76
|
+
allowed_url(registry, platform.allowed_hosts, registry=True)
|
|
77
|
+
if platform.registry.rstrip("/") == platform.preview_registry.rstrip("/"):
|
|
78
|
+
raise ValueError("preview-registry must be separate from registry")
|
|
79
|
+
if platform.chart.startswith("oci://"):
|
|
80
|
+
allowed_url(platform.chart, platform.allowed_hosts)
|
|
81
|
+
else:
|
|
82
|
+
relative(platform.chart)
|
|
83
|
+
for name, project in pipeline.projects.items():
|
|
84
|
+
for check_name, check in project.checks.items():
|
|
85
|
+
if check and not check.script:
|
|
86
|
+
raise ValueError(f"projects.{name}.checks.{check_name}.script: required for a custom check")
|
|
87
|
+
for check in [*project.checks.values(), *project.steps.values()]:
|
|
88
|
+
if check and check.gitlab.image:
|
|
89
|
+
allowed_url(check.gitlab.image, platform.allowed_hosts, registry=True)
|
|
90
|
+
for dependency in project.depends_on:
|
|
91
|
+
if dependency not in pipeline.projects:
|
|
92
|
+
raise ValueError(f"projects.{name}.depends-on: unknown project {dependency}")
|
|
93
|
+
for mandatory in platform.mandatory_checks:
|
|
94
|
+
if not project.checks.get(mandatory):
|
|
95
|
+
raise ValueError(f"projects.{name}.checks.{mandatory}: platform-required check is missing/disabled")
|
|
96
|
+
if project.container and project.container.repository:
|
|
97
|
+
allowed_url(project.container.repository, platform.allowed_hosts, registry=True)
|
|
98
|
+
for dep_name, deployment in project.deploy.items():
|
|
99
|
+
prefix = f"projects.{name}.deploy.{dep_name}"
|
|
100
|
+
if deployment.target not in platform.targets:
|
|
101
|
+
raise ValueError(f"{prefix}.target: unknown target {deployment.target}")
|
|
102
|
+
if deployment.preview and platform.targets[deployment.target].production:
|
|
103
|
+
raise ValueError(f"{prefix}: previews require a nonproduction target")
|
|
104
|
+
if deployment.chart:
|
|
105
|
+
if deployment.chart.startswith("oci://"):
|
|
106
|
+
allowed_url(deployment.chart, platform.allowed_hosts)
|
|
107
|
+
if not deployment.chart_version:
|
|
108
|
+
raise ValueError(f"{prefix}: custom OCI chart requires chart-version")
|
|
109
|
+
else:
|
|
110
|
+
relative(deployment.chart)
|
|
111
|
+
if not deployment.images:
|
|
112
|
+
raise ValueError(f"{prefix}.images: custom chart requires explicit image bindings")
|
|
113
|
+
checks = deployment.checks if deployment.checks is not None else [k for k, v in project.checks.items() if v]
|
|
114
|
+
for check in checks:
|
|
115
|
+
if not project.checks.get(check):
|
|
116
|
+
raise ValueError(f"{prefix}.checks: unknown or disabled check {check}")
|
|
117
|
+
if not set(platform.mandatory_checks) <= set(checks):
|
|
118
|
+
raise ValueError(f"{prefix}.checks: cannot remove platform-required checks")
|
|
119
|
+
for source in deployment.images:
|
|
120
|
+
if source not in pipeline.projects or not pipeline.projects[source].container:
|
|
121
|
+
raise ValueError(f"{prefix}.images: {source} is not a container project")
|
|
122
|
+
if project.release:
|
|
123
|
+
for dep in project.release.needs:
|
|
124
|
+
if dep not in pipeline.projects or not pipeline.projects[dep].release:
|
|
125
|
+
raise ValueError(f"projects.{name}.release.needs: unknown release {dep}")
|
|
126
|
+
if pipeline.projects[dep].release.tag != project.release.tag:
|
|
127
|
+
raise ValueError(f"projects.{name}.release.needs: same-pipeline release dependencies require a common tag convention")
|
|
128
|
+
owners = [p.release.tag for p in pipeline.projects.values() if p.release and p.release.gitlab_release]
|
|
129
|
+
if len(owners) != len(set(owners)):
|
|
130
|
+
raise ValueError("a shared release tag must have exactly one gitlab-release owner")
|
|
131
|
+
return pipeline, platform, origins
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def affected(pipeline, changed):
|
|
135
|
+
"""None means unavailable baseline: full run. Empty means no source change."""
|
|
136
|
+
if changed is None:
|
|
137
|
+
return set(pipeline["projects"])
|
|
138
|
+
selected = set()
|
|
139
|
+
for name, project in pipeline["projects"].items():
|
|
140
|
+
root = project["path"].rstrip("/")
|
|
141
|
+
for path in changed:
|
|
142
|
+
if root == "." or path == root or path.startswith(root + "/") or any(fnmatch.fnmatchcase(path, pat) for pat in project["watch"]):
|
|
143
|
+
selected.add(name)
|
|
144
|
+
while True:
|
|
145
|
+
expanded = selected | {name for name, p in pipeline["projects"].items() if set(p["depends_on"]) & selected}
|
|
146
|
+
if expanded == selected:
|
|
147
|
+
return selected
|
|
148
|
+
selected = expanded
|