gitlab-compliance 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gitlab_compliance-2.0.0.dist-info/METADATA +92 -0
- gitlab_compliance-2.0.0.dist-info/RECORD +50 -0
- gitlab_compliance-2.0.0.dist-info/WHEEL +4 -0
- gitlab_compliance-2.0.0.dist-info/entry_points.txt +4 -0
- gitlab_compliance-2.0.0.dist-info/licenses/LICENSE +21 -0
- src/__init__.py +1 -0
- src/compliance/__init__.py +1 -0
- src/compliance/api_config.py +94 -0
- src/compliance/api_enrichment.py +52 -0
- src/compliance/behave_support/environment.py +43 -0
- src/compliance/behave_support/steps/__init__.py +0 -0
- src/compliance/behave_support/steps/given_steps.py +154 -0
- src/compliance/behave_support/steps/then_steps.py +234 -0
- src/compliance/behave_support/steps/when_steps.py +168 -0
- src/compliance/console.py +95 -0
- src/compliance/container_fix.py +135 -0
- src/compliance/gitlab_api.py +94 -0
- src/compliance/image_versions.py +316 -0
- src/compliance/include_fix.py +112 -0
- src/compliance/include_versions.py +250 -0
- src/compliance/metadata.py +245 -0
- src/compliance/model.py +277 -0
- src/compliance/models.py +29 -0
- src/compliance/oci_registry.py +147 -0
- src/compliance/policy_doc.py +177 -0
- src/compliance/release_cache.py +51 -0
- src/compliance/render.py +360 -0
- src/compliance/runner.py +310 -0
- src/compliance/stash.py +203 -0
- src/compliance/supply_chain_fix.py +69 -0
- src/gitlab_compliance.py +636 -0
- src/gitlab_docs.py +40 -0
- src/modules/__init__.py +1 -0
- src/modules/command_reference.py +220 -0
- src/modules/common.py +144 -0
- src/modules/constants.py +22 -0
- src/modules/doc_controller.py +178 -0
- src/modules/logging.py +36 -0
- src/modules/pipeline_data.py +364 -0
- src/modules/release.py +343 -0
- src/modules/swagger_html.py +408 -0
- src/modules/yaml_lines.py +70 -0
- src/modules/yaml_md_table.py +29 -0
- src/properties/__init__.py +0 -0
- src/properties/extract_job_attribute.py +76 -0
- src/properties/includes.py +178 -0
- src/properties/inputs.py +86 -0
- src/properties/jobs.py +167 -0
- src/properties/variables.py +91 -0
- src/properties/workflows.py +54 -0
src/compliance/model.py
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"""Load GitLab CI YAML into compliance entity stash."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from src.modules.pipeline_data import collect_pipeline_data
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _job_entity(name: str, config: dict, source_file: str = "", line: int = 0) -> dict:
|
|
12
|
+
return {
|
|
13
|
+
"address": f"job.{name}",
|
|
14
|
+
"type": "job",
|
|
15
|
+
"name": name,
|
|
16
|
+
"values": dict(config),
|
|
17
|
+
"source_file": source_file,
|
|
18
|
+
"line": line,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _include_entity(index: int, include: dict) -> dict:
|
|
23
|
+
version = include.get("version", "")
|
|
24
|
+
return {
|
|
25
|
+
"address": f"include.{index}",
|
|
26
|
+
"type": "include",
|
|
27
|
+
"include_type": include["include_type"],
|
|
28
|
+
"name": include.get("project", ""),
|
|
29
|
+
"project": include.get("project", ""),
|
|
30
|
+
"version": version,
|
|
31
|
+
"valid_version": include.get("valid_version", False),
|
|
32
|
+
"latest_version": include.get("latest_version", ""),
|
|
33
|
+
"update_available": include.get("update_available", False),
|
|
34
|
+
"release_metadata_resolved": include.get("release_metadata_resolved", False),
|
|
35
|
+
"version_released_at": include.get("version_released_at", ""),
|
|
36
|
+
"latest_version_released_at": include.get("latest_version_released_at", ""),
|
|
37
|
+
"latest_release_age_days": include.get("latest_release_age_days"),
|
|
38
|
+
"release_lag_days": include.get("release_lag_days"),
|
|
39
|
+
"version_tag_rank": include.get("version_tag_rank"),
|
|
40
|
+
"semver_tag_count": include.get("semver_tag_count", 0),
|
|
41
|
+
"file": include.get("file", ""),
|
|
42
|
+
"values": include,
|
|
43
|
+
"source_file": include.get("source_file", ""),
|
|
44
|
+
"line": include.get("line", 0),
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _container_image_entity(index: int, image: dict) -> dict:
|
|
49
|
+
image_ref = image.get("image", image.get("project", ""))
|
|
50
|
+
return {
|
|
51
|
+
"address": f"container_image.{index}",
|
|
52
|
+
"type": "container_image",
|
|
53
|
+
"include_type": "container_image",
|
|
54
|
+
"image_source": image.get("image_source", ""),
|
|
55
|
+
"parent_job": image.get("parent_job", ""),
|
|
56
|
+
"name": image_ref,
|
|
57
|
+
"project": image_ref,
|
|
58
|
+
"image": image_ref,
|
|
59
|
+
"version": image.get("version", ""),
|
|
60
|
+
"valid_version": image.get("valid_version", False),
|
|
61
|
+
"registry": image.get("registry", ""),
|
|
62
|
+
"repository": image.get("repository", ""),
|
|
63
|
+
"digest": image.get("digest", ""),
|
|
64
|
+
"latest_version": image.get("latest_version", ""),
|
|
65
|
+
"latest_digest": image.get("latest_digest", ""),
|
|
66
|
+
"update_available": image.get("update_available", False),
|
|
67
|
+
"release_metadata_resolved": image.get("release_metadata_resolved", False),
|
|
68
|
+
"latest_release_age_days": image.get("latest_release_age_days"),
|
|
69
|
+
"release_lag_days": image.get("release_lag_days"),
|
|
70
|
+
"version_tag_rank": image.get("version_tag_rank"),
|
|
71
|
+
"semver_tag_count": image.get("semver_tag_count", 0),
|
|
72
|
+
"values": image,
|
|
73
|
+
"source_file": image.get("source_file", ""),
|
|
74
|
+
"line": image.get("line", 0),
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _variable_entity(
|
|
79
|
+
key: str, value: Any, source_file: str = "", line: int = 0
|
|
80
|
+
) -> dict:
|
|
81
|
+
return {
|
|
82
|
+
"address": f"variable.{key}",
|
|
83
|
+
"type": "variable",
|
|
84
|
+
"name": key,
|
|
85
|
+
"key": key,
|
|
86
|
+
"value": value,
|
|
87
|
+
"values": {"value": value},
|
|
88
|
+
"source_file": source_file,
|
|
89
|
+
"line": line,
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _workflow_rule_entity(
|
|
94
|
+
index: int, rule: Any, source_file: str = "", line: int = 0
|
|
95
|
+
) -> dict:
|
|
96
|
+
return {
|
|
97
|
+
"address": f"workflow.rule.{index}",
|
|
98
|
+
"type": "workflow_rule",
|
|
99
|
+
"name": f"rule-{index}",
|
|
100
|
+
"values": rule if isinstance(rule, dict) else {"rule": rule},
|
|
101
|
+
"source_file": source_file,
|
|
102
|
+
"line": line,
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def load_yaml_entities(
|
|
107
|
+
pipeline_file: str,
|
|
108
|
+
include_nested: bool = True,
|
|
109
|
+
) -> dict[str, list[dict]]:
|
|
110
|
+
if not os.path.exists(pipeline_file):
|
|
111
|
+
raise FileNotFoundError(f"Pipeline file not found: {pipeline_file}")
|
|
112
|
+
|
|
113
|
+
pipeline_data = collect_pipeline_data(
|
|
114
|
+
config_file=pipeline_file,
|
|
115
|
+
detailed=True,
|
|
116
|
+
include_nested=include_nested,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
line_index = pipeline_data.get("line_index") or {}
|
|
120
|
+
workflow_rule_lines = line_index.get("workflow_rules", [])
|
|
121
|
+
|
|
122
|
+
entities: dict[str, list[dict]] = {
|
|
123
|
+
"jobs": [
|
|
124
|
+
_job_entity(
|
|
125
|
+
job["name"],
|
|
126
|
+
_job_values(job),
|
|
127
|
+
job.get("source_file", pipeline_file),
|
|
128
|
+
job.get("line", 0),
|
|
129
|
+
)
|
|
130
|
+
for job in pipeline_data["jobs"]
|
|
131
|
+
],
|
|
132
|
+
"includes": [
|
|
133
|
+
_include_entity(index, include)
|
|
134
|
+
for index, include in enumerate(pipeline_data["includes"])
|
|
135
|
+
],
|
|
136
|
+
"variables": [
|
|
137
|
+
_variable_entity(
|
|
138
|
+
item["key"],
|
|
139
|
+
item["value"],
|
|
140
|
+
item.get("source_file", pipeline_file),
|
|
141
|
+
item.get("line", 0),
|
|
142
|
+
)
|
|
143
|
+
for item in pipeline_data["variables"]
|
|
144
|
+
],
|
|
145
|
+
"workflow_rules": [
|
|
146
|
+
_workflow_rule_entity(
|
|
147
|
+
index,
|
|
148
|
+
rule,
|
|
149
|
+
pipeline_file,
|
|
150
|
+
(
|
|
151
|
+
workflow_rule_lines[index - 1]
|
|
152
|
+
if index - 1 < len(workflow_rule_lines)
|
|
153
|
+
else 0
|
|
154
|
+
),
|
|
155
|
+
)
|
|
156
|
+
for index, rule in enumerate(pipeline_data["workflow_rules"], 1)
|
|
157
|
+
],
|
|
158
|
+
"project_settings": [],
|
|
159
|
+
"project_ci_variables": [],
|
|
160
|
+
"group_settings": [],
|
|
161
|
+
"container_images": [
|
|
162
|
+
_container_image_entity(index, image)
|
|
163
|
+
for index, image in enumerate(pipeline_data["container_images"], 1)
|
|
164
|
+
],
|
|
165
|
+
}
|
|
166
|
+
return entities
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _job_values(job: dict) -> dict:
|
|
170
|
+
values = {}
|
|
171
|
+
for attribute in job.get("attributes", []):
|
|
172
|
+
values[attribute["key"]] = attribute["value"]
|
|
173
|
+
if job.get("rules"):
|
|
174
|
+
values["rules"] = job["rules"]
|
|
175
|
+
for nested in job.get("nested", []):
|
|
176
|
+
if nested["attribute"] == "variables":
|
|
177
|
+
values.setdefault("variables", {})
|
|
178
|
+
values["variables"][nested["key"]] = nested["value"]
|
|
179
|
+
elif nested["attribute"] == "needs":
|
|
180
|
+
values.setdefault("needs", [])
|
|
181
|
+
values["needs"].append(nested["value"])
|
|
182
|
+
return values
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def load_pipeline_entities(
|
|
186
|
+
pipeline_file: str,
|
|
187
|
+
include_nested: bool = True,
|
|
188
|
+
gitlab_url: str | None = None,
|
|
189
|
+
token: str | None = None,
|
|
190
|
+
project: str | None = None,
|
|
191
|
+
group: str | None = None,
|
|
192
|
+
enrich_includes: bool = True,
|
|
193
|
+
enrich_images: bool = True,
|
|
194
|
+
load_api_entities: bool = True,
|
|
195
|
+
) -> dict[str, list[dict]]:
|
|
196
|
+
from src.compliance.api_config import resolve_group, resolve_project, resolve_token
|
|
197
|
+
from src.compliance.release_cache import ReleaseMetadataCache
|
|
198
|
+
|
|
199
|
+
entities = load_yaml_entities(pipeline_file, include_nested=include_nested)
|
|
200
|
+
|
|
201
|
+
userdata = {
|
|
202
|
+
"project": project or "",
|
|
203
|
+
"group": group or "",
|
|
204
|
+
}
|
|
205
|
+
api_token = token or resolve_token(userdata)
|
|
206
|
+
resolved_project = resolve_project(userdata)
|
|
207
|
+
resolved_group = resolve_group(userdata)
|
|
208
|
+
|
|
209
|
+
cache = ReleaseMetadataCache()
|
|
210
|
+
gl = None
|
|
211
|
+
resolved_gitlab_url = (
|
|
212
|
+
gitlab_url
|
|
213
|
+
or os.getenv("CI_SERVER_URL")
|
|
214
|
+
or os.getenv("GITLAB_URL")
|
|
215
|
+
or "https://gitlab.com"
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
needs_gitlab_client = api_token and (
|
|
219
|
+
enrich_includes
|
|
220
|
+
or enrich_images
|
|
221
|
+
or (load_api_entities and (resolved_project or resolved_group))
|
|
222
|
+
)
|
|
223
|
+
if needs_gitlab_client:
|
|
224
|
+
try:
|
|
225
|
+
import gitlab
|
|
226
|
+
|
|
227
|
+
gl = gitlab.Gitlab(resolved_gitlab_url, private_token=api_token)
|
|
228
|
+
gl.auth()
|
|
229
|
+
except Exception:
|
|
230
|
+
gl = None
|
|
231
|
+
|
|
232
|
+
if api_token and enrich_includes:
|
|
233
|
+
from src.compliance.include_versions import enrich_includes_with_releases
|
|
234
|
+
|
|
235
|
+
enriched_includes = enrich_includes_with_releases(
|
|
236
|
+
[include["values"] for include in entities["includes"]],
|
|
237
|
+
gitlab_url=gitlab_url,
|
|
238
|
+
token=api_token,
|
|
239
|
+
gl=gl,
|
|
240
|
+
cache=cache,
|
|
241
|
+
)
|
|
242
|
+
entities["includes"] = [
|
|
243
|
+
_include_entity(index, include)
|
|
244
|
+
for index, include in enumerate(enriched_includes)
|
|
245
|
+
]
|
|
246
|
+
|
|
247
|
+
if api_token and enrich_images:
|
|
248
|
+
from src.compliance.image_versions import enrich_container_images_with_releases
|
|
249
|
+
|
|
250
|
+
enriched_images = enrich_container_images_with_releases(
|
|
251
|
+
[image["values"] for image in entities["container_images"]],
|
|
252
|
+
gitlab_url=gitlab_url,
|
|
253
|
+
token=api_token,
|
|
254
|
+
cache=cache,
|
|
255
|
+
)
|
|
256
|
+
entities["container_images"] = [
|
|
257
|
+
_container_image_entity(index, image)
|
|
258
|
+
for index, image in enumerate(enriched_images, 1)
|
|
259
|
+
]
|
|
260
|
+
|
|
261
|
+
if api_token and load_api_entities and (resolved_project or resolved_group) and gl:
|
|
262
|
+
from src.compliance.gitlab_api import load_api_entities as fetch_api_entities
|
|
263
|
+
|
|
264
|
+
api_entities = fetch_api_entities(
|
|
265
|
+
gitlab_url=gitlab_url,
|
|
266
|
+
token=api_token,
|
|
267
|
+
project=resolved_project,
|
|
268
|
+
group=resolved_group,
|
|
269
|
+
gl=gl,
|
|
270
|
+
)
|
|
271
|
+
entities["project_settings"].extend(api_entities.get("project_settings", []))
|
|
272
|
+
entities["project_ci_variables"].extend(
|
|
273
|
+
api_entities.get("project_ci_variables", [])
|
|
274
|
+
)
|
|
275
|
+
entities["group_settings"].extend(api_entities.get("group_settings", []))
|
|
276
|
+
|
|
277
|
+
return entities
|
src/compliance/models.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Compliance result models."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class ScenarioResult:
|
|
10
|
+
feature: str
|
|
11
|
+
name: str
|
|
12
|
+
status: str
|
|
13
|
+
message: str = ""
|
|
14
|
+
policy_id: str = ""
|
|
15
|
+
title: str = ""
|
|
16
|
+
description: str = ""
|
|
17
|
+
severity: str = ""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class ComplianceResult:
|
|
22
|
+
success: bool
|
|
23
|
+
exit_code: int
|
|
24
|
+
features: int = 0
|
|
25
|
+
scenarios: int = 0
|
|
26
|
+
passed: int = 0
|
|
27
|
+
failed: int = 0
|
|
28
|
+
skipped: int = 0
|
|
29
|
+
scenario_results: list[ScenarioResult] = field(default_factory=list)
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Push and pull GitLab compliance policy bundles to OCI registries."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import shutil
|
|
8
|
+
import tempfile
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import oras.client
|
|
12
|
+
from oras.utils import extract_targz, make_targz
|
|
13
|
+
|
|
14
|
+
POLICY_BUNDLE_MEDIA_TYPE = "application/vnd.gitlab-compliance.policy.bundle.v1+tar+gzip"
|
|
15
|
+
OCI_SCHEME = "oci://"
|
|
16
|
+
DEFAULT_POLICY_DIR = "policy"
|
|
17
|
+
REGISTRY_REFERENCE = re.compile(
|
|
18
|
+
r"^(?:oci://)?" r"(?:localhost(?::\d+)?|[\w.-]+\.[a-zA-Z]{2,})(?::\d+)?/"
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def is_oci_reference(value: str) -> bool:
|
|
23
|
+
if value.startswith(OCI_SCHEME):
|
|
24
|
+
return True
|
|
25
|
+
if os.path.isdir(value) or os.path.isfile(value):
|
|
26
|
+
return False
|
|
27
|
+
return bool(REGISTRY_REFERENCE.match(value))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def normalize_oci_reference(reference: str) -> str:
|
|
31
|
+
reference = reference.strip()
|
|
32
|
+
if reference.startswith(OCI_SCHEME):
|
|
33
|
+
reference = reference[len(OCI_SCHEME) :]
|
|
34
|
+
if ":" not in reference.split("/")[-1]:
|
|
35
|
+
reference = f"{reference}:latest"
|
|
36
|
+
return reference
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def bundle_policies_dir(features_dir: str) -> str:
|
|
40
|
+
if not os.path.isdir(features_dir):
|
|
41
|
+
raise FileNotFoundError(f"Policies directory not found: {features_dir}")
|
|
42
|
+
|
|
43
|
+
feature_files = list(Path(features_dir).rglob("*.feature"))
|
|
44
|
+
if not feature_files:
|
|
45
|
+
raise FileNotFoundError(f"No .feature files found in {features_dir}")
|
|
46
|
+
|
|
47
|
+
bundle_path = os.path.join(
|
|
48
|
+
tempfile.mkdtemp(prefix="gitlab-compliance-policy-bundle-"), "policies.tar.gz"
|
|
49
|
+
)
|
|
50
|
+
return make_targz(features_dir, bundle_path)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def extract_policy_bundle(bundle_path: str, output_dir: str) -> str:
|
|
54
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
55
|
+
extract_targz(bundle_path, output_dir)
|
|
56
|
+
return os.path.abspath(output_dir)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _client() -> oras.client.OrasClient:
|
|
60
|
+
return oras.client.OrasClient()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def push_policies(features_dir: str, target: str) -> str:
|
|
64
|
+
target = normalize_oci_reference(target)
|
|
65
|
+
bundle_path = bundle_policies_dir(features_dir)
|
|
66
|
+
bundle_dir = os.path.dirname(bundle_path)
|
|
67
|
+
|
|
68
|
+
try:
|
|
69
|
+
client = _client()
|
|
70
|
+
response = client.push(
|
|
71
|
+
target=target,
|
|
72
|
+
files=[f"{bundle_path}:{POLICY_BUNDLE_MEDIA_TYPE}"],
|
|
73
|
+
manifest_annotations={
|
|
74
|
+
"org.opencontainers.image.title": "GitLab Docs compliance policies",
|
|
75
|
+
"org.opencontainers.image.description": "Gherkin compliance policy bundle",
|
|
76
|
+
},
|
|
77
|
+
)
|
|
78
|
+
digest = ""
|
|
79
|
+
if hasattr(response, "json"):
|
|
80
|
+
try:
|
|
81
|
+
digest = response.json().get("digest", "")
|
|
82
|
+
except Exception:
|
|
83
|
+
digest = ""
|
|
84
|
+
if digest:
|
|
85
|
+
return digest
|
|
86
|
+
return getattr(response, "reason", "pushed")
|
|
87
|
+
finally:
|
|
88
|
+
shutil.rmtree(bundle_dir, ignore_errors=True)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def pull_policies(target: str, output_dir: str | None = None) -> str:
|
|
92
|
+
target = normalize_oci_reference(target)
|
|
93
|
+
output_dir = os.path.abspath(output_dir or DEFAULT_POLICY_DIR)
|
|
94
|
+
temp_root = tempfile.mkdtemp(prefix="gitlab-compliance-policy-pull-")
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
client = _client()
|
|
98
|
+
pulled_files = client.pull(
|
|
99
|
+
target=target,
|
|
100
|
+
outdir=temp_root,
|
|
101
|
+
allowed_media_type=[
|
|
102
|
+
POLICY_BUNDLE_MEDIA_TYPE,
|
|
103
|
+
"application/vnd.oci.image.layer.v1.tar+gzip",
|
|
104
|
+
],
|
|
105
|
+
overwrite=True,
|
|
106
|
+
)
|
|
107
|
+
bundle_path = _find_bundle_file(temp_root, pulled_files)
|
|
108
|
+
if os.path.exists(output_dir):
|
|
109
|
+
shutil.rmtree(output_dir)
|
|
110
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
111
|
+
extract_policy_bundle(bundle_path, output_dir)
|
|
112
|
+
return output_dir
|
|
113
|
+
finally:
|
|
114
|
+
shutil.rmtree(temp_root, ignore_errors=True)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _find_bundle_file(temp_root: str, pulled_files: list[str] | None) -> str:
|
|
118
|
+
candidates: list[str] = []
|
|
119
|
+
if pulled_files:
|
|
120
|
+
candidates.extend(pulled_files)
|
|
121
|
+
for root, _dirs, files in os.walk(temp_root):
|
|
122
|
+
for filename in files:
|
|
123
|
+
if filename.endswith(".tar.gz") or filename.endswith(".tgz"):
|
|
124
|
+
candidates.append(os.path.join(root, filename))
|
|
125
|
+
|
|
126
|
+
for candidate in candidates:
|
|
127
|
+
if os.path.isfile(candidate):
|
|
128
|
+
return candidate
|
|
129
|
+
|
|
130
|
+
raise FileNotFoundError("Policy bundle not found in pulled OCI artifact.")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def resolve_features_dir(features_dir: str, cache_dir: str | None = None) -> str:
|
|
134
|
+
if os.path.isdir(features_dir):
|
|
135
|
+
return os.path.abspath(features_dir)
|
|
136
|
+
|
|
137
|
+
if is_oci_reference(features_dir):
|
|
138
|
+
cache_dir = cache_dir or os.path.join(
|
|
139
|
+
tempfile.gettempdir(), "gitlab-compliance-policies"
|
|
140
|
+
)
|
|
141
|
+
os.makedirs(cache_dir, exist_ok=True)
|
|
142
|
+
return pull_policies(features_dir, output_dir=cache_dir)
|
|
143
|
+
|
|
144
|
+
raise FileNotFoundError(
|
|
145
|
+
f"Policies source not found: {features_dir}. "
|
|
146
|
+
"Provide a local directory or an OCI reference such as oci://registry.example.com/policies:1.0.0"
|
|
147
|
+
)
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Render compliance policy catalogs from Conftest-style annotations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import html
|
|
6
|
+
import os
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
|
|
9
|
+
from src.compliance.metadata import (
|
|
10
|
+
PolicyAnnotation,
|
|
11
|
+
PolicyCatalog,
|
|
12
|
+
collect_policy_index,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _relative_path(features_dir: str, feature_file: str) -> str:
|
|
17
|
+
try:
|
|
18
|
+
return os.path.relpath(feature_file, os.path.abspath(features_dir))
|
|
19
|
+
except ValueError:
|
|
20
|
+
return feature_file
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _location(features_dir: str, annotation: PolicyAnnotation) -> str:
|
|
24
|
+
rel_path = _relative_path(features_dir, annotation.feature_file)
|
|
25
|
+
if annotation.line:
|
|
26
|
+
return f"{rel_path}:{annotation.line}"
|
|
27
|
+
return rel_path
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def render_policy_catalog_markdown(catalog: PolicyCatalog, features_dir: str) -> str:
|
|
31
|
+
lines = [
|
|
32
|
+
"# GitLab CI Compliance Policy Catalog",
|
|
33
|
+
"",
|
|
34
|
+
f"- **Policies directory:** `{features_dir}`",
|
|
35
|
+
f"- **Generated:** {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}",
|
|
36
|
+
"",
|
|
37
|
+
"## Index",
|
|
38
|
+
"",
|
|
39
|
+
"| ID | Title | Scope | Location |",
|
|
40
|
+
"|----|-------|-------|----------|",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
for policy in collect_policy_index(catalog):
|
|
44
|
+
lines.append(
|
|
45
|
+
f"| `{policy.policy_id}` | {policy.title} | {policy.scope} | `{_location(features_dir, policy)}` |"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
lines.append("")
|
|
49
|
+
for feature in catalog.features:
|
|
50
|
+
rel_feature = _relative_path(features_dir, feature.feature_file)
|
|
51
|
+
lines.extend(
|
|
52
|
+
[
|
|
53
|
+
f"## {feature.feature_name}",
|
|
54
|
+
"",
|
|
55
|
+
f"**File:** `{rel_feature}`",
|
|
56
|
+
]
|
|
57
|
+
)
|
|
58
|
+
if feature.annotation:
|
|
59
|
+
lines.extend(
|
|
60
|
+
[
|
|
61
|
+
f"**Feature ID:** `{feature.annotation.policy_id}`",
|
|
62
|
+
"",
|
|
63
|
+
feature.annotation.description
|
|
64
|
+
or "_No feature description provided._",
|
|
65
|
+
"",
|
|
66
|
+
]
|
|
67
|
+
)
|
|
68
|
+
if feature.scenarios:
|
|
69
|
+
lines.append("### Scenarios")
|
|
70
|
+
lines.append("")
|
|
71
|
+
for scenario in feature.scenarios:
|
|
72
|
+
lines.extend(
|
|
73
|
+
[
|
|
74
|
+
f"#### `{scenario.policy_id}` — {scenario.title}",
|
|
75
|
+
"",
|
|
76
|
+
f"- **Location:** `{_location(features_dir, scenario)}`",
|
|
77
|
+
]
|
|
78
|
+
)
|
|
79
|
+
if scenario.description:
|
|
80
|
+
lines.append(f"- **Description:** {scenario.description}")
|
|
81
|
+
if scenario.custom:
|
|
82
|
+
custom_bits = ", ".join(
|
|
83
|
+
f"`{key}`: {value}"
|
|
84
|
+
for key, value in scenario.custom.items()
|
|
85
|
+
if key != "id"
|
|
86
|
+
)
|
|
87
|
+
if custom_bits:
|
|
88
|
+
lines.append(f"- **Custom:** {custom_bits}")
|
|
89
|
+
lines.append("")
|
|
90
|
+
|
|
91
|
+
return "\n".join(lines).rstrip() + "\n"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def render_policy_catalog_html(catalog: PolicyCatalog, features_dir: str) -> str:
|
|
95
|
+
index_rows = []
|
|
96
|
+
for policy in collect_policy_index(catalog):
|
|
97
|
+
index_rows.append(
|
|
98
|
+
"<tr>"
|
|
99
|
+
f"<td><code>{html.escape(policy.policy_id)}</code></td>"
|
|
100
|
+
f"<td>{html.escape(policy.title)}</td>"
|
|
101
|
+
f"<td>{html.escape(policy.scope)}</td>"
|
|
102
|
+
f"<td><code>{html.escape(_location(features_dir, policy))}</code></td>"
|
|
103
|
+
"</tr>"
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
detail_sections = []
|
|
107
|
+
for feature in catalog.features:
|
|
108
|
+
rel_feature = html.escape(_relative_path(features_dir, feature.feature_file))
|
|
109
|
+
scenario_blocks = []
|
|
110
|
+
for scenario in feature.scenarios:
|
|
111
|
+
custom_html = ""
|
|
112
|
+
if scenario.custom:
|
|
113
|
+
items = "".join(
|
|
114
|
+
f"<li><code>{html.escape(key)}</code>: {html.escape(str(value))}</li>"
|
|
115
|
+
for key, value in scenario.custom.items()
|
|
116
|
+
if key != "id"
|
|
117
|
+
)
|
|
118
|
+
if items:
|
|
119
|
+
custom_html = f"<ul>{items}</ul>"
|
|
120
|
+
scenario_blocks.append(
|
|
121
|
+
"<section>"
|
|
122
|
+
f"<h4><code>{html.escape(scenario.policy_id)}</code> — {html.escape(scenario.title)}</h4>"
|
|
123
|
+
f"<p><code>{html.escape(_location(features_dir, scenario))}</code></p>"
|
|
124
|
+
f"<p>{html.escape(scenario.description)}</p>"
|
|
125
|
+
f"{custom_html}"
|
|
126
|
+
"</section>"
|
|
127
|
+
)
|
|
128
|
+
detail_sections.append(
|
|
129
|
+
"<section>"
|
|
130
|
+
f"<h2>{html.escape(feature.feature_name)}</h2>"
|
|
131
|
+
f"<p><strong>File:</strong> <code>{rel_feature}</code></p>"
|
|
132
|
+
+ (
|
|
133
|
+
f"<p><strong>Feature ID:</strong> <code>{html.escape(feature.annotation.policy_id)}</code></p>"
|
|
134
|
+
f"<p>{html.escape(feature.annotation.description)}</p>"
|
|
135
|
+
if feature.annotation
|
|
136
|
+
else ""
|
|
137
|
+
)
|
|
138
|
+
+ "".join(scenario_blocks)
|
|
139
|
+
+ "</section>"
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
return f"""<!DOCTYPE html>
|
|
143
|
+
<html lang="en">
|
|
144
|
+
<head>
|
|
145
|
+
<meta charset="utf-8">
|
|
146
|
+
<title>GitLab CI Compliance Policy Catalog</title>
|
|
147
|
+
<style>
|
|
148
|
+
body {{ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; margin: 2rem; color: #1f2937; }}
|
|
149
|
+
table {{ width: 100%; border-collapse: collapse; margin: 1rem 0 2rem; }}
|
|
150
|
+
th, td {{ border: 1px solid #e5e7eb; padding: 0.75rem; text-align: left; vertical-align: top; }}
|
|
151
|
+
th {{ background: #f9fafb; }}
|
|
152
|
+
section {{ margin-bottom: 2rem; }}
|
|
153
|
+
.meta {{ color: #6b7280; margin-bottom: 1.5rem; }}
|
|
154
|
+
</style>
|
|
155
|
+
</head>
|
|
156
|
+
<body>
|
|
157
|
+
<h1>GitLab CI Compliance Policy Catalog</h1>
|
|
158
|
+
<p class="meta"><strong>Policies:</strong> {html.escape(features_dir)}<br>
|
|
159
|
+
<strong>Generated:</strong> {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}</p>
|
|
160
|
+
<h2>Index</h2>
|
|
161
|
+
<table>
|
|
162
|
+
<thead><tr><th>ID</th><th>Title</th><th>Scope</th><th>Location</th></tr></thead>
|
|
163
|
+
<tbody>{''.join(index_rows)}</tbody>
|
|
164
|
+
</table>
|
|
165
|
+
{''.join(detail_sections)}
|
|
166
|
+
</body>
|
|
167
|
+
</html>
|
|
168
|
+
"""
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def render_policy_catalog(
|
|
172
|
+
catalog: PolicyCatalog, features_dir: str, output_format: str
|
|
173
|
+
) -> str:
|
|
174
|
+
fmt = output_format.lower()
|
|
175
|
+
if fmt == "html":
|
|
176
|
+
return render_policy_catalog_html(catalog, features_dir)
|
|
177
|
+
return render_policy_catalog_markdown(catalog, features_dir)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Per-run cache for GitLab and registry release metadata."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class ReleaseMetadataCache:
|
|
11
|
+
gitlab_tag_dates: dict[tuple[str, str], dict[str, datetime]] = field(
|
|
12
|
+
default_factory=dict
|
|
13
|
+
)
|
|
14
|
+
registry_tags: dict[tuple[str, str], list[str]] = field(default_factory=dict)
|
|
15
|
+
registry_digests: dict[tuple[str, str, str], str] = field(default_factory=dict)
|
|
16
|
+
docker_auth_tokens: dict[str, str] = field(default_factory=dict)
|
|
17
|
+
|
|
18
|
+
def get_gitlab_tag_dates(
|
|
19
|
+
self, gitlab_url: str, project_path: str
|
|
20
|
+
) -> dict[str, datetime] | None:
|
|
21
|
+
return self.gitlab_tag_dates.get((gitlab_url, project_path))
|
|
22
|
+
|
|
23
|
+
def set_gitlab_tag_dates(
|
|
24
|
+
self, gitlab_url: str, project_path: str, tag_dates: dict[str, datetime]
|
|
25
|
+
) -> None:
|
|
26
|
+
self.gitlab_tag_dates[(gitlab_url, project_path)] = tag_dates
|
|
27
|
+
|
|
28
|
+
def get_registry_tags(self, registry: str, repository: str) -> list[str] | None:
|
|
29
|
+
return self.registry_tags.get((registry, repository))
|
|
30
|
+
|
|
31
|
+
def set_registry_tags(
|
|
32
|
+
self, registry: str, repository: str, tags: list[str]
|
|
33
|
+
) -> None:
|
|
34
|
+
self.registry_tags[(registry, repository)] = tags
|
|
35
|
+
|
|
36
|
+
def get_registry_digest(
|
|
37
|
+
self, registry: str, repository: str, tag: str
|
|
38
|
+
) -> str | None:
|
|
39
|
+
digest = self.registry_digests.get((registry, repository, tag))
|
|
40
|
+
return digest if digest is not None else None
|
|
41
|
+
|
|
42
|
+
def set_registry_digest(
|
|
43
|
+
self, registry: str, repository: str, tag: str, digest: str
|
|
44
|
+
) -> None:
|
|
45
|
+
self.registry_digests[(registry, repository, tag)] = digest
|
|
46
|
+
|
|
47
|
+
def get_docker_token(self, repository: str) -> str | None:
|
|
48
|
+
return self.docker_auth_tokens.get(repository)
|
|
49
|
+
|
|
50
|
+
def set_docker_token(self, repository: str, token: str) -> None:
|
|
51
|
+
self.docker_auth_tokens[repository] = token
|