seanerstack 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seanerstack/__init__.py +5 -0
- seanerstack/auth.py +143 -0
- seanerstack/bootstrap.py +161 -0
- seanerstack/buildcache.py +68 -0
- seanerstack/cli.py +397 -0
- seanerstack/config.py +93 -0
- seanerstack/docsgen.py +146 -0
- seanerstack/isolation.py +243 -0
- seanerstack/paths.py +52 -0
- seanerstack/recipe.py +108 -0
- seanerstack/secrets.py +60 -0
- seanerstack/sync.py +381 -0
- seanerstack/validate.py +202 -0
- seanerstack-0.1.3.data/data/share/seanerstack/config.toml +37 -0
- seanerstack-0.1.3.data/data/share/seanerstack/environments/demobus-dev/spack.yaml +10 -0
- seanerstack-0.1.3.data/data/share/seanerstack/environments/flux-core-dev/spack.yaml +10 -0
- seanerstack-0.1.3.data/data/share/seanerstack/environments/properdocs-dev/spack.yaml +10 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack-repo-index.yaml +4 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack_repo/seanerstack/packages/demobus/package.py +38 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack_repo/seanerstack/packages/properdocs/package.py +34 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack_repo/seanerstack/packages/seanergys_meta/package.py +13 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack_repo/seanerstack/repo.yaml +4 -0
- seanerstack-0.1.3.dist-info/METADATA +224 -0
- seanerstack-0.1.3.dist-info/RECORD +27 -0
- seanerstack-0.1.3.dist-info/WHEEL +4 -0
- seanerstack-0.1.3.dist-info/entry_points.txt +3 -0
- seanerstack-0.1.3.dist-info/licenses/LICENSE +202 -0
seanerstack/sync.py
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
import hashlib
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import re
|
|
8
|
+
import subprocess
|
|
9
|
+
import urllib.error
|
|
10
|
+
import urllib.request
|
|
11
|
+
from urllib.parse import urlsplit
|
|
12
|
+
|
|
13
|
+
from .auth import GitAuth, git_environment, http_auth_headers
|
|
14
|
+
from .config import Config
|
|
15
|
+
from .secrets import SecretStore
|
|
16
|
+
from typing import Collection, Iterable
|
|
17
|
+
|
|
18
|
+
from packaging.version import InvalidVersion, Version
|
|
19
|
+
|
|
20
|
+
from .paths import repository_path
|
|
21
|
+
from .recipe import Recipe, parse_recipe
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class RemoteRefs:
|
|
26
|
+
tags: dict[str, str]
|
|
27
|
+
heads: dict[str, str]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class SyncResult:
|
|
32
|
+
package: str
|
|
33
|
+
changed: bool
|
|
34
|
+
missing_versions: tuple[str, ...]
|
|
35
|
+
missing_branches: tuple[str, ...]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class TagConvention:
|
|
40
|
+
prefix: str
|
|
41
|
+
suffix: str
|
|
42
|
+
|
|
43
|
+
def version_from_tag(self, tag: str) -> str | None:
|
|
44
|
+
if not tag.startswith(self.prefix):
|
|
45
|
+
return None
|
|
46
|
+
if self.suffix and not tag.endswith(self.suffix):
|
|
47
|
+
return None
|
|
48
|
+
end = len(tag) - len(self.suffix) if self.suffix else len(tag)
|
|
49
|
+
version = tag[len(self.prefix):end]
|
|
50
|
+
if not version:
|
|
51
|
+
return None
|
|
52
|
+
# Keep synchronization conservative. Spack accepts a broad version
|
|
53
|
+
# grammar, but existing Seanerstack release recipes use PEP 440-compatible
|
|
54
|
+
# version strings and packaging gives us reliable ordering as well.
|
|
55
|
+
try:
|
|
56
|
+
Version(version)
|
|
57
|
+
except InvalidVersion:
|
|
58
|
+
return None
|
|
59
|
+
return version
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def package_paths(repository: Path | None = None) -> dict[str, Path]:
|
|
63
|
+
packages = (repository or repository_path()) / "packages"
|
|
64
|
+
if not packages.is_dir():
|
|
65
|
+
return {}
|
|
66
|
+
result: dict[str, Path] = {}
|
|
67
|
+
for directory in sorted(path for path in packages.iterdir() if path.is_dir()):
|
|
68
|
+
recipe = directory / "package.py"
|
|
69
|
+
if recipe.is_file():
|
|
70
|
+
result[directory.name.replace("_", "-")] = recipe
|
|
71
|
+
return result
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def read_remote_refs(remote: str, *, auth: GitAuth | None = None) -> RemoteRefs:
|
|
75
|
+
with git_environment(auth) as env:
|
|
76
|
+
process = subprocess.run(
|
|
77
|
+
["git", "ls-remote", "--heads", "--tags", remote],
|
|
78
|
+
check=True,
|
|
79
|
+
text=True,
|
|
80
|
+
capture_output=True,
|
|
81
|
+
env=env,
|
|
82
|
+
)
|
|
83
|
+
tags: dict[str, str] = {}
|
|
84
|
+
dereferenced: dict[str, str] = {}
|
|
85
|
+
heads: dict[str, str] = {}
|
|
86
|
+
for line in process.stdout.splitlines():
|
|
87
|
+
if not line.strip():
|
|
88
|
+
continue
|
|
89
|
+
commit, ref = line.split(maxsplit=1)
|
|
90
|
+
if ref.startswith("refs/heads/"):
|
|
91
|
+
heads[ref.removeprefix("refs/heads/")] = commit
|
|
92
|
+
elif ref.startswith("refs/tags/"):
|
|
93
|
+
tag = ref.removeprefix("refs/tags/")
|
|
94
|
+
if tag.endswith("^{}"):
|
|
95
|
+
dereferenced[tag[:-3]] = commit
|
|
96
|
+
else:
|
|
97
|
+
tags[tag] = commit
|
|
98
|
+
tags.update(dereferenced)
|
|
99
|
+
return RemoteRefs(tags=tags, heads=heads)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _release_versions(recipe: Recipe) -> set[str]:
|
|
103
|
+
return {
|
|
104
|
+
str(directive.args[0])
|
|
105
|
+
for directive in recipe.directives
|
|
106
|
+
if directive.name == "version" and directive.args
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _tracked_branches(recipe: Recipe) -> dict[str, str]:
|
|
111
|
+
return {
|
|
112
|
+
str(directive.kwargs["branch"]): str(directive.args[0])
|
|
113
|
+
for directive in recipe.directives
|
|
114
|
+
if directive.name == "version"
|
|
115
|
+
and directive.args
|
|
116
|
+
and isinstance(directive.kwargs.get("branch"), str)
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _release_kind(recipe: Recipe) -> str:
|
|
121
|
+
"""Return "tag" for Git-fetched releases or "sha256" for archive releases."""
|
|
122
|
+
tagged: list[str] = []
|
|
123
|
+
archived: list[str] = []
|
|
124
|
+
for directive in recipe.directives:
|
|
125
|
+
if directive.name != "version" or not directive.args:
|
|
126
|
+
continue
|
|
127
|
+
if "tag" in directive.kwargs:
|
|
128
|
+
tagged.append(str(directive.args[0]))
|
|
129
|
+
elif "sha256" in directive.kwargs and not {"branch", "commit"} & directive.kwargs.keys():
|
|
130
|
+
archived.append(str(directive.args[0]))
|
|
131
|
+
if tagged and archived:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"{recipe.path}: recipe mixes tag= releases ({', '.join(tagged)}) "
|
|
134
|
+
f"and sha256= releases ({', '.join(archived)}); use one release kind"
|
|
135
|
+
)
|
|
136
|
+
return "sha256" if archived else "tag"
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _version_pattern(version: str) -> re.Pattern[str]:
|
|
140
|
+
# Match a whole version: "0.2" must not match inside "0.2.1" or "10.2".
|
|
141
|
+
return re.compile(rf"(?<![0-9.]){re.escape(version)}(?!\.?[0-9])")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _url_version(recipe: Recipe) -> str:
|
|
145
|
+
"""Return the release version the recipe's url points to."""
|
|
146
|
+
assert recipe.url is not None
|
|
147
|
+
candidates = [
|
|
148
|
+
str(directive.args[0])
|
|
149
|
+
for directive in recipe.directives
|
|
150
|
+
if directive.name == "version"
|
|
151
|
+
and directive.args
|
|
152
|
+
and "sha256" in directive.kwargs
|
|
153
|
+
and _version_pattern(str(directive.args[0])).search(recipe.url)
|
|
154
|
+
]
|
|
155
|
+
if len(candidates) != 1:
|
|
156
|
+
found = ", ".join(candidates) if candidates else "none"
|
|
157
|
+
raise ValueError(
|
|
158
|
+
f"{recipe.path}: url must contain exactly one sha256 release version "
|
|
159
|
+
f"(found: {found}): {recipe.url}"
|
|
160
|
+
)
|
|
161
|
+
return candidates[0]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def archive_url(recipe: Recipe, version: str) -> str:
|
|
165
|
+
"""Substitute version into the recipe url, like Spack's url_for_version."""
|
|
166
|
+
assert recipe.url is not None
|
|
167
|
+
return _version_pattern(_url_version(recipe)).sub(lambda _: version, recipe.url)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _url_tag_convention(recipe: Recipe, tags: Collection[str] | None) -> TagConvention:
|
|
171
|
+
if not recipe.url:
|
|
172
|
+
raise ValueError(f"{recipe.path}: sha256 releases require a url to infer the release-tag convention")
|
|
173
|
+
version = _url_version(recipe)
|
|
174
|
+
pattern = _version_pattern(version)
|
|
175
|
+
conventions: set[tuple[str, str]] = set()
|
|
176
|
+
for segment in urlsplit(recipe.url).path.split("/"):
|
|
177
|
+
matches = list(pattern.finditer(segment))
|
|
178
|
+
if len(matches) == 1:
|
|
179
|
+
conventions.add((segment[:matches[0].start()], segment[matches[0].end():]))
|
|
180
|
+
if tags is not None:
|
|
181
|
+
conventions = {(prefix, suffix) for prefix, suffix in conventions if f"{prefix}{version}{suffix}" in tags}
|
|
182
|
+
if len(conventions) != 1:
|
|
183
|
+
rendered = ", ".join(f"{prefix}{{version}}{suffix}" for prefix, suffix in sorted(conventions)) or "none"
|
|
184
|
+
raise ValueError(
|
|
185
|
+
f"{recipe.path}: cannot infer one release-tag convention for {version} "
|
|
186
|
+
f"from url {recipe.url} (candidates: {rendered})"
|
|
187
|
+
)
|
|
188
|
+
prefix, suffix = next(iter(conventions))
|
|
189
|
+
return TagConvention(prefix=prefix, suffix=suffix)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def infer_tag_convention(recipe: Recipe, tags: Collection[str] | None = None) -> TagConvention:
|
|
193
|
+
"""Infer how release tags are named from the recipe's existing releases.
|
|
194
|
+
|
|
195
|
+
Tag releases carry the tag name directly. For sha256 releases the tag is
|
|
196
|
+
taken from the url path segment holding the version; upstream tags, when
|
|
197
|
+
given, pick among several such segments.
|
|
198
|
+
"""
|
|
199
|
+
if _release_kind(recipe) == "sha256":
|
|
200
|
+
return _url_tag_convention(recipe, tags)
|
|
201
|
+
conventions: set[tuple[str, str]] = set()
|
|
202
|
+
for directive in recipe.directives:
|
|
203
|
+
if directive.name != "version" or not directive.args:
|
|
204
|
+
continue
|
|
205
|
+
tag = directive.kwargs.get("tag")
|
|
206
|
+
if not isinstance(tag, str):
|
|
207
|
+
continue
|
|
208
|
+
version = str(directive.args[0])
|
|
209
|
+
first = tag.find(version)
|
|
210
|
+
if first < 0 or tag.find(version, first + len(version)) >= 0:
|
|
211
|
+
raise ValueError(
|
|
212
|
+
f"{recipe.path}: cannot infer tag convention from "
|
|
213
|
+
f'version("{version}", tag="{tag}")'
|
|
214
|
+
)
|
|
215
|
+
conventions.add((tag[:first], tag[first + len(version):]))
|
|
216
|
+
|
|
217
|
+
if not conventions:
|
|
218
|
+
raise ValueError(
|
|
219
|
+
f"{recipe.path}: cannot infer release-tag convention; "
|
|
220
|
+
"add at least one version(..., tag=...) or version(..., sha256=...) directive"
|
|
221
|
+
)
|
|
222
|
+
if len(conventions) != 1:
|
|
223
|
+
rendered = ", ".join(f"{prefix}{{version}}{suffix}" for prefix, suffix in sorted(conventions))
|
|
224
|
+
raise ValueError(f"{recipe.path}: inconsistent release-tag conventions: {rendered}")
|
|
225
|
+
|
|
226
|
+
prefix, suffix = next(iter(conventions))
|
|
227
|
+
return TagConvention(prefix=prefix, suffix=suffix)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _version_key(value: str) -> tuple[int, Version | str]:
|
|
231
|
+
try:
|
|
232
|
+
return (1, Version(value))
|
|
233
|
+
except InvalidVersion:
|
|
234
|
+
return (0, value)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _urlopen(request: urllib.request.Request):
|
|
238
|
+
return urllib.request.urlopen(request, timeout=120)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def archive_sha256(url: str, *, auth: GitAuth | None = None) -> str:
|
|
242
|
+
"""Download a .tar.gz archive and return its sha256 without keeping it."""
|
|
243
|
+
request = urllib.request.Request(url)
|
|
244
|
+
for name, value in http_auth_headers(auth).items():
|
|
245
|
+
# Unredirected headers are not sent again if the server redirects.
|
|
246
|
+
request.add_unredirected_header(name, value)
|
|
247
|
+
digest = hashlib.sha256()
|
|
248
|
+
try:
|
|
249
|
+
with _urlopen(request) as response:
|
|
250
|
+
content_type = response.headers.get_content_type()
|
|
251
|
+
head = response.read(2)
|
|
252
|
+
digest.update(head)
|
|
253
|
+
while chunk := response.read(1 << 16):
|
|
254
|
+
digest.update(chunk)
|
|
255
|
+
except urllib.error.HTTPError as error:
|
|
256
|
+
raise ValueError(f"{url}: download failed: HTTP {error.code} {error.reason}") from None
|
|
257
|
+
except urllib.error.URLError as error:
|
|
258
|
+
raise ValueError(f"{url}: download failed: {error.reason}") from None
|
|
259
|
+
|
|
260
|
+
# A private archive fetched without valid credentials is GitLab's sign-in
|
|
261
|
+
# page, served with HTTP 200.
|
|
262
|
+
if content_type == "text/html" or head != b"\x1f\x8b": # gzip magic bytes
|
|
263
|
+
raise ValueError(
|
|
264
|
+
f"{url}: download is not a gzip archive (Content-Type {content_type}); "
|
|
265
|
+
"check that the recipe's seanerstack_resource credentials grant access"
|
|
266
|
+
)
|
|
267
|
+
return digest.hexdigest()
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _insert_releases(path: Path, recipe: Recipe, directives: list[str]) -> None:
|
|
271
|
+
if not directives:
|
|
272
|
+
return
|
|
273
|
+
|
|
274
|
+
lines = path.read_text(encoding="utf-8").splitlines(keepends=True)
|
|
275
|
+
version_directives = [directive for directive in recipe.directives if directive.name == "version"]
|
|
276
|
+
if version_directives:
|
|
277
|
+
branch_directives = [directive for directive in version_directives if directive.kwargs.get("branch")]
|
|
278
|
+
anchor = (branch_directives[-1] if branch_directives else version_directives[0]).end_lineno
|
|
279
|
+
else:
|
|
280
|
+
raise ValueError(f"{path}: at least one version directive is required for synchronization")
|
|
281
|
+
|
|
282
|
+
additions = [f" {directive}\n" for directive in directives]
|
|
283
|
+
lines[anchor:anchor] = additions
|
|
284
|
+
path.write_text("".join(lines), encoding="utf-8")
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def sync_package(
|
|
288
|
+
package: str,
|
|
289
|
+
*,
|
|
290
|
+
repository: Path | None = None,
|
|
291
|
+
check: bool = False,
|
|
292
|
+
dry_run: bool = False,
|
|
293
|
+
config: Config | None = None,
|
|
294
|
+
secrets: SecretStore | None = None,
|
|
295
|
+
) -> SyncResult:
|
|
296
|
+
paths = package_paths(repository)
|
|
297
|
+
if package not in paths:
|
|
298
|
+
raise ValueError(f"unknown Seanerstack package: {package}")
|
|
299
|
+
|
|
300
|
+
path = paths[package]
|
|
301
|
+
recipe = parse_recipe(path)
|
|
302
|
+
if not recipe.git:
|
|
303
|
+
raise ValueError(f"{package}: recipe has no git URL")
|
|
304
|
+
|
|
305
|
+
kind = _release_kind(recipe)
|
|
306
|
+
if kind == "sha256" and not recipe.url:
|
|
307
|
+
raise ValueError(f"{package}: recipe has sha256 releases but no url")
|
|
308
|
+
|
|
309
|
+
auth = None
|
|
310
|
+
if recipe.seanerstack_resource:
|
|
311
|
+
if config is None:
|
|
312
|
+
raise ValueError(f"{package}: recipe requires resource {recipe.seanerstack_resource!r} but no Seanerstack config was supplied")
|
|
313
|
+
resource = config.resource(recipe.seanerstack_resource)
|
|
314
|
+
if resource.kind != "git":
|
|
315
|
+
raise ValueError(f"{package}: resource {resource.name!r} is {resource.kind!r}, expected 'git'")
|
|
316
|
+
if not recipe.git.startswith(resource.url):
|
|
317
|
+
raise ValueError(f"{package}: Git URL {recipe.git!r} does not belong to resource {resource.name!r} ({resource.url!r})")
|
|
318
|
+
if kind == "sha256":
|
|
319
|
+
assert recipe.url is not None
|
|
320
|
+
if not recipe.url.startswith(resource.url):
|
|
321
|
+
raise ValueError(f"{package}: archive URL {recipe.url!r} does not belong to resource {resource.name!r} ({resource.url!r})")
|
|
322
|
+
if resource.credentials and not recipe.url.startswith("https://"):
|
|
323
|
+
raise ValueError(f"{package}: refusing to send credentials of {resource.name!r} to non-HTTPS URL {recipe.url!r}")
|
|
324
|
+
if resource.credentials:
|
|
325
|
+
if secrets is None:
|
|
326
|
+
raise ValueError(f"{package}: resource {resource.name!r} requires credentials")
|
|
327
|
+
from .auth import git_auth
|
|
328
|
+
auth = git_auth(resource, secrets)
|
|
329
|
+
refs = read_remote_refs(recipe.git, auth=auth) if auth is not None else read_remote_refs(recipe.git)
|
|
330
|
+
convention = infer_tag_convention(recipe, tags=refs.tags)
|
|
331
|
+
existing = _release_versions(recipe)
|
|
332
|
+
|
|
333
|
+
releases: list[tuple[str, str, str]] = []
|
|
334
|
+
for tag, commit in refs.tags.items():
|
|
335
|
+
version = convention.version_from_tag(tag)
|
|
336
|
+
if version is not None and version not in existing:
|
|
337
|
+
releases.append((version, tag, commit))
|
|
338
|
+
releases.sort(key=lambda item: _version_key(item[0]), reverse=True)
|
|
339
|
+
|
|
340
|
+
tracked_branches = _tracked_branches(recipe)
|
|
341
|
+
missing_branches = tuple(sorted(branch for branch in tracked_branches if branch not in refs.heads))
|
|
342
|
+
|
|
343
|
+
if releases and not (check or dry_run):
|
|
344
|
+
if kind == "sha256":
|
|
345
|
+
# Download everything before touching the recipe.
|
|
346
|
+
directives = [
|
|
347
|
+
f'version("{version}", sha256="{archive_sha256(archive_url(recipe, version), auth=auth)}")'
|
|
348
|
+
for version, _, _ in releases
|
|
349
|
+
]
|
|
350
|
+
else:
|
|
351
|
+
directives = [f'version("{version}", tag="{tag}", commit="{commit}")' for version, tag, commit in releases]
|
|
352
|
+
_insert_releases(path, recipe, directives)
|
|
353
|
+
|
|
354
|
+
return SyncResult(
|
|
355
|
+
package=package,
|
|
356
|
+
changed=bool(releases or missing_branches),
|
|
357
|
+
missing_versions=tuple(version for version, _, _ in releases),
|
|
358
|
+
missing_branches=missing_branches,
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def sync_many(
|
|
363
|
+
packages: Iterable[str],
|
|
364
|
+
*,
|
|
365
|
+
repository: Path | None = None,
|
|
366
|
+
check: bool = False,
|
|
367
|
+
dry_run: bool = False,
|
|
368
|
+
config: Config | None = None,
|
|
369
|
+
secrets: SecretStore | None = None,
|
|
370
|
+
) -> list[SyncResult]:
|
|
371
|
+
return [
|
|
372
|
+
sync_package(
|
|
373
|
+
package,
|
|
374
|
+
repository=repository,
|
|
375
|
+
check=check,
|
|
376
|
+
dry_run=dry_run,
|
|
377
|
+
config=config,
|
|
378
|
+
secrets=secrets,
|
|
379
|
+
)
|
|
380
|
+
for package in packages
|
|
381
|
+
]
|
seanerstack/validate.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import ast
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
import tempfile
|
|
10
|
+
|
|
11
|
+
import yaml
|
|
12
|
+
|
|
13
|
+
from .bootstrap import DEFAULT_LANGUAGES
|
|
14
|
+
from .config import Config
|
|
15
|
+
from .secrets import SecretStore
|
|
16
|
+
from .paths import environments_path, repository_path
|
|
17
|
+
from .sync import package_paths, sync_many
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class Finding:
|
|
22
|
+
level: str
|
|
23
|
+
subject: str
|
|
24
|
+
message: str
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _validate_packages(repository: Path) -> list[Finding]:
|
|
28
|
+
findings: list[Finding] = []
|
|
29
|
+
for name, path in package_paths(repository).items():
|
|
30
|
+
try:
|
|
31
|
+
ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
|
32
|
+
findings.append(Finding("ok", f"package:{name}", "Python syntax is valid"))
|
|
33
|
+
except SyntaxError as error:
|
|
34
|
+
findings.append(Finding("error", f"package:{name}", str(error)))
|
|
35
|
+
return findings
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _validate_environments(environments: Path) -> list[Finding]:
|
|
39
|
+
findings: list[Finding] = []
|
|
40
|
+
if not environments.is_dir():
|
|
41
|
+
return [Finding("error", "environments", "environment directory does not exist")]
|
|
42
|
+
|
|
43
|
+
for directory in sorted(path for path in environments.iterdir() if path.is_dir()):
|
|
44
|
+
manifest = directory / "spack.yaml"
|
|
45
|
+
if not manifest.is_file():
|
|
46
|
+
findings.append(Finding("error", f"environment:{directory.name}", "spack.yaml is missing"))
|
|
47
|
+
continue
|
|
48
|
+
try:
|
|
49
|
+
data = yaml.safe_load(manifest.read_text(encoding="utf-8")) or {}
|
|
50
|
+
except yaml.YAMLError as error:
|
|
51
|
+
findings.append(Finding("error", f"environment:{directory.name}", f"invalid YAML: {error}"))
|
|
52
|
+
continue
|
|
53
|
+
if not isinstance(data.get("spack"), dict):
|
|
54
|
+
findings.append(Finding("error", f"environment:{directory.name}", "top-level 'spack' mapping is missing"))
|
|
55
|
+
continue
|
|
56
|
+
findings.append(Finding("ok", f"environment:{directory.name}", "manifest is structurally valid"))
|
|
57
|
+
|
|
58
|
+
develop = data["spack"].get("develop", {}) or {}
|
|
59
|
+
if isinstance(develop, dict):
|
|
60
|
+
for package, entry in develop.items():
|
|
61
|
+
if not isinstance(entry, dict) or "path" not in entry:
|
|
62
|
+
continue
|
|
63
|
+
target = (directory / str(entry["path"])).resolve()
|
|
64
|
+
if target.exists():
|
|
65
|
+
findings.append(Finding("ok", f"develop:{package}", f"{target} exists"))
|
|
66
|
+
else:
|
|
67
|
+
findings.append(Finding("error", f"develop:{package}", f"{target} does not exist"))
|
|
68
|
+
return findings
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _requirements(value) -> list[str]:
|
|
72
|
+
if value is None:
|
|
73
|
+
return []
|
|
74
|
+
if isinstance(value, str):
|
|
75
|
+
return [value]
|
|
76
|
+
if isinstance(value, list):
|
|
77
|
+
result: list[str] = []
|
|
78
|
+
for item in value:
|
|
79
|
+
if isinstance(item, str):
|
|
80
|
+
result.append(item)
|
|
81
|
+
elif isinstance(item, dict) and isinstance(item.get("spec"), str):
|
|
82
|
+
result.append(item["spec"])
|
|
83
|
+
return result
|
|
84
|
+
return []
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _validate_compiler(compiler: str, languages: tuple[str, ...], command: list[str], env) -> list[Finding]:
|
|
88
|
+
process = subprocess.run(
|
|
89
|
+
[*command, "config", "get", "packages"],
|
|
90
|
+
text=True,
|
|
91
|
+
capture_output=True, env=env,
|
|
92
|
+
)
|
|
93
|
+
if process.returncode != 0:
|
|
94
|
+
message = process.stderr.strip() or process.stdout.strip() or "could not read Spack packages configuration"
|
|
95
|
+
return [Finding("error", "compiler", message)]
|
|
96
|
+
|
|
97
|
+
data = yaml.safe_load(process.stdout) or {}
|
|
98
|
+
packages = data.get("packages", {}) if isinstance(data, dict) else {}
|
|
99
|
+
findings: list[Finding] = []
|
|
100
|
+
for language in languages:
|
|
101
|
+
require = _requirements(packages.get(language, {}).get("require") if isinstance(packages.get(language), dict) else None)
|
|
102
|
+
if compiler in require:
|
|
103
|
+
findings.append(Finding("ok", f"compiler:{language}", f"requires {compiler} in the isolated Seanerstack configuration"))
|
|
104
|
+
else:
|
|
105
|
+
findings.append(
|
|
106
|
+
Finding(
|
|
107
|
+
"error",
|
|
108
|
+
f"compiler:{language}",
|
|
109
|
+
f"does not require {compiler} in the isolated Seanerstack configuration; run `seanerstack toolchain bootstrap {compiler}`",
|
|
110
|
+
)
|
|
111
|
+
)
|
|
112
|
+
return findings
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _validate_with_spack(environments: Path, command: list[str], env) -> list[Finding]:
|
|
116
|
+
findings: list[Finding] = []
|
|
117
|
+
for directory in sorted(path for path in environments.iterdir() if path.is_dir() and (path / "spack.yaml").is_file()):
|
|
118
|
+
with tempfile.TemporaryDirectory(prefix="seanerstack-validate-") as temp:
|
|
119
|
+
temp_env = Path(temp) / directory.name
|
|
120
|
+
shutil.copytree(directory, temp_env)
|
|
121
|
+
process = subprocess.run(
|
|
122
|
+
[*command, "-e", str(temp_env), "concretize", "-f"],
|
|
123
|
+
text=True,
|
|
124
|
+
capture_output=True, env=env,
|
|
125
|
+
)
|
|
126
|
+
if process.returncode == 0:
|
|
127
|
+
findings.append(Finding("ok", f"spack:{directory.name}", "environment concretizes"))
|
|
128
|
+
else:
|
|
129
|
+
message = process.stderr.strip() or process.stdout.strip() or "concretization failed"
|
|
130
|
+
findings.append(Finding("error", f"spack:{directory.name}", message))
|
|
131
|
+
return findings
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _validate_resources(repository: Path, config: Config) -> list[Finding]:
|
|
135
|
+
findings: list[Finding] = []
|
|
136
|
+
for name, path in package_paths(repository).items():
|
|
137
|
+
from .recipe import parse_recipe
|
|
138
|
+
recipe = parse_recipe(path)
|
|
139
|
+
if not recipe.seanerstack_resource:
|
|
140
|
+
continue
|
|
141
|
+
try:
|
|
142
|
+
resource = config.resource(recipe.seanerstack_resource)
|
|
143
|
+
except ValueError as error:
|
|
144
|
+
findings.append(Finding("error", f"resource:{name}", str(error)))
|
|
145
|
+
continue
|
|
146
|
+
if resource.kind != "git":
|
|
147
|
+
findings.append(Finding("error", f"resource:{name}", f"{resource.name} is {resource.kind}, expected git"))
|
|
148
|
+
elif recipe.git and not recipe.git.startswith(resource.url):
|
|
149
|
+
findings.append(Finding("error", f"resource:{name}", f"Git URL is outside resource {resource.name}"))
|
|
150
|
+
elif recipe.url and any(
|
|
151
|
+
directive.name == "version" and "sha256" in directive.kwargs for directive in recipe.directives
|
|
152
|
+
) and not recipe.url.startswith(resource.url):
|
|
153
|
+
findings.append(Finding("error", f"resource:{name}", f"archive URL is outside resource {resource.name}"))
|
|
154
|
+
else:
|
|
155
|
+
findings.append(Finding("ok", f"resource:{name}", f"uses {resource.name}"))
|
|
156
|
+
return findings
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def validate(
|
|
160
|
+
*,
|
|
161
|
+
online: bool = False,
|
|
162
|
+
with_spack: bool = False,
|
|
163
|
+
compiler: str | None = None,
|
|
164
|
+
languages: tuple[str, ...] = DEFAULT_LANGUAGES,
|
|
165
|
+
spack_command: list[str] | None = None,
|
|
166
|
+
spack_env=None,
|
|
167
|
+
repository: Path | None = None,
|
|
168
|
+
environments: Path | None = None,
|
|
169
|
+
config: Config | None = None,
|
|
170
|
+
secrets: SecretStore | None = None,
|
|
171
|
+
) -> list[Finding]:
|
|
172
|
+
repository = repository or repository_path()
|
|
173
|
+
environments = environments or environments_path()
|
|
174
|
+
findings = _validate_packages(repository) + _validate_environments(environments)
|
|
175
|
+
if config is not None:
|
|
176
|
+
findings.extend(_validate_resources(repository, config))
|
|
177
|
+
|
|
178
|
+
if online:
|
|
179
|
+
results = sync_many(
|
|
180
|
+
package_paths(repository),
|
|
181
|
+
repository=repository,
|
|
182
|
+
check=True,
|
|
183
|
+
config=config,
|
|
184
|
+
secrets=secrets,
|
|
185
|
+
)
|
|
186
|
+
for result in results:
|
|
187
|
+
if result.changed:
|
|
188
|
+
missing = ", ".join(result.missing_versions + result.missing_branches)
|
|
189
|
+
findings.append(Finding("error", f"sync:{result.package}", f"recipe is out of sync: {missing}"))
|
|
190
|
+
else:
|
|
191
|
+
findings.append(Finding("ok", f"sync:{result.package}", "recipe is synchronized with upstream"))
|
|
192
|
+
|
|
193
|
+
if compiler:
|
|
194
|
+
if spack_command is None:
|
|
195
|
+
raise ValueError("compiler validation requires isolated Spack runtime")
|
|
196
|
+
findings.extend(_validate_compiler(compiler, languages, spack_command, spack_env))
|
|
197
|
+
|
|
198
|
+
if with_spack:
|
|
199
|
+
if spack_command is None:
|
|
200
|
+
raise ValueError("Spack validation requires isolated Spack runtime")
|
|
201
|
+
findings.extend(_validate_with_spack(environments, spack_command, spack_env))
|
|
202
|
+
return findings
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# External resources are named here. Recipes and build-cache configuration refer
|
|
3
|
+
# to the resource name; credential values live only in the keyloom-encrypted
|
|
4
|
+
# secrets file.
|
|
5
|
+
|
|
6
|
+
# Private SEANERGYS Git resource. A recipe names the resource, never the
|
|
7
|
+
# credential; the token itself lives in the secrets file under the credential name
|
|
8
|
+
# below. Seanerstack authenticates `git ls-remote` with it, and sends it as
|
|
9
|
+
# the PRIVATE-TOKEN header when downloading a release archive to checksum it.
|
|
10
|
+
[resources.seanergys-gitlab]
|
|
11
|
+
kind = "git"
|
|
12
|
+
url = "https://gitlab.seanergys.fz-juelich.de/"
|
|
13
|
+
credentials = "seanergys-gitlab"
|
|
14
|
+
|
|
15
|
+
# Project binary build cache in the seanerstack GitLab container registry. Spack
|
|
16
|
+
# cannot sign OCI build caches, so registry access control is the trust boundary.
|
|
17
|
+
[resources.seanerstack-buildcache]
|
|
18
|
+
kind = "spack-buildcache"
|
|
19
|
+
url = "oci://registry.seanergys.fz-juelich.de/software/seanerstack/buildcache"
|
|
20
|
+
credentials = "seanergys-registry"
|
|
21
|
+
signed = false
|
|
22
|
+
|
|
23
|
+
# Example S3-backed Spack binary build cache:
|
|
24
|
+
# [resources.jsc-buildcache]
|
|
25
|
+
# kind = "spack-buildcache"
|
|
26
|
+
# url = "s3://seanerstack-buildcache"
|
|
27
|
+
# endpoint_url = "https://objectstore.example.org"
|
|
28
|
+
# credentials = "jsc-objectstore"
|
|
29
|
+
# autopush = true
|
|
30
|
+
# signed = false
|
|
31
|
+
|
|
32
|
+
# Example build cache in a directory on a local or shared filesystem. It takes
|
|
33
|
+
# no credentials and is unsigned unless signed = true is set:
|
|
34
|
+
# [resources.local-buildcache]
|
|
35
|
+
# kind = "spack-buildcache"
|
|
36
|
+
# url = "file:///scratch/seanerstack/buildcache"
|
|
37
|
+
# autopush = true
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
spack:
|
|
3
|
+
specs:
|
|
4
|
+
- demobus@main
|
|
5
|
+
|
|
6
|
+
# A source checkout is deliberately not committed here. After activating this
|
|
7
|
+
# environment, bind one with:
|
|
8
|
+
# spack develop --path /path/to/demobus demobus@main
|
|
9
|
+
concretizer:
|
|
10
|
+
unify: true
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
spack:
|
|
3
|
+
specs:
|
|
4
|
+
- flux-core
|
|
5
|
+
|
|
6
|
+
# A source checkout is deliberately not committed here. After activating this
|
|
7
|
+
# environment, bind one with:
|
|
8
|
+
# spack develop --path /path/to/flux-core flux-core@<dev_version>
|
|
9
|
+
concretizer:
|
|
10
|
+
unify: true
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
spack:
|
|
3
|
+
specs:
|
|
4
|
+
- properdocs@master
|
|
5
|
+
|
|
6
|
+
# A source checkout is deliberately not committed here. After activating this
|
|
7
|
+
# environment, bind one with:
|
|
8
|
+
# spack develop --path /path/to/properdocs properdocs@master
|
|
9
|
+
concretizer:
|
|
10
|
+
unify: true
|