seanerstack 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seanerstack/__init__.py +5 -0
- seanerstack/auth.py +143 -0
- seanerstack/bootstrap.py +161 -0
- seanerstack/buildcache.py +68 -0
- seanerstack/cli.py +397 -0
- seanerstack/config.py +93 -0
- seanerstack/docsgen.py +146 -0
- seanerstack/isolation.py +243 -0
- seanerstack/paths.py +52 -0
- seanerstack/recipe.py +108 -0
- seanerstack/secrets.py +60 -0
- seanerstack/sync.py +381 -0
- seanerstack/validate.py +202 -0
- seanerstack-0.1.3.data/data/share/seanerstack/config.toml +37 -0
- seanerstack-0.1.3.data/data/share/seanerstack/environments/demobus-dev/spack.yaml +10 -0
- seanerstack-0.1.3.data/data/share/seanerstack/environments/flux-core-dev/spack.yaml +10 -0
- seanerstack-0.1.3.data/data/share/seanerstack/environments/properdocs-dev/spack.yaml +10 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack-repo-index.yaml +4 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack_repo/seanerstack/packages/demobus/package.py +38 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack_repo/seanerstack/packages/properdocs/package.py +34 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack_repo/seanerstack/packages/seanergys_meta/package.py +13 -0
- seanerstack-0.1.3.data/data/share/seanerstack/spack_repo/seanerstack/repo.yaml +4 -0
- seanerstack-0.1.3.dist-info/METADATA +224 -0
- seanerstack-0.1.3.dist-info/RECORD +27 -0
- seanerstack-0.1.3.dist-info/WHEEL +4 -0
- seanerstack-0.1.3.dist-info/entry_points.txt +3 -0
- seanerstack-0.1.3.dist-info/licenses/LICENSE +202 -0
seanerstack/__init__.py
ADDED
seanerstack/auth.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import base64
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import re
|
|
10
|
+
import stat
|
|
11
|
+
import subprocess
|
|
12
|
+
import tempfile
|
|
13
|
+
from typing import Iterator, Mapping
|
|
14
|
+
|
|
15
|
+
from .config import Resource
|
|
16
|
+
from .secrets import SecretStore
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class GitAuth:
|
|
21
|
+
username: str
|
|
22
|
+
password: str
|
|
23
|
+
url: str
|
|
24
|
+
"""The resource URL; credentials are sent only to URLs under it."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def git_auth(resource: Resource, secrets: SecretStore) -> GitAuth | None:
|
|
28
|
+
if resource.credentials is None:
|
|
29
|
+
return None
|
|
30
|
+
values = secrets.credentials(resource.credentials)
|
|
31
|
+
username = values.get("username", "oauth2")
|
|
32
|
+
password = values.get("password", values.get("token"))
|
|
33
|
+
if not isinstance(username, str) or not isinstance(password, str):
|
|
34
|
+
raise ValueError(
|
|
35
|
+
f"credentials {resource.credentials!r} for {resource.name!r} require username/password or token"
|
|
36
|
+
)
|
|
37
|
+
return GitAuth(username=username, password=password, url=resource.url)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# GIT_CONFIG_COUNT, GIT_CONFIG_KEY_n and GIT_CONFIG_VALUE_n, which carry the
|
|
41
|
+
# credentials header, were introduced in Git 2.31. Older versions ignore them.
|
|
42
|
+
MINIMUM_GIT_VERSION = (2, 31)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def parse_git_version(output: str) -> tuple[int, int]:
|
|
46
|
+
"""Return the major and minor version from `git --version` output."""
|
|
47
|
+
match = re.match(r"git version (\d+)\.(\d+)", output.strip())
|
|
48
|
+
if match is None:
|
|
49
|
+
raise RuntimeError(f"cannot determine the Git version from {output.strip()!r}")
|
|
50
|
+
return int(match.group(1)), int(match.group(2))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def git_version() -> tuple[int, int]:
|
|
54
|
+
process = subprocess.run(["git", "--version"], check=True, text=True, capture_output=True)
|
|
55
|
+
return parse_git_version(process.stdout)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def require_git_for_credentials(auth: GitAuth) -> None:
|
|
59
|
+
"""Refuse to run Git with credentials it would not send."""
|
|
60
|
+
found = git_version()
|
|
61
|
+
if found < MINIMUM_GIT_VERSION:
|
|
62
|
+
required = ".".join(map(str, MINIMUM_GIT_VERSION))
|
|
63
|
+
raise RuntimeError(
|
|
64
|
+
f"Git {found[0]}.{found[1]} cannot send the credentials for {auth.url}; "
|
|
65
|
+
f"Git {required} or later is required"
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@contextmanager
|
|
70
|
+
def git_environment(auth: GitAuth | None) -> Iterator[Mapping[str, str]]:
|
|
71
|
+
env = os.environ.copy()
|
|
72
|
+
if auth is None:
|
|
73
|
+
yield env
|
|
74
|
+
return
|
|
75
|
+
|
|
76
|
+
require_git_for_credentials(auth)
|
|
77
|
+
with tempfile.TemporaryDirectory(prefix="seanerstack-git-auth-") as temp:
|
|
78
|
+
helper = Path(temp) / "askpass"
|
|
79
|
+
helper.write_text(
|
|
80
|
+
"#!/bin/sh\n"
|
|
81
|
+
"case \"$1\" in\n"
|
|
82
|
+
" *sername*) printf '%s\\n' \"$SEANERSTACK_GIT_USERNAME\" ;;\n"
|
|
83
|
+
" *) printf '%s\\n' \"$SEANERSTACK_GIT_PASSWORD\" ;;\n"
|
|
84
|
+
"esac\n",
|
|
85
|
+
encoding="utf-8",
|
|
86
|
+
)
|
|
87
|
+
helper.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
|
|
88
|
+
# Git otherwise sends credentials only after a 401, which a server
|
|
89
|
+
# never answers for a public repository, so an invalid token would
|
|
90
|
+
# go unnoticed. The header is set through the environment rather than
|
|
91
|
+
# `git -c`, which would put it on the command line.
|
|
92
|
+
index = int(env.get("GIT_CONFIG_COUNT") or 0)
|
|
93
|
+
basic = base64.b64encode(f"{auth.username}:{auth.password}".encode("utf-8")).decode("ascii")
|
|
94
|
+
env.update(
|
|
95
|
+
{
|
|
96
|
+
f"GIT_CONFIG_KEY_{index}": f"http.{auth.url}.extraHeader",
|
|
97
|
+
f"GIT_CONFIG_VALUE_{index}": f"Authorization: Basic {basic}",
|
|
98
|
+
"GIT_CONFIG_COUNT": str(index + 1),
|
|
99
|
+
}
|
|
100
|
+
)
|
|
101
|
+
env.update(
|
|
102
|
+
{
|
|
103
|
+
"GIT_ASKPASS": str(helper),
|
|
104
|
+
"GIT_TERMINAL_PROMPT": "0",
|
|
105
|
+
"SEANERSTACK_GIT_USERNAME": auth.username,
|
|
106
|
+
"SEANERSTACK_GIT_PASSWORD": auth.password,
|
|
107
|
+
}
|
|
108
|
+
)
|
|
109
|
+
yield env
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def http_auth_headers(auth: GitAuth | None) -> dict[str, str]:
|
|
113
|
+
"""Headers authenticating an HTTPS download from a Git resource.
|
|
114
|
+
|
|
115
|
+
GitLab accepts a token as PRIVATE-TOKEN for repository archives; it does
|
|
116
|
+
not accept HTTP basic authentication there.
|
|
117
|
+
"""
|
|
118
|
+
if auth is None:
|
|
119
|
+
return {}
|
|
120
|
+
return {"PRIVATE-TOKEN": auth.password}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def spack_resource_environment(resource: Resource, secrets: SecretStore) -> dict[str, str]:
|
|
124
|
+
if resource.credentials is None:
|
|
125
|
+
return {}
|
|
126
|
+
user_variable, secret_variable = resource.credential_variables
|
|
127
|
+
if resource.oci:
|
|
128
|
+
# A GitLab container registry takes the same user name and token as Git.
|
|
129
|
+
auth = git_auth(resource, secrets)
|
|
130
|
+
assert auth is not None
|
|
131
|
+
return {user_variable: auth.username, secret_variable: auth.password}
|
|
132
|
+
values = secrets.credentials(resource.credentials)
|
|
133
|
+
access_key = values.get("access_key_id", values.get("access_key"))
|
|
134
|
+
secret_key = values.get("secret_access_key", values.get("secret_key"))
|
|
135
|
+
token = values.get("session_token", values.get("access_token"))
|
|
136
|
+
if not isinstance(access_key, str) or not isinstance(secret_key, str):
|
|
137
|
+
raise ValueError(
|
|
138
|
+
f"credentials {resource.credentials!r} for {resource.name!r} require access_key_id and secret_access_key"
|
|
139
|
+
)
|
|
140
|
+
env = {user_variable: access_key, secret_variable: secret_key}
|
|
141
|
+
if isinstance(token, str):
|
|
142
|
+
env[f"{resource.env_prefix}_SESSION_TOKEN"] = token
|
|
143
|
+
return env
|
seanerstack/bootstrap.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import subprocess
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
DEFAULT_LANGUAGES = ("c", "cxx", "fortran")
|
|
11
|
+
DEFAULT_PROBE = "zlib"
|
|
12
|
+
DEFAULT_PROBE_TIMEOUT = 300
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class BootstrapResult:
|
|
17
|
+
compiler: str
|
|
18
|
+
installed: bool
|
|
19
|
+
registered: bool
|
|
20
|
+
languages: tuple[str, ...]
|
|
21
|
+
probe: str
|
|
22
|
+
probe_ok: bool
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _run(
|
|
26
|
+
args: list[str], *, check: bool = True, env=None, timeout: float | None = None
|
|
27
|
+
) -> subprocess.CompletedProcess[str]:
|
|
28
|
+
return subprocess.run(
|
|
29
|
+
args, check=check, text=True, capture_output=True, env=env, timeout=timeout
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _run_streaming(args: list[str], *, env=None) -> None:
|
|
34
|
+
# Building a compiler takes upwards of an hour; capturing the output would
|
|
35
|
+
# make a long build indistinguishable from a hang.
|
|
36
|
+
subprocess.run(args, check=True, text=True, env=env)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _compiler_prefix(command: list[str], compiler: str, env) -> str | None:
|
|
40
|
+
# Once a build has used the compiler, the database also holds an external
|
|
41
|
+
# record for the same prefix, so `location -i COMPILER` is ambiguous. Only a
|
|
42
|
+
# real installation counts.
|
|
43
|
+
result = _run([*command, "find", "--json", compiler], check=False, env=env)
|
|
44
|
+
if result.returncode != 0:
|
|
45
|
+
return None
|
|
46
|
+
hashes = [spec["hash"] for spec in json.loads(result.stdout or "[]") if not spec.get("external")]
|
|
47
|
+
if not hashes:
|
|
48
|
+
return None
|
|
49
|
+
if len(hashes) > 1:
|
|
50
|
+
raise RuntimeError(f"{compiler!r} matches {len(hashes)} installations; use a more specific spec")
|
|
51
|
+
result = _run([*command, "location", "-i", f"/{hashes[0]}"], check=False, env=env)
|
|
52
|
+
if result.returncode != 0:
|
|
53
|
+
return None
|
|
54
|
+
return result.stdout.strip() or None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _ensure_system_compiler(command: list[str], env) -> None:
|
|
58
|
+
"""Register whatever compiler is visible on PATH as a Spack external.
|
|
59
|
+
|
|
60
|
+
Seanerstack's isolated packages.yaml starts out empty, so on a fresh state
|
|
61
|
+
directory nothing provides the `c`/`cxx` virtuals that building the
|
|
62
|
+
project's own compiler itself requires: concretizing `install COMPILER`
|
|
63
|
+
fails outright with "Only external, or concrete, compilers are allowed".
|
|
64
|
+
A no-op when a compiler is already registered.
|
|
65
|
+
"""
|
|
66
|
+
_run([*command, "compiler", "find"], check=False, env=env)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _register_compiler(command: list[str], prefix: str, env) -> None:
|
|
70
|
+
"""Register an installed compiler as an external package.
|
|
71
|
+
|
|
72
|
+
Must run before the language requirements are written. Those apply to
|
|
73
|
+
every node needing `c`/`cxx`/`fortran`, including the ones in the
|
|
74
|
+
compiler's own build tree, which the system compiler built, so they forbid
|
|
75
|
+
reusing the compiler just installed and leave no solution. An external node
|
|
76
|
+
has no build dependencies and so contradicts nothing.
|
|
77
|
+
"""
|
|
78
|
+
_run([*command, "compiler", "find", prefix], env=env)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _compiler_name(compiler: str) -> str:
|
|
82
|
+
match = re.match(r"([A-Za-z0-9_.+-]+)", compiler)
|
|
83
|
+
if not match:
|
|
84
|
+
raise ValueError(f"cannot determine compiler package name from {compiler!r}")
|
|
85
|
+
return match.group(1)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _set_language_requirement(command: list[str], language: str, compiler: str, env) -> None:
|
|
89
|
+
# Remove an earlier Seanerstack-style single requirement before adding the new
|
|
90
|
+
# one. A missing path is harmless and is intentionally ignored.
|
|
91
|
+
_run(
|
|
92
|
+
[*command, "config", "remove", f"packages:{language}:require"],
|
|
93
|
+
check=False, env=env,
|
|
94
|
+
)
|
|
95
|
+
_run(
|
|
96
|
+
[*command, "config", "add", f"packages:{language}:require:{compiler}"], env=env
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def bootstrap(
|
|
101
|
+
compiler: str,
|
|
102
|
+
*,
|
|
103
|
+
languages: tuple[str, ...] = DEFAULT_LANGUAGES,
|
|
104
|
+
probe: str = DEFAULT_PROBE,
|
|
105
|
+
command: list[str] | None = None,
|
|
106
|
+
env=None,
|
|
107
|
+
force: bool = False,
|
|
108
|
+
dry_run: bool = False,
|
|
109
|
+
probe_timeout: float | None = DEFAULT_PROBE_TIMEOUT,
|
|
110
|
+
) -> BootstrapResult:
|
|
111
|
+
command = command or ["spack"]
|
|
112
|
+
if not languages:
|
|
113
|
+
raise ValueError("at least one compiler language is required")
|
|
114
|
+
|
|
115
|
+
prefix = _compiler_prefix(command, compiler, env)
|
|
116
|
+
already_installed = prefix is not None
|
|
117
|
+
if (force or not already_installed) and not dry_run:
|
|
118
|
+
_ensure_system_compiler(command, env)
|
|
119
|
+
_run_streaming([*command, "install", compiler], env=env)
|
|
120
|
+
prefix = _compiler_prefix(command, compiler, env)
|
|
121
|
+
|
|
122
|
+
registered = False
|
|
123
|
+
if not dry_run:
|
|
124
|
+
if prefix is None:
|
|
125
|
+
raise RuntimeError(
|
|
126
|
+
f"Spack reports no installation prefix for {compiler!r}; "
|
|
127
|
+
"cannot register it as a compiler"
|
|
128
|
+
)
|
|
129
|
+
_register_compiler(command, prefix, env)
|
|
130
|
+
registered = True
|
|
131
|
+
for language in languages:
|
|
132
|
+
_set_language_requirement(command, language, compiler, env)
|
|
133
|
+
|
|
134
|
+
probe_ok = False
|
|
135
|
+
if not dry_run:
|
|
136
|
+
try:
|
|
137
|
+
probe_result = _run(
|
|
138
|
+
[*command, "spec", probe], check=False, env=env, timeout=probe_timeout
|
|
139
|
+
)
|
|
140
|
+
except subprocess.TimeoutExpired as error:
|
|
141
|
+
raise RuntimeError(
|
|
142
|
+
f"Spack did not concretize compiler probe {probe!r} within "
|
|
143
|
+
f"{probe_timeout:g} seconds. This usually means the {compiler} "
|
|
144
|
+
"requirement cannot be satisfied: check that `spack compiler list` "
|
|
145
|
+
"reports it and that `spack config get packages` lists it under "
|
|
146
|
+
"`gcc: externals`."
|
|
147
|
+
) from error
|
|
148
|
+
if probe_result.returncode != 0:
|
|
149
|
+
message = probe_result.stderr.strip() or probe_result.stdout.strip()
|
|
150
|
+
raise RuntimeError(f"Spack could not concretize compiler probe {probe!r}:\n{message}")
|
|
151
|
+
compiler_name = _compiler_name(compiler)
|
|
152
|
+
probe_ok = compiler_name in (probe_result.stdout + probe_result.stderr)
|
|
153
|
+
|
|
154
|
+
return BootstrapResult(
|
|
155
|
+
compiler=compiler,
|
|
156
|
+
installed=already_installed or not dry_run,
|
|
157
|
+
registered=registered,
|
|
158
|
+
languages=languages,
|
|
159
|
+
probe=probe,
|
|
160
|
+
probe_ok=probe_ok,
|
|
161
|
+
)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
import subprocess
|
|
6
|
+
|
|
7
|
+
from .auth import spack_resource_environment
|
|
8
|
+
from .config import Config, Resource
|
|
9
|
+
from .secrets import SecretStore
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def buildcaches(config: Config) -> tuple[Resource, ...]:
|
|
13
|
+
return tuple(resource for resource in config.resources.values() if resource.kind == "spack-buildcache")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _mirror_args(resource: Resource, command: list[str]) -> list[str]:
|
|
17
|
+
args = [*command, "mirror", "add", "--scope", "user", "--type", "binary"]
|
|
18
|
+
if resource.autopush:
|
|
19
|
+
args.append("--autopush")
|
|
20
|
+
args.append("--signed" if resource.signed else "--unsigned")
|
|
21
|
+
if resource.endpoint_url:
|
|
22
|
+
args.extend(["--s3-endpoint-url", resource.endpoint_url])
|
|
23
|
+
if resource.credentials:
|
|
24
|
+
user_variable, secret_variable = resource.credential_variables
|
|
25
|
+
if resource.oci:
|
|
26
|
+
args.extend(["--oci-username-variable", user_variable, "--oci-password-variable", secret_variable])
|
|
27
|
+
else:
|
|
28
|
+
args.extend(["--s3-access-key-id-variable", user_variable, "--s3-access-key-secret-variable", secret_variable])
|
|
29
|
+
# The session-token variable is configured only when a token is present at runtime.
|
|
30
|
+
args.extend([resource.name, resource.url])
|
|
31
|
+
return args
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def configure_buildcaches(config: Config, *, command: list[str] | None = None, env=None) -> None:
|
|
35
|
+
command = command or ["spack"]
|
|
36
|
+
for resource in buildcaches(config):
|
|
37
|
+
# `-C` also loads the private configuration as an immutable scope, and an
|
|
38
|
+
# unscoped remove picks that one and fails. Name the writable scope.
|
|
39
|
+
subprocess.run(
|
|
40
|
+
[*command, "mirror", "remove", "--scope", "user", resource.name],
|
|
41
|
+
check=False,
|
|
42
|
+
text=True,
|
|
43
|
+
capture_output=True, env=env,
|
|
44
|
+
)
|
|
45
|
+
process = subprocess.run(_mirror_args(resource, command), check=False, text=True, capture_output=True, env=env)
|
|
46
|
+
if process.returncode != 0:
|
|
47
|
+
message = process.stderr.strip() or process.stdout.strip()
|
|
48
|
+
raise RuntimeError(f"could not configure build cache {resource.name!r}: {message}")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def check_buildcaches(config: Config, secrets: SecretStore, *, command: list[str] | None = None, base_env=None) -> list[tuple[str, bool, str]]:
|
|
52
|
+
command = command or ["spack"]
|
|
53
|
+
results: list[tuple[str, bool, str]] = []
|
|
54
|
+
for resource in buildcaches(config):
|
|
55
|
+
env = dict(os.environ if base_env is None else base_env)
|
|
56
|
+
# An activated session already carries the credentials; only then is the vault skipped.
|
|
57
|
+
if not all(name in env for name in resource.credential_variables):
|
|
58
|
+
env.update(spack_resource_environment(resource, secrets))
|
|
59
|
+
process = subprocess.run(
|
|
60
|
+
[*command, "buildcache", "list", "--allarch"],
|
|
61
|
+
check=False,
|
|
62
|
+
text=True,
|
|
63
|
+
capture_output=True,
|
|
64
|
+
env=env,
|
|
65
|
+
)
|
|
66
|
+
message = process.stderr.strip() or process.stdout.strip()
|
|
67
|
+
results.append((resource.name, process.returncode == 0, message))
|
|
68
|
+
return results
|