seanerstack 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Seanerstack repository maintenance tooling."""
3
+
4
+ # The version is defined once, in pyproject.toml.
5
+ __version__ = __import__('importlib.metadata').metadata.version(__package__)
seanerstack/auth.py ADDED
@@ -0,0 +1,143 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ from __future__ import annotations
3
+
4
+ import base64
5
+ from contextlib import contextmanager
6
+ from dataclasses import dataclass
7
+ import os
8
+ from pathlib import Path
9
+ import re
10
+ import stat
11
+ import subprocess
12
+ import tempfile
13
+ from typing import Iterator, Mapping
14
+
15
+ from .config import Resource
16
+ from .secrets import SecretStore
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class GitAuth:
21
+ username: str
22
+ password: str
23
+ url: str
24
+ """The resource URL; credentials are sent only to URLs under it."""
25
+
26
+
27
+ def git_auth(resource: Resource, secrets: SecretStore) -> GitAuth | None:
28
+ if resource.credentials is None:
29
+ return None
30
+ values = secrets.credentials(resource.credentials)
31
+ username = values.get("username", "oauth2")
32
+ password = values.get("password", values.get("token"))
33
+ if not isinstance(username, str) or not isinstance(password, str):
34
+ raise ValueError(
35
+ f"credentials {resource.credentials!r} for {resource.name!r} require username/password or token"
36
+ )
37
+ return GitAuth(username=username, password=password, url=resource.url)
38
+
39
+
40
+ # GIT_CONFIG_COUNT, GIT_CONFIG_KEY_n and GIT_CONFIG_VALUE_n, which carry the
41
+ # credentials header, were introduced in Git 2.31. Older versions ignore them.
42
+ MINIMUM_GIT_VERSION = (2, 31)
43
+
44
+
45
+ def parse_git_version(output: str) -> tuple[int, int]:
46
+ """Return the major and minor version from `git --version` output."""
47
+ match = re.match(r"git version (\d+)\.(\d+)", output.strip())
48
+ if match is None:
49
+ raise RuntimeError(f"cannot determine the Git version from {output.strip()!r}")
50
+ return int(match.group(1)), int(match.group(2))
51
+
52
+
53
+ def git_version() -> tuple[int, int]:
54
+ process = subprocess.run(["git", "--version"], check=True, text=True, capture_output=True)
55
+ return parse_git_version(process.stdout)
56
+
57
+
58
+ def require_git_for_credentials(auth: GitAuth) -> None:
59
+ """Refuse to run Git with credentials it would not send."""
60
+ found = git_version()
61
+ if found < MINIMUM_GIT_VERSION:
62
+ required = ".".join(map(str, MINIMUM_GIT_VERSION))
63
+ raise RuntimeError(
64
+ f"Git {found[0]}.{found[1]} cannot send the credentials for {auth.url}; "
65
+ f"Git {required} or later is required"
66
+ )
67
+
68
+
69
+ @contextmanager
70
+ def git_environment(auth: GitAuth | None) -> Iterator[Mapping[str, str]]:
71
+ env = os.environ.copy()
72
+ if auth is None:
73
+ yield env
74
+ return
75
+
76
+ require_git_for_credentials(auth)
77
+ with tempfile.TemporaryDirectory(prefix="seanerstack-git-auth-") as temp:
78
+ helper = Path(temp) / "askpass"
79
+ helper.write_text(
80
+ "#!/bin/sh\n"
81
+ "case \"$1\" in\n"
82
+ " *sername*) printf '%s\\n' \"$SEANERSTACK_GIT_USERNAME\" ;;\n"
83
+ " *) printf '%s\\n' \"$SEANERSTACK_GIT_PASSWORD\" ;;\n"
84
+ "esac\n",
85
+ encoding="utf-8",
86
+ )
87
+ helper.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
88
+ # Git otherwise sends credentials only after a 401, which a server
89
+ # never answers for a public repository, so an invalid token would
90
+ # go unnoticed. The header is set through the environment rather than
91
+ # `git -c`, which would put it on the command line.
92
+ index = int(env.get("GIT_CONFIG_COUNT") or 0)
93
+ basic = base64.b64encode(f"{auth.username}:{auth.password}".encode("utf-8")).decode("ascii")
94
+ env.update(
95
+ {
96
+ f"GIT_CONFIG_KEY_{index}": f"http.{auth.url}.extraHeader",
97
+ f"GIT_CONFIG_VALUE_{index}": f"Authorization: Basic {basic}",
98
+ "GIT_CONFIG_COUNT": str(index + 1),
99
+ }
100
+ )
101
+ env.update(
102
+ {
103
+ "GIT_ASKPASS": str(helper),
104
+ "GIT_TERMINAL_PROMPT": "0",
105
+ "SEANERSTACK_GIT_USERNAME": auth.username,
106
+ "SEANERSTACK_GIT_PASSWORD": auth.password,
107
+ }
108
+ )
109
+ yield env
110
+
111
+
112
+ def http_auth_headers(auth: GitAuth | None) -> dict[str, str]:
113
+ """Headers authenticating an HTTPS download from a Git resource.
114
+
115
+ GitLab accepts a token as PRIVATE-TOKEN for repository archives; it does
116
+ not accept HTTP basic authentication there.
117
+ """
118
+ if auth is None:
119
+ return {}
120
+ return {"PRIVATE-TOKEN": auth.password}
121
+
122
+
123
+ def spack_resource_environment(resource: Resource, secrets: SecretStore) -> dict[str, str]:
124
+ if resource.credentials is None:
125
+ return {}
126
+ user_variable, secret_variable = resource.credential_variables
127
+ if resource.oci:
128
+ # A GitLab container registry takes the same user name and token as Git.
129
+ auth = git_auth(resource, secrets)
130
+ assert auth is not None
131
+ return {user_variable: auth.username, secret_variable: auth.password}
132
+ values = secrets.credentials(resource.credentials)
133
+ access_key = values.get("access_key_id", values.get("access_key"))
134
+ secret_key = values.get("secret_access_key", values.get("secret_key"))
135
+ token = values.get("session_token", values.get("access_token"))
136
+ if not isinstance(access_key, str) or not isinstance(secret_key, str):
137
+ raise ValueError(
138
+ f"credentials {resource.credentials!r} for {resource.name!r} require access_key_id and secret_access_key"
139
+ )
140
+ env = {user_variable: access_key, secret_variable: secret_key}
141
+ if isinstance(token, str):
142
+ env[f"{resource.env_prefix}_SESSION_TOKEN"] = token
143
+ return env
@@ -0,0 +1,161 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ from __future__ import annotations
3
+
4
+ from dataclasses import dataclass
5
+ import json
6
+ import re
7
+ import subprocess
8
+
9
+
10
+ DEFAULT_LANGUAGES = ("c", "cxx", "fortran")
11
+ DEFAULT_PROBE = "zlib"
12
+ DEFAULT_PROBE_TIMEOUT = 300
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class BootstrapResult:
17
+ compiler: str
18
+ installed: bool
19
+ registered: bool
20
+ languages: tuple[str, ...]
21
+ probe: str
22
+ probe_ok: bool
23
+
24
+
25
+ def _run(
26
+ args: list[str], *, check: bool = True, env=None, timeout: float | None = None
27
+ ) -> subprocess.CompletedProcess[str]:
28
+ return subprocess.run(
29
+ args, check=check, text=True, capture_output=True, env=env, timeout=timeout
30
+ )
31
+
32
+
33
+ def _run_streaming(args: list[str], *, env=None) -> None:
34
+ # Building a compiler takes upwards of an hour; capturing the output would
35
+ # make a long build indistinguishable from a hang.
36
+ subprocess.run(args, check=True, text=True, env=env)
37
+
38
+
39
+ def _compiler_prefix(command: list[str], compiler: str, env) -> str | None:
40
+ # Once a build has used the compiler, the database also holds an external
41
+ # record for the same prefix, so `location -i COMPILER` is ambiguous. Only a
42
+ # real installation counts.
43
+ result = _run([*command, "find", "--json", compiler], check=False, env=env)
44
+ if result.returncode != 0:
45
+ return None
46
+ hashes = [spec["hash"] for spec in json.loads(result.stdout or "[]") if not spec.get("external")]
47
+ if not hashes:
48
+ return None
49
+ if len(hashes) > 1:
50
+ raise RuntimeError(f"{compiler!r} matches {len(hashes)} installations; use a more specific spec")
51
+ result = _run([*command, "location", "-i", f"/{hashes[0]}"], check=False, env=env)
52
+ if result.returncode != 0:
53
+ return None
54
+ return result.stdout.strip() or None
55
+
56
+
57
+ def _ensure_system_compiler(command: list[str], env) -> None:
58
+ """Register whatever compiler is visible on PATH as a Spack external.
59
+
60
+ Seanerstack's isolated packages.yaml starts out empty, so on a fresh state
61
+ directory nothing provides the `c`/`cxx` virtuals that building the
62
+ project's own compiler itself requires: concretizing `install COMPILER`
63
+ fails outright with "Only external, or concrete, compilers are allowed".
64
+ A no-op when a compiler is already registered.
65
+ """
66
+ _run([*command, "compiler", "find"], check=False, env=env)
67
+
68
+
69
+ def _register_compiler(command: list[str], prefix: str, env) -> None:
70
+ """Register an installed compiler as an external package.
71
+
72
+ Must run before the language requirements are written. Those apply to
73
+ every node needing `c`/`cxx`/`fortran`, including the ones in the
74
+ compiler's own build tree, which the system compiler built, so they forbid
75
+ reusing the compiler just installed and leave no solution. An external node
76
+ has no build dependencies and so contradicts nothing.
77
+ """
78
+ _run([*command, "compiler", "find", prefix], env=env)
79
+
80
+
81
+ def _compiler_name(compiler: str) -> str:
82
+ match = re.match(r"([A-Za-z0-9_.+-]+)", compiler)
83
+ if not match:
84
+ raise ValueError(f"cannot determine compiler package name from {compiler!r}")
85
+ return match.group(1)
86
+
87
+
88
+ def _set_language_requirement(command: list[str], language: str, compiler: str, env) -> None:
89
+ # Remove an earlier Seanerstack-style single requirement before adding the new
90
+ # one. A missing path is harmless and is intentionally ignored.
91
+ _run(
92
+ [*command, "config", "remove", f"packages:{language}:require"],
93
+ check=False, env=env,
94
+ )
95
+ _run(
96
+ [*command, "config", "add", f"packages:{language}:require:{compiler}"], env=env
97
+ )
98
+
99
+
100
+ def bootstrap(
101
+ compiler: str,
102
+ *,
103
+ languages: tuple[str, ...] = DEFAULT_LANGUAGES,
104
+ probe: str = DEFAULT_PROBE,
105
+ command: list[str] | None = None,
106
+ env=None,
107
+ force: bool = False,
108
+ dry_run: bool = False,
109
+ probe_timeout: float | None = DEFAULT_PROBE_TIMEOUT,
110
+ ) -> BootstrapResult:
111
+ command = command or ["spack"]
112
+ if not languages:
113
+ raise ValueError("at least one compiler language is required")
114
+
115
+ prefix = _compiler_prefix(command, compiler, env)
116
+ already_installed = prefix is not None
117
+ if (force or not already_installed) and not dry_run:
118
+ _ensure_system_compiler(command, env)
119
+ _run_streaming([*command, "install", compiler], env=env)
120
+ prefix = _compiler_prefix(command, compiler, env)
121
+
122
+ registered = False
123
+ if not dry_run:
124
+ if prefix is None:
125
+ raise RuntimeError(
126
+ f"Spack reports no installation prefix for {compiler!r}; "
127
+ "cannot register it as a compiler"
128
+ )
129
+ _register_compiler(command, prefix, env)
130
+ registered = True
131
+ for language in languages:
132
+ _set_language_requirement(command, language, compiler, env)
133
+
134
+ probe_ok = False
135
+ if not dry_run:
136
+ try:
137
+ probe_result = _run(
138
+ [*command, "spec", probe], check=False, env=env, timeout=probe_timeout
139
+ )
140
+ except subprocess.TimeoutExpired as error:
141
+ raise RuntimeError(
142
+ f"Spack did not concretize compiler probe {probe!r} within "
143
+ f"{probe_timeout:g} seconds. This usually means the {compiler} "
144
+ "requirement cannot be satisfied: check that `spack compiler list` "
145
+ "reports it and that `spack config get packages` lists it under "
146
+ "`gcc: externals`."
147
+ ) from error
148
+ if probe_result.returncode != 0:
149
+ message = probe_result.stderr.strip() or probe_result.stdout.strip()
150
+ raise RuntimeError(f"Spack could not concretize compiler probe {probe!r}:\n{message}")
151
+ compiler_name = _compiler_name(compiler)
152
+ probe_ok = compiler_name in (probe_result.stdout + probe_result.stderr)
153
+
154
+ return BootstrapResult(
155
+ compiler=compiler,
156
+ installed=already_installed or not dry_run,
157
+ registered=registered,
158
+ languages=languages,
159
+ probe=probe,
160
+ probe_ok=probe_ok,
161
+ )
@@ -0,0 +1,68 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ from __future__ import annotations
3
+
4
+ import os
5
+ import subprocess
6
+
7
+ from .auth import spack_resource_environment
8
+ from .config import Config, Resource
9
+ from .secrets import SecretStore
10
+
11
+
12
+ def buildcaches(config: Config) -> tuple[Resource, ...]:
13
+ return tuple(resource for resource in config.resources.values() if resource.kind == "spack-buildcache")
14
+
15
+
16
+ def _mirror_args(resource: Resource, command: list[str]) -> list[str]:
17
+ args = [*command, "mirror", "add", "--scope", "user", "--type", "binary"]
18
+ if resource.autopush:
19
+ args.append("--autopush")
20
+ args.append("--signed" if resource.signed else "--unsigned")
21
+ if resource.endpoint_url:
22
+ args.extend(["--s3-endpoint-url", resource.endpoint_url])
23
+ if resource.credentials:
24
+ user_variable, secret_variable = resource.credential_variables
25
+ if resource.oci:
26
+ args.extend(["--oci-username-variable", user_variable, "--oci-password-variable", secret_variable])
27
+ else:
28
+ args.extend(["--s3-access-key-id-variable", user_variable, "--s3-access-key-secret-variable", secret_variable])
29
+ # The session-token variable is configured only when a token is present at runtime.
30
+ args.extend([resource.name, resource.url])
31
+ return args
32
+
33
+
34
+ def configure_buildcaches(config: Config, *, command: list[str] | None = None, env=None) -> None:
35
+ command = command or ["spack"]
36
+ for resource in buildcaches(config):
37
+ # `-C` also loads the private configuration as an immutable scope, and an
38
+ # unscoped remove picks that one and fails. Name the writable scope.
39
+ subprocess.run(
40
+ [*command, "mirror", "remove", "--scope", "user", resource.name],
41
+ check=False,
42
+ text=True,
43
+ capture_output=True, env=env,
44
+ )
45
+ process = subprocess.run(_mirror_args(resource, command), check=False, text=True, capture_output=True, env=env)
46
+ if process.returncode != 0:
47
+ message = process.stderr.strip() or process.stdout.strip()
48
+ raise RuntimeError(f"could not configure build cache {resource.name!r}: {message}")
49
+
50
+
51
+ def check_buildcaches(config: Config, secrets: SecretStore, *, command: list[str] | None = None, base_env=None) -> list[tuple[str, bool, str]]:
52
+ command = command or ["spack"]
53
+ results: list[tuple[str, bool, str]] = []
54
+ for resource in buildcaches(config):
55
+ env = dict(os.environ if base_env is None else base_env)
56
+ # An activated session already carries the credentials; only then is the vault skipped.
57
+ if not all(name in env for name in resource.credential_variables):
58
+ env.update(spack_resource_environment(resource, secrets))
59
+ process = subprocess.run(
60
+ [*command, "buildcache", "list", "--allarch"],
61
+ check=False,
62
+ text=True,
63
+ capture_output=True,
64
+ env=env,
65
+ )
66
+ message = process.stderr.strip() or process.stdout.strip()
67
+ results.append((resource.name, process.returncode == 0, message))
68
+ return results