clusterhop 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clusterhop/__init__.py +20 -0
- clusterhop/api.py +282 -0
- clusterhop/cli/__init__.py +188 -0
- clusterhop/github2lab/__init__.py +6 -0
- clusterhop/github2lab/cancellation.py +178 -0
- clusterhop/github2lab/config.py +351 -0
- clusterhop/github2lab/fallback.py +178 -0
- clusterhop/github2lab/git.py +332 -0
- clusterhop/github2lab/gitlab.py +454 -0
- clusterhop/github2lab/logs.py +151 -0
- clusterhop/github2lab/machine.py +32 -0
- clusterhop/github2lab/models.py +33 -0
- clusterhop/github2lab/project.py +111 -0
- clusterhop/github2lab/runner.py +590 -0
- clusterhop/github2lab/state.py +250 -0
- clusterhop/github2lab/summary.py +216 -0
- clusterhop/github2lab/urls.py +191 -0
- clusterhop/github_action.py +383 -0
- clusterhop/metadata.py +6 -0
- clusterhop-0.2.0.data/data/share/.gitkeep +0 -0
- clusterhop-0.2.0.dist-info/METADATA +61 -0
- clusterhop-0.2.0.dist-info/RECORD +25 -0
- clusterhop-0.2.0.dist-info/WHEEL +4 -0
- clusterhop-0.2.0.dist-info/entry_points.txt +2 -0
- clusterhop-0.2.0.dist-info/licenses/LICENSE +201 -0
clusterhop/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# --------------------------------------------------------------------------------------------------
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
# Copyright (C) 2026 Jayesh Badwaik <j.badwaik@fz-juelich.de>
|
|
4
|
+
# --------------------------------------------------------------------------------------------------
|
|
5
|
+
|
|
6
|
+
"""Public ClusterHop Python API."""
|
|
7
|
+
|
|
8
|
+
from clusterhop.api import (
|
|
9
|
+
RunConfigurationError,
|
|
10
|
+
RunResult,
|
|
11
|
+
UnsupportedDestinationError,
|
|
12
|
+
run,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"RunConfigurationError",
|
|
17
|
+
"RunResult",
|
|
18
|
+
"UnsupportedDestinationError",
|
|
19
|
+
"run",
|
|
20
|
+
]
|
clusterhop/api.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
# --------------------------------------------------------------------------------------------------
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
# Copyright (C) 2026 Jayesh Badwaik <j.badwaik@fz-juelich.de>
|
|
4
|
+
# --------------------------------------------------------------------------------------------------
|
|
5
|
+
|
|
6
|
+
"""Public Python API for ClusterHop runs."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Literal
|
|
16
|
+
|
|
17
|
+
from clusterhop.github2lab.config import (
|
|
18
|
+
GITLAB_VARIABLES_ENVIRONMENT_NAME,
|
|
19
|
+
ConfigurationError,
|
|
20
|
+
validate_extra_pipeline_variables,
|
|
21
|
+
validate_environment_before_project_resolution,
|
|
22
|
+
)
|
|
23
|
+
from clusterhop.github2lab.machine import MachineNameError, jsc_machine_pipeline_variable
|
|
24
|
+
from clusterhop.github2lab.project import (
|
|
25
|
+
GitLabProjectResolutionError,
|
|
26
|
+
resolve_gitlab_project,
|
|
27
|
+
)
|
|
28
|
+
from clusterhop.github2lab.runner import run_from_environment
|
|
29
|
+
|
|
30
|
+
RunStatus = Literal["success", "failed", "canceled", "skipped", "timeout", "error"]
|
|
31
|
+
InputObject = Mapping[str, object] | object
|
|
32
|
+
|
|
33
|
+
_MISSING = object()
|
|
34
|
+
_CONFIG_ENVIRONMENT_FIELDS = {
|
|
35
|
+
"gitlab_url": "GITLAB_URL",
|
|
36
|
+
"gitlab_remote_url": "GITLAB_REMOTE_URL",
|
|
37
|
+
"gitlab_project_id": "GITLAB_PROJECT_ID",
|
|
38
|
+
"gitlab_project_path": "GITLAB_PROJECT_PATH",
|
|
39
|
+
"github_commit_sha": "GITHUB_SHA",
|
|
40
|
+
"github_repository": "GITHUB_REPOSITORY",
|
|
41
|
+
"github_run_id": "GITHUB_RUN_ID",
|
|
42
|
+
}
|
|
43
|
+
_OPTION_ENVIRONMENT_FIELDS = {
|
|
44
|
+
"poll_interval_seconds": "CLUSTERHOP_POLL_INTERVAL_SECONDS",
|
|
45
|
+
"pipeline_timeout_seconds": "CLUSTERHOP_PIPELINE_TIMEOUT_SECONDS",
|
|
46
|
+
"pipeline_state_file": "CLUSTERHOP_PIPELINE_STATE_FILE",
|
|
47
|
+
"cleanup_timeout_seconds": "CLUSTERHOP_CLEANUP_TIMEOUT_SECONDS",
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class RunConfigurationError(ValueError):
|
|
52
|
+
"""Raised when public run inputs are missing or invalid."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class UnsupportedDestinationError(RunConfigurationError):
|
|
56
|
+
"""Raised when no implementation exists for the requested destination type."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class RunResult:
|
|
61
|
+
"""Stable, destination-neutral result returned by :func:`clusterhop.run`."""
|
|
62
|
+
|
|
63
|
+
status: RunStatus
|
|
64
|
+
exit_code: int
|
|
65
|
+
summary: str = field(repr=False)
|
|
66
|
+
error_message: str | None = field(default=None, repr=False)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def run(
|
|
70
|
+
*,
|
|
71
|
+
config: InputObject,
|
|
72
|
+
repo_path: str | Path,
|
|
73
|
+
options: InputObject | None = None,
|
|
74
|
+
secrets: InputObject | None = None,
|
|
75
|
+
) -> RunResult:
|
|
76
|
+
"""Run ClusterHop against an already-loaded destination configuration.
|
|
77
|
+
|
|
78
|
+
This initial public adapter supports only the existing GitLab destination. Configuration,
|
|
79
|
+
options, and secrets are Python mappings or objects with matching attributes; loading files
|
|
80
|
+
and decrypting secrets intentionally remain outside this function.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
destination_type = _destination_type(config)
|
|
84
|
+
if destination_type.casefold() != "gitlab":
|
|
85
|
+
raise UnsupportedDestinationError(
|
|
86
|
+
"The requested destination type is not supported by this ClusterHop version."
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
try:
|
|
90
|
+
environment = _prepare_gitlab_environment(
|
|
91
|
+
config,
|
|
92
|
+
options,
|
|
93
|
+
secrets,
|
|
94
|
+
validate_run_inputs=True,
|
|
95
|
+
)
|
|
96
|
+
internal_result = run_from_environment(
|
|
97
|
+
environment,
|
|
98
|
+
repository_path=repo_path,
|
|
99
|
+
)
|
|
100
|
+
except ConfigurationError as error:
|
|
101
|
+
raise RunConfigurationError(str(error)) from None
|
|
102
|
+
|
|
103
|
+
return RunResult(
|
|
104
|
+
status=internal_result.status,
|
|
105
|
+
exit_code=internal_result.exit_code,
|
|
106
|
+
summary=internal_result.summary,
|
|
107
|
+
error_message=internal_result.error_message,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _prepare_gitlab_environment(
|
|
112
|
+
config: InputObject,
|
|
113
|
+
options: InputObject | None,
|
|
114
|
+
secrets: InputObject | None,
|
|
115
|
+
*,
|
|
116
|
+
project_resolution_timeout_seconds: float | None = None,
|
|
117
|
+
validate_run_inputs: bool = False,
|
|
118
|
+
) -> dict[str, str]:
|
|
119
|
+
"""Adapt public GitLab inputs into the existing validated environment contract."""
|
|
120
|
+
|
|
121
|
+
environment = _github_metadata_from_environment()
|
|
122
|
+
_copy_config_fields(config, environment)
|
|
123
|
+
_copy_secret_fields(secrets, environment)
|
|
124
|
+
_copy_option_fields(options, environment)
|
|
125
|
+
if validate_run_inputs:
|
|
126
|
+
validate_environment_before_project_resolution(environment)
|
|
127
|
+
_resolve_project_if_remote_is_missing(
|
|
128
|
+
environment,
|
|
129
|
+
request_timeout_seconds=project_resolution_timeout_seconds,
|
|
130
|
+
)
|
|
131
|
+
return environment
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _resolve_project_if_remote_is_missing(
|
|
135
|
+
environment: dict[str, str],
|
|
136
|
+
*,
|
|
137
|
+
request_timeout_seconds: float | None = None,
|
|
138
|
+
) -> None:
|
|
139
|
+
if "GITLAB_REMOTE_URL" in environment:
|
|
140
|
+
return
|
|
141
|
+
|
|
142
|
+
project_identifier = environment.get("GITLAB_PROJECT_PATH") or environment.get(
|
|
143
|
+
"GITLAB_PROJECT_ID"
|
|
144
|
+
)
|
|
145
|
+
gitlab_url = environment.get("GITLAB_URL")
|
|
146
|
+
gitlab_token = environment.get("GITLAB_TOKEN")
|
|
147
|
+
if project_identifier is None or gitlab_url is None or gitlab_token is None:
|
|
148
|
+
return
|
|
149
|
+
|
|
150
|
+
try:
|
|
151
|
+
if request_timeout_seconds is None:
|
|
152
|
+
project = resolve_gitlab_project(
|
|
153
|
+
gitlab_url=gitlab_url,
|
|
154
|
+
project_identifier=project_identifier,
|
|
155
|
+
gitlab_token=gitlab_token,
|
|
156
|
+
)
|
|
157
|
+
else:
|
|
158
|
+
project = resolve_gitlab_project(
|
|
159
|
+
gitlab_url=gitlab_url,
|
|
160
|
+
project_identifier=project_identifier,
|
|
161
|
+
gitlab_token=gitlab_token,
|
|
162
|
+
request_timeout_seconds=request_timeout_seconds,
|
|
163
|
+
)
|
|
164
|
+
except GitLabProjectResolutionError as error:
|
|
165
|
+
raise RunConfigurationError(str(error)) from None
|
|
166
|
+
|
|
167
|
+
environment["GITLAB_REMOTE_URL"] = project.http_url_to_repo
|
|
168
|
+
environment["GITLAB_PROJECT_ID"] = str(project.id)
|
|
169
|
+
environment.pop("GITLAB_PROJECT_PATH", None)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _destination_type(config: InputObject) -> str:
|
|
173
|
+
destination_type = _input_value(config, "destination_type")
|
|
174
|
+
legacy_type = _input_value(config, "type")
|
|
175
|
+
if destination_type is not _MISSING and legacy_type is not _MISSING:
|
|
176
|
+
raise RunConfigurationError(
|
|
177
|
+
"Specify only one of destination_type or type in the run configuration."
|
|
178
|
+
)
|
|
179
|
+
value = destination_type if destination_type is not _MISSING else legacy_type
|
|
180
|
+
if not isinstance(value, str) or not value.strip():
|
|
181
|
+
raise RunConfigurationError(
|
|
182
|
+
"Required run configuration field destination_type is missing or empty."
|
|
183
|
+
)
|
|
184
|
+
return value.strip()
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _github_metadata_from_environment() -> dict[str, str]:
|
|
188
|
+
environment: dict[str, str] = {}
|
|
189
|
+
for name in (
|
|
190
|
+
"GITHUB_SHA",
|
|
191
|
+
"GITHUB_REPOSITORY",
|
|
192
|
+
"GITHUB_RUN_ID",
|
|
193
|
+
"GITHUB_STEP_SUMMARY",
|
|
194
|
+
"CLUSTERHOP_LOG_SUMMARY_LIMIT_BYTES",
|
|
195
|
+
):
|
|
196
|
+
value = os.environ.get(name)
|
|
197
|
+
if value is not None:
|
|
198
|
+
environment[name] = value
|
|
199
|
+
return environment
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _copy_config_fields(source: InputObject, environment: dict[str, str]) -> None:
|
|
203
|
+
for field_name, environment_name in _CONFIG_ENVIRONMENT_FIELDS.items():
|
|
204
|
+
value = _input_value(source, field_name)
|
|
205
|
+
if value is _MISSING or value is None:
|
|
206
|
+
continue
|
|
207
|
+
if (
|
|
208
|
+
field_name == "gitlab_project_id"
|
|
209
|
+
and isinstance(value, int)
|
|
210
|
+
and not isinstance(value, bool)
|
|
211
|
+
):
|
|
212
|
+
environment[environment_name] = str(value)
|
|
213
|
+
continue
|
|
214
|
+
if not isinstance(value, str):
|
|
215
|
+
raise RunConfigurationError(f"Run configuration field {field_name} must be a string.")
|
|
216
|
+
environment[environment_name] = value
|
|
217
|
+
|
|
218
|
+
pipeline_variables_input = _input_value(source, "extra_pipeline_variables")
|
|
219
|
+
machine_input = _input_value(source, "jsc_machine")
|
|
220
|
+
if pipeline_variables_input is _MISSING and machine_input is _MISSING:
|
|
221
|
+
return
|
|
222
|
+
|
|
223
|
+
if pipeline_variables_input is _MISSING:
|
|
224
|
+
validated_variables: dict[str, str] = {}
|
|
225
|
+
else:
|
|
226
|
+
try:
|
|
227
|
+
validated_variables = validate_extra_pipeline_variables(pipeline_variables_input)
|
|
228
|
+
except ConfigurationError as error:
|
|
229
|
+
raise RunConfigurationError(str(error)) from None
|
|
230
|
+
|
|
231
|
+
if machine_input is not _MISSING:
|
|
232
|
+
try:
|
|
233
|
+
variable_name, variable_value = jsc_machine_pipeline_variable(machine_input)
|
|
234
|
+
except MachineNameError as error:
|
|
235
|
+
raise RunConfigurationError(str(error)) from None
|
|
236
|
+
if variable_name in validated_variables:
|
|
237
|
+
raise RunConfigurationError(
|
|
238
|
+
"Run configuration field jsc_machine conflicts with extra_pipeline_variables."
|
|
239
|
+
)
|
|
240
|
+
validated_variables[variable_name] = variable_value
|
|
241
|
+
|
|
242
|
+
environment[GITLAB_VARIABLES_ENVIRONMENT_NAME] = json.dumps(
|
|
243
|
+
validated_variables,
|
|
244
|
+
separators=(",", ":"),
|
|
245
|
+
sort_keys=True,
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _copy_secret_fields(secrets: InputObject | None, environment: dict[str, str]) -> None:
|
|
250
|
+
if secrets is None:
|
|
251
|
+
raise RunConfigurationError("Required run secrets are missing.")
|
|
252
|
+
for field_name, environment_name in (
|
|
253
|
+
("gitlab_token", "GITLAB_TOKEN"),
|
|
254
|
+
("gitlab_trigger_token", "GITLAB_TRIGGER_TOKEN"),
|
|
255
|
+
):
|
|
256
|
+
value = _input_value(secrets, field_name)
|
|
257
|
+
if not isinstance(value, str) or not value.strip():
|
|
258
|
+
raise RunConfigurationError(
|
|
259
|
+
f"Required run secret field {field_name} is missing or empty."
|
|
260
|
+
)
|
|
261
|
+
environment[environment_name] = value
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _copy_option_fields(options: InputObject | None, environment: dict[str, str]) -> None:
|
|
265
|
+
if options is None:
|
|
266
|
+
return
|
|
267
|
+
for field_name, environment_name in _OPTION_ENVIRONMENT_FIELDS.items():
|
|
268
|
+
value = _input_value(options, field_name)
|
|
269
|
+
if value is _MISSING or value is None:
|
|
270
|
+
continue
|
|
271
|
+
if isinstance(value, bool) or not isinstance(value, (str, int, float, Path)):
|
|
272
|
+
raise RunConfigurationError(f"Run option {field_name} has an invalid value.")
|
|
273
|
+
environment[environment_name] = str(value)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _input_value(source: InputObject, name: str) -> object:
|
|
277
|
+
if isinstance(source, Mapping):
|
|
278
|
+
return source.get(name, _MISSING)
|
|
279
|
+
try:
|
|
280
|
+
return getattr(source, name, _MISSING)
|
|
281
|
+
except Exception:
|
|
282
|
+
raise RunConfigurationError("Unable to read the supplied run input object.") from None
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# --------------------------------------------------------------------------------------------------
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
# Copyright (C) 2026 Jayesh Badwaik <j.badwaik@fz-juelich.de>
|
|
4
|
+
# --------------------------------------------------------------------------------------------------
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import shutil
|
|
12
|
+
import sys
|
|
13
|
+
from collections.abc import Mapping, Sequence
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import yaml
|
|
17
|
+
from keyloom import KeyloomError, load as keyloom_load # type: ignore[import-untyped]
|
|
18
|
+
from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode
|
|
19
|
+
|
|
20
|
+
from clusterhop import RunConfigurationError, run as clusterhop_run
|
|
21
|
+
|
|
22
|
+
_NEW_RUN_OPTIONS = ("--config", "--options", "--secrets")
|
|
23
|
+
_KEYLOOM_PASSWORD_ENVIRONMENT_VARIABLE = "KEYLOOM_PASSWORD"
|
|
24
|
+
_KEYLOOM_MARKER = "__keyloom__"
|
|
25
|
+
_SEANVAULT_MARKER = "__seanvault__"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CliInputError(ValueError):
|
|
29
|
+
"""A user-facing, secret-safe CLI input error."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def main() -> None:
|
|
33
|
+
arguments = sys.argv[1:]
|
|
34
|
+
if arguments and arguments[0] == "run" and not _is_legacy_run(arguments):
|
|
35
|
+
raise SystemExit(_run_command(arguments[1:]))
|
|
36
|
+
|
|
37
|
+
_run_legacy_command()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _is_legacy_run(arguments: Sequence[str]) -> bool:
|
|
41
|
+
return (
|
|
42
|
+
len(arguments) == 3
|
|
43
|
+
and arguments[0] == "run"
|
|
44
|
+
and not any(
|
|
45
|
+
argument == option or argument.startswith(f"{option}=")
|
|
46
|
+
for argument in arguments[1:]
|
|
47
|
+
for option in _NEW_RUN_OPTIONS
|
|
48
|
+
)
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _run_legacy_command() -> None:
|
|
53
|
+
"""Preserve the original positional command behavior exactly."""
|
|
54
|
+
|
|
55
|
+
print(json.dumps({"arguments": sys.argv}))
|
|
56
|
+
|
|
57
|
+
if sys.argv[1] == "run":
|
|
58
|
+
print("Running the application...")
|
|
59
|
+
|
|
60
|
+
shutil.copy(sys.argv[2], "/home/work/xlab/2601-025-runner/{}.sh".format(sys.argv[3]))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _run_command(arguments: Sequence[str]) -> int:
|
|
64
|
+
parser = _run_argument_parser()
|
|
65
|
+
parsed = parser.parse_args(arguments)
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
config = _load_yaml_mapping(parsed.config, "configuration")
|
|
69
|
+
options = (
|
|
70
|
+
_load_yaml_mapping(parsed.options, "options") if parsed.options is not None else None
|
|
71
|
+
)
|
|
72
|
+
secrets = _load_secrets_mapping(parsed.secrets)
|
|
73
|
+
result = clusterhop_run(
|
|
74
|
+
config=config,
|
|
75
|
+
repo_path=parsed.repo_path,
|
|
76
|
+
options=options,
|
|
77
|
+
secrets=secrets,
|
|
78
|
+
)
|
|
79
|
+
except (CliInputError, RunConfigurationError) as error:
|
|
80
|
+
print(f"ClusterHop error: {error}", file=sys.stderr)
|
|
81
|
+
return 2
|
|
82
|
+
|
|
83
|
+
print(f"ClusterHop run finished with status: {result.status}")
|
|
84
|
+
return result.exit_code
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _run_argument_parser() -> argparse.ArgumentParser:
|
|
88
|
+
parser = argparse.ArgumentParser(
|
|
89
|
+
prog="clusterhop run",
|
|
90
|
+
description="Run ClusterHop using already-defined YAML inputs.",
|
|
91
|
+
)
|
|
92
|
+
parser.add_argument("--config", type=Path, required=True, help="configuration YAML file")
|
|
93
|
+
parser.add_argument("--options", type=Path, help="optional runtime options YAML file")
|
|
94
|
+
parser.add_argument("repo_path", type=Path, metavar="REPO_PATH")
|
|
95
|
+
parser.add_argument("--secrets", type=Path, required=True, help="secrets YAML file")
|
|
96
|
+
return parser
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _load_yaml_mapping(path: Path, label: str) -> Mapping[str, object]:
|
|
100
|
+
try:
|
|
101
|
+
loaded: object = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
102
|
+
except (OSError, UnicodeError, yaml.YAMLError):
|
|
103
|
+
raise CliInputError(f"Unable to load the {label} file.") from None
|
|
104
|
+
|
|
105
|
+
return _validate_loaded_mapping(loaded, label)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _load_secrets_mapping(
|
|
109
|
+
path: Path,
|
|
110
|
+
environ: Mapping[str, str] | None = None,
|
|
111
|
+
) -> Mapping[str, object]:
|
|
112
|
+
loaded = _load_secrets_yaml_mapping(path)
|
|
113
|
+
if _contains_marker(loaded, _SEANVAULT_MARKER):
|
|
114
|
+
raise CliInputError(
|
|
115
|
+
"Legacy Seanvault secrets are not supported. Decrypt and re-encrypt "
|
|
116
|
+
"the secrets file with Keyloom."
|
|
117
|
+
)
|
|
118
|
+
if not _contains_marker(loaded, _KEYLOOM_MARKER):
|
|
119
|
+
return loaded
|
|
120
|
+
|
|
121
|
+
source = os.environ if environ is None else environ
|
|
122
|
+
password = source.get(_KEYLOOM_PASSWORD_ENVIRONMENT_VARIABLE)
|
|
123
|
+
if not password:
|
|
124
|
+
raise CliInputError(
|
|
125
|
+
"The KEYLOOM_PASSWORD environment variable is required for a "
|
|
126
|
+
"Keyloom-marked secrets file."
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
try:
|
|
130
|
+
resolved: object = keyloom_load(path=path, password=password)
|
|
131
|
+
except (KeyloomError, OSError, UnicodeError):
|
|
132
|
+
raise CliInputError("Unable to decrypt or validate the Keyloom secrets file.") from None
|
|
133
|
+
|
|
134
|
+
return _validate_loaded_mapping(resolved, "decrypted secrets")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _load_secrets_yaml_mapping(path: Path) -> Mapping[str, object]:
|
|
138
|
+
try:
|
|
139
|
+
text = path.read_text(encoding="utf-8")
|
|
140
|
+
events = yaml.parse(text, Loader=yaml.SafeLoader)
|
|
141
|
+
if any(getattr(event, "anchor", None) is not None for event in events):
|
|
142
|
+
raise CliInputError("The secrets file must not contain YAML anchors or aliases.")
|
|
143
|
+
root = yaml.compose(text, Loader=yaml.SafeLoader)
|
|
144
|
+
if root is not None:
|
|
145
|
+
_validate_unique_yaml_keys(root)
|
|
146
|
+
loaded: object = yaml.safe_load(text)
|
|
147
|
+
except (OSError, UnicodeError, yaml.YAMLError):
|
|
148
|
+
raise CliInputError("Unable to load the secrets file.") from None
|
|
149
|
+
|
|
150
|
+
return _validate_loaded_mapping(loaded, "secrets")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _validate_unique_yaml_keys(node: Node) -> None:
|
|
154
|
+
if isinstance(node, MappingNode):
|
|
155
|
+
seen: set[tuple[str, str]] = set()
|
|
156
|
+
for key_node, value_node in node.value:
|
|
157
|
+
if isinstance(key_node, ScalarNode):
|
|
158
|
+
identity = (key_node.tag, key_node.value)
|
|
159
|
+
if identity in seen:
|
|
160
|
+
raise CliInputError("The secrets file must not contain duplicate mapping keys.")
|
|
161
|
+
seen.add(identity)
|
|
162
|
+
_validate_unique_yaml_keys(key_node)
|
|
163
|
+
_validate_unique_yaml_keys(value_node)
|
|
164
|
+
elif isinstance(node, SequenceNode):
|
|
165
|
+
for item in node.value:
|
|
166
|
+
_validate_unique_yaml_keys(item)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _contains_marker(value: object, marker: str) -> bool:
|
|
170
|
+
if isinstance(value, Mapping):
|
|
171
|
+
if marker in value:
|
|
172
|
+
return True
|
|
173
|
+
return any(_contains_marker(item, marker) for item in value.values())
|
|
174
|
+
if isinstance(value, list):
|
|
175
|
+
return any(_contains_marker(item, marker) for item in value)
|
|
176
|
+
return False
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _validate_loaded_mapping(loaded: object, label: str) -> Mapping[str, object]:
|
|
180
|
+
if not isinstance(loaded, Mapping) or not loaded:
|
|
181
|
+
raise CliInputError(f"The {label} file must contain a non-empty mapping.")
|
|
182
|
+
|
|
183
|
+
result: dict[str, object] = {}
|
|
184
|
+
for key, value in loaded.items():
|
|
185
|
+
if not isinstance(key, str):
|
|
186
|
+
raise CliInputError(f"The {label} file must use string field names.")
|
|
187
|
+
result[key] = value
|
|
188
|
+
return result
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# --------------------------------------------------------------------------------------------------
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
# Copyright (C) 2026 Jayesh Badwaik <j.badwaik@fz-juelich.de>
|
|
4
|
+
# --------------------------------------------------------------------------------------------------
|
|
5
|
+
|
|
6
|
+
"""GitHub-to-GitLab bridge support for ClusterHop."""
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# --------------------------------------------------------------------------------------------------
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
# Copyright (C) 2026 Jayesh Badwaik <j.badwaik@fz-juelich.de>
|
|
4
|
+
# --------------------------------------------------------------------------------------------------
|
|
5
|
+
|
|
6
|
+
"""Cancellation state and signal handling for active GitLab pipelines."""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import signal
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from types import FrameType, TracebackType
|
|
13
|
+
from typing import Any, Literal, cast
|
|
14
|
+
|
|
15
|
+
CleanupCallback = Callable[[], None]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class CancellationError(RuntimeError):
|
|
19
|
+
"""Base error for cancellation-controller failures."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class CancellationSetupError(CancellationError):
|
|
23
|
+
"""Raised when signal handlers cannot be installed or restored safely."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class CancellationCleanupError(CancellationError):
|
|
27
|
+
"""A sanitized error raised after cancellation cleanup fails."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CancellationController:
|
|
31
|
+
"""Track cancellation and coordinate one idempotent cleanup attempt.
|
|
32
|
+
|
|
33
|
+
The controller deliberately knows nothing about GitLab. A runner supplies a callback whose
|
|
34
|
+
closure can inspect the runner's current pipeline state and cancel an active pipeline when one
|
|
35
|
+
exists. The callback is attempted at most once, even if both supported signals are received.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, cleanup: CleanupCallback | None = None) -> None:
|
|
39
|
+
self._cleanup = cleanup
|
|
40
|
+
self._cancellation_requested = False
|
|
41
|
+
self._cleanup_started = False
|
|
42
|
+
self._cleanup_error: CancellationCleanupError | None = None
|
|
43
|
+
self._signal_number: int | None = None
|
|
44
|
+
self._previous_handlers: dict[int, object] = {}
|
|
45
|
+
self._is_active = False
|
|
46
|
+
|
|
47
|
+
def __repr__(self) -> str:
|
|
48
|
+
"""Return state without exposing callback internals."""
|
|
49
|
+
|
|
50
|
+
return (
|
|
51
|
+
f"{type(self).__name__}("
|
|
52
|
+
f"cancellation_requested={self._cancellation_requested}, "
|
|
53
|
+
f"cleanup_started={self._cleanup_started})"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def cancellation_requested(self) -> bool:
|
|
58
|
+
"""Whether SIGINT, SIGTERM, or an explicit request has been received."""
|
|
59
|
+
|
|
60
|
+
return self._cancellation_requested
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def cleanup_started(self) -> bool:
|
|
64
|
+
"""Whether a registered cleanup callback has been invoked."""
|
|
65
|
+
|
|
66
|
+
return self._cleanup_started
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def cleanup_failed(self) -> bool:
|
|
70
|
+
"""Whether the cleanup callback raised an exception."""
|
|
71
|
+
|
|
72
|
+
return self._cleanup_error is not None
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def signal_number(self) -> int | None:
|
|
76
|
+
"""The first signal number that requested cancellation, if any."""
|
|
77
|
+
|
|
78
|
+
return self._signal_number
|
|
79
|
+
|
|
80
|
+
def register_cleanup(self, cleanup: CleanupCallback) -> None:
|
|
81
|
+
"""Register the cleanup callback, running it if cancellation already arrived."""
|
|
82
|
+
|
|
83
|
+
if not callable(cleanup):
|
|
84
|
+
raise TypeError("cleanup must be callable.")
|
|
85
|
+
if self._cleanup_started:
|
|
86
|
+
raise CancellationError("Cancellation cleanup has already started.")
|
|
87
|
+
self._cleanup = cleanup
|
|
88
|
+
if self._cancellation_requested:
|
|
89
|
+
self._attempt_cleanup()
|
|
90
|
+
|
|
91
|
+
def request_cancellation(self, signal_number: int | None = None) -> None:
|
|
92
|
+
"""Latch cancellation and attempt the registered cleanup exactly once."""
|
|
93
|
+
|
|
94
|
+
if self._cancellation_requested:
|
|
95
|
+
return
|
|
96
|
+
self._cancellation_requested = True
|
|
97
|
+
self._signal_number = signal_number
|
|
98
|
+
self._attempt_cleanup()
|
|
99
|
+
|
|
100
|
+
def raise_if_cleanup_failed(self) -> None:
|
|
101
|
+
"""Raise the sanitized cleanup error, if cleanup failed."""
|
|
102
|
+
|
|
103
|
+
if self._cleanup_error is not None:
|
|
104
|
+
raise self._cleanup_error from None
|
|
105
|
+
|
|
106
|
+
def __enter__(self) -> CancellationController:
|
|
107
|
+
"""Install temporary SIGINT and SIGTERM handlers."""
|
|
108
|
+
|
|
109
|
+
if self._is_active:
|
|
110
|
+
raise CancellationSetupError("Cancellation controller is already active.")
|
|
111
|
+
|
|
112
|
+
installed_signals: list[int] = []
|
|
113
|
+
try:
|
|
114
|
+
for signal_number in (signal.SIGINT, signal.SIGTERM):
|
|
115
|
+
self._previous_handlers[signal_number] = signal.getsignal(signal_number)
|
|
116
|
+
signal.signal(signal_number, self._handle_signal)
|
|
117
|
+
installed_signals.append(signal_number)
|
|
118
|
+
except (OSError, RuntimeError, ValueError):
|
|
119
|
+
self._restore_handlers(installed_signals, suppress_errors=True)
|
|
120
|
+
self._previous_handlers.clear()
|
|
121
|
+
raise CancellationSetupError(
|
|
122
|
+
"Unable to install cancellation signal handlers."
|
|
123
|
+
) from None
|
|
124
|
+
|
|
125
|
+
self._is_active = True
|
|
126
|
+
return self
|
|
127
|
+
|
|
128
|
+
def __exit__(
|
|
129
|
+
self,
|
|
130
|
+
exception_type: type[BaseException] | None,
|
|
131
|
+
exception: BaseException | None,
|
|
132
|
+
traceback: TracebackType | None,
|
|
133
|
+
) -> Literal[False]:
|
|
134
|
+
"""Restore every signal handler saved on context entry."""
|
|
135
|
+
|
|
136
|
+
del exception_type, exception, traceback
|
|
137
|
+
previous_handlers = tuple(self._previous_handlers)
|
|
138
|
+
try:
|
|
139
|
+
self._restore_handlers(previous_handlers, suppress_errors=False)
|
|
140
|
+
finally:
|
|
141
|
+
self._previous_handlers.clear()
|
|
142
|
+
self._is_active = False
|
|
143
|
+
return False
|
|
144
|
+
|
|
145
|
+
def _handle_signal(self, signal_number: int, frame: FrameType | None) -> None:
|
|
146
|
+
del frame
|
|
147
|
+
self.request_cancellation(signal_number)
|
|
148
|
+
|
|
149
|
+
def _attempt_cleanup(self) -> None:
|
|
150
|
+
cleanup = self._cleanup
|
|
151
|
+
if cleanup is None or self._cleanup_started:
|
|
152
|
+
return
|
|
153
|
+
|
|
154
|
+
# Set this before invoking user code so a second signal cannot re-enter cleanup.
|
|
155
|
+
self._cleanup_started = True
|
|
156
|
+
try:
|
|
157
|
+
cleanup()
|
|
158
|
+
except Exception:
|
|
159
|
+
self._cleanup_error = CancellationCleanupError("Cancellation cleanup failed.")
|
|
160
|
+
|
|
161
|
+
def _restore_handlers(
|
|
162
|
+
self,
|
|
163
|
+
signal_numbers: tuple[int, ...] | list[int],
|
|
164
|
+
*,
|
|
165
|
+
suppress_errors: bool,
|
|
166
|
+
) -> None:
|
|
167
|
+
restoration_failed = False
|
|
168
|
+
for signal_number in reversed(signal_numbers):
|
|
169
|
+
previous_handler = self._previous_handlers.get(signal_number)
|
|
170
|
+
if previous_handler is None:
|
|
171
|
+
continue
|
|
172
|
+
try:
|
|
173
|
+
signal.signal(signal_number, cast(Any, previous_handler))
|
|
174
|
+
except (OSError, RuntimeError, ValueError):
|
|
175
|
+
restoration_failed = True
|
|
176
|
+
|
|
177
|
+
if restoration_failed and not suppress_errors:
|
|
178
|
+
raise CancellationSetupError("Unable to restore cancellation signal handlers.")
|