hegemony-steps-shell 0.48.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hegemony_steps_shell/__init__.py +35 -0
- hegemony_steps_shell/docs/shell.execute.md +126 -0
- hegemony_steps_shell/docs/shell.file_download.md +164 -0
- hegemony_steps_shell/execute.py +468 -0
- hegemony_steps_shell/file_download.py +664 -0
- hegemony_steps_shell/templates.py +309 -0
- hegemony_steps_shell-0.48.0.dist-info/METADATA +40 -0
- hegemony_steps_shell-0.48.0.dist-info/RECORD +11 -0
- hegemony_steps_shell-0.48.0.dist-info/WHEEL +4 -0
- hegemony_steps_shell-0.48.0.dist-info/entry_points.txt +3 -0
- hegemony_steps_shell-0.48.0.dist-info/licenses/LICENSE-AGPL-3.0-or-later.txt +235 -0
|
@@ -0,0 +1,664 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Rexonix s.r.o.
|
|
2
|
+
# SPDX-FileContributor: Jakub Trávník <jakub.travnik@gmail.com>
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
5
|
+
|
|
6
|
+
"""ShellFileDownloadHandler: each host downloads a file-repository file with curl.
|
|
7
|
+
|
|
8
|
+
The Linux/Unix counterpart of ``netcli.file_download``. The file is named by
|
|
9
|
+
a file reference (``file``: repository name and path, picked in the editor or
|
|
10
|
+
taken from a run input) or, in older flows, by its id (``file_ids``). The step
|
|
11
|
+
asks the platform for a short-lived download link (``services.open_file_download``),
|
|
12
|
+
has the host fetch it with ``curl`` over the device's shell transport, and
|
|
13
|
+
checks the SHA-256 on the host before the file takes its place. The file never
|
|
14
|
+
passes through the worker. Each host gets a link of its own, issued right
|
|
15
|
+
before its download and ended right after it.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import logging
|
|
21
|
+
import secrets
|
|
22
|
+
import shlex
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from pydantic import ConfigDict, Field, field_validator
|
|
27
|
+
|
|
28
|
+
from hegemony_step_sdk import (
|
|
29
|
+
MAX_CHAIN_OUTPUT_CHARS,
|
|
30
|
+
MAX_PARALLEL_DEVICES_LIMIT,
|
|
31
|
+
BaseHandler,
|
|
32
|
+
FileDownload,
|
|
33
|
+
FileRef,
|
|
34
|
+
HandlerContext,
|
|
35
|
+
HandlerResult,
|
|
36
|
+
HandlerTargeting,
|
|
37
|
+
InvalidConfigError,
|
|
38
|
+
OutputField,
|
|
39
|
+
StepKind,
|
|
40
|
+
for_each_target_device,
|
|
41
|
+
parse_file_ref,
|
|
42
|
+
)
|
|
43
|
+
from hegemony_step_sdk.ssh import SshSecurityConfig
|
|
44
|
+
|
|
45
|
+
from .execute import _open_shell
|
|
46
|
+
|
|
47
|
+
logger = logging.getLogger(__name__)
|
|
48
|
+
|
|
49
|
+
_DEFAULT_TRANSFER_TIMEOUT_SECONDS = 3600
|
|
50
|
+
#: How long a host has to start its download once the link exists. The link is
|
|
51
|
+
#: asked for after the connection is open, so this only has to cover the start
|
|
52
|
+
#: of the transfer; a transfer under way is not cut off at its end.
|
|
53
|
+
_LINK_TTL_SECONDS = 600
|
|
54
|
+
#: Time the checksum and the final move get on the host.
|
|
55
|
+
_FINISH_TIMEOUT_SECONDS = 600
|
|
56
|
+
#: Time the transport gets beyond curl's own limit to return curl's answer.
|
|
57
|
+
_RUN_MARGIN_SECONDS = 15
|
|
58
|
+
#: What a link is replaced with wherever the step reports text a host printed.
|
|
59
|
+
_LINK_PLACEHOLDER = "<download link>"
|
|
60
|
+
#: Suffix of the file while it downloads (after a few random characters, so
|
|
61
|
+
#: every transfer has a staging file of its own). It becomes the destination
|
|
62
|
+
#: file only once it is complete (and verified), so a broken download never
|
|
63
|
+
#: looks done.
|
|
64
|
+
_PARTIAL_SUFFIX = ".part"
|
|
65
|
+
|
|
66
|
+
_NO_DOWNLOAD_LINKS = (
|
|
67
|
+
"This Hegemony version cannot issue download links for file "
|
|
68
|
+
"repository files; upgrade the platform to use this step"
|
|
69
|
+
)
|
|
70
|
+
_NO_REPOSITORY_PATHS = (
|
|
71
|
+
"This Hegemony version cannot open files by repository and path; "
|
|
72
|
+
"upgrade the platform or use a file id"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _is_unset(value: Any) -> bool:
|
|
77
|
+
"""Whether a file setting is empty (absent, blank text, an empty list or dict)."""
|
|
78
|
+
if value is None:
|
|
79
|
+
return True
|
|
80
|
+
if isinstance(value, str):
|
|
81
|
+
# An empty list or object rendered as text (an older platform passes
|
|
82
|
+
# the setting on as text) is as empty as the value itself.
|
|
83
|
+
return value.strip() in ("", "[]", "{}")
|
|
84
|
+
if isinstance(value, (list, tuple, dict)):
|
|
85
|
+
return not value
|
|
86
|
+
return False
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _file_details(download: FileDownload) -> dict[str, Any]:
|
|
90
|
+
"""What the step reports about the downloaded file (never the link)."""
|
|
91
|
+
return {
|
|
92
|
+
"file_id": download.file_id,
|
|
93
|
+
"filename": download.filename,
|
|
94
|
+
"size_bytes": download.size_bytes,
|
|
95
|
+
"sha256": download.sha256,
|
|
96
|
+
"md5": download.md5,
|
|
97
|
+
# getattr: a host older than these fields builds an older FileDownload.
|
|
98
|
+
"repository": getattr(download, "repository", ""),
|
|
99
|
+
"repository_id": getattr(download, "repository_id", ""),
|
|
100
|
+
"path": getattr(download, "path", ""),
|
|
101
|
+
"folder": getattr(download, "folder", ""),
|
|
102
|
+
"content_type": getattr(download, "content_type", None),
|
|
103
|
+
"sha512": getattr(download, "sha512", None),
|
|
104
|
+
"metadata": dict(getattr(download, "metadata", None) or {}),
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _file_ref_of(file_info: dict[str, Any] | None) -> dict[str, str] | None:
|
|
109
|
+
"""The reference a later step can take for the file the host named, or None."""
|
|
110
|
+
if not file_info or not file_info.get("repository") or not file_info.get("path"):
|
|
111
|
+
return None
|
|
112
|
+
return {"repository": file_info["repository"], "path": file_info["path"]}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _file_output(file_info: dict[str, Any], ref: FileRef | None) -> dict[str, Any]:
|
|
116
|
+
"""The step output's file keys, ``file`` being a reference a later step can take.
|
|
117
|
+
|
|
118
|
+
The host's answer wins; without one (no target got a link) the configured
|
|
119
|
+
reference fills in. What neither knows (a host older than these fields,
|
|
120
|
+
given a file id) is None.
|
|
121
|
+
"""
|
|
122
|
+
repository = file_info.get("repository") or (ref.repository if ref else "")
|
|
123
|
+
path = file_info.get("path") or (ref.path if ref else "")
|
|
124
|
+
known = bool(file_info.get("repository"))
|
|
125
|
+
return {
|
|
126
|
+
"repository": repository or None,
|
|
127
|
+
"repository_id": file_info.get("repository_id") or None,
|
|
128
|
+
"path": path or None,
|
|
129
|
+
# "" is the repository root, so it is kept when the host named the file.
|
|
130
|
+
"folder": file_info.get("folder", "") if known else None,
|
|
131
|
+
"content_type": file_info.get("content_type"),
|
|
132
|
+
"sha512": file_info.get("sha512"),
|
|
133
|
+
"metadata": file_info.get("metadata") or {},
|
|
134
|
+
"file": {"repository": repository, "path": path} if repository and path else None,
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class ShellFileDownloadConfig(SshSecurityConfig):
|
|
139
|
+
"""Config for ``shell.file_download``."""
|
|
140
|
+
|
|
141
|
+
model_config = ConfigDict(extra="allow")
|
|
142
|
+
|
|
143
|
+
# A file reference ({"repository": ..., "path": ...}), a list of one, or
|
|
144
|
+
# a template such as "{{ inputs.package }}". Listed in
|
|
145
|
+
# native_config_keys, so a run input arrives as the reference itself.
|
|
146
|
+
file: dict[str, Any] | list[Any] | str | None = Field(
|
|
147
|
+
default=None,
|
|
148
|
+
title="File",
|
|
149
|
+
description=(
|
|
150
|
+
"The file each host downloads. Pick a file from a file repository, or "
|
|
151
|
+
"reference a run-form field such as {{ inputs.package }}."
|
|
152
|
+
),
|
|
153
|
+
json_schema_extra={"x_widget": "repository-file-ref", "x_col_span": 2},
|
|
154
|
+
)
|
|
155
|
+
# Named file_ids on purpose: the platform refuses to delete a stored file
|
|
156
|
+
# that a flow lists under this key. Kept for flows saved before ``file``.
|
|
157
|
+
file_ids: list[str] = Field(
|
|
158
|
+
default_factory=list,
|
|
159
|
+
title="File by id (legacy)",
|
|
160
|
+
description=(
|
|
161
|
+
"The file each host downloads, by its id, as older flows store it. "
|
|
162
|
+
"Leave empty when File is set."
|
|
163
|
+
),
|
|
164
|
+
json_schema_extra={
|
|
165
|
+
"x_widget": "repository-file-select",
|
|
166
|
+
"x_multiple": False,
|
|
167
|
+
"x_file_type_filter": "file",
|
|
168
|
+
"x_col_span": 2,
|
|
169
|
+
"x_advanced": True,
|
|
170
|
+
},
|
|
171
|
+
)
|
|
172
|
+
dest_dir: str = Field(
|
|
173
|
+
default="/var/tmp",
|
|
174
|
+
title="Destination Directory",
|
|
175
|
+
description=(
|
|
176
|
+
"Directory on the host the file goes into, as an absolute path. It must "
|
|
177
|
+
'exist and the login account must be able to write to it. Defaults to "/var/tmp".'
|
|
178
|
+
),
|
|
179
|
+
json_schema_extra={"x_placeholder": "/var/tmp"},
|
|
180
|
+
)
|
|
181
|
+
dest_filename: str = Field(
|
|
182
|
+
default="",
|
|
183
|
+
title="Destination Filename",
|
|
184
|
+
description="File name on the host. Leave empty to keep the stored file's name.",
|
|
185
|
+
json_schema_extra={"x_placeholder": "app-release-1.4.2.tar.gz"},
|
|
186
|
+
)
|
|
187
|
+
verify_checksum: bool = Field(
|
|
188
|
+
default=True,
|
|
189
|
+
title="Verify checksum",
|
|
190
|
+
description=(
|
|
191
|
+
"Compare the downloaded file's SHA-256 with the stored file's before it "
|
|
192
|
+
"takes its place (needs sha256sum on the host). On by default."
|
|
193
|
+
),
|
|
194
|
+
)
|
|
195
|
+
transfer_timeout_seconds: int = Field(
|
|
196
|
+
_DEFAULT_TRANSFER_TIMEOUT_SECONDS,
|
|
197
|
+
ge=60,
|
|
198
|
+
le=86400,
|
|
199
|
+
title="Transfer timeout (seconds)",
|
|
200
|
+
description=(
|
|
201
|
+
"How long one host's download may take before it fails. The step "
|
|
202
|
+
"timeout must still cover every host. Defaults to one hour."
|
|
203
|
+
),
|
|
204
|
+
)
|
|
205
|
+
max_parallel_devices: int = Field(
|
|
206
|
+
default=1,
|
|
207
|
+
ge=1,
|
|
208
|
+
le=MAX_PARALLEL_DEVICES_LIMIT,
|
|
209
|
+
title="Max Parallel Devices",
|
|
210
|
+
description="Hosts downloading concurrently (1 = sequential)",
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
@field_validator("file_ids", mode="before")
|
|
214
|
+
@classmethod
|
|
215
|
+
def _single_file_id(cls, value: Any) -> Any:
|
|
216
|
+
# One id given as text, as a rendered template gives it, is a list of one.
|
|
217
|
+
return [value] if isinstance(value, str) else value
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
@dataclass
|
|
221
|
+
class _DeviceOutcome:
|
|
222
|
+
"""One host's result, merged in device order after the fan-out."""
|
|
223
|
+
|
|
224
|
+
evidence: list[dict] = field(default_factory=list)
|
|
225
|
+
errors: list[str] = field(default_factory=list)
|
|
226
|
+
success: bool = True
|
|
227
|
+
transferred: bool = False
|
|
228
|
+
verified: bool = False
|
|
229
|
+
file: dict[str, Any] | None = None
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _has_control_characters(value: str) -> bool:
|
|
233
|
+
"""Whether ``value`` holds a line break or another non-printable character.
|
|
234
|
+
|
|
235
|
+
Every value is quoted into the command the host runs, so none can become
|
|
236
|
+
shell syntax; a control character in a path is refused all the same,
|
|
237
|
+
because no real directory or file name needs one and a terminal or a log
|
|
238
|
+
that shows it can be misled by it.
|
|
239
|
+
"""
|
|
240
|
+
return any(not character.isprintable() for character in value)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _is_plain_file_name(value: str) -> bool:
|
|
244
|
+
"""Whether ``value`` names a file and nothing else: no directory part."""
|
|
245
|
+
return bool(value) and "/" not in value and value not in (".", "..")
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _without_link(text: str, url: str) -> str:
|
|
249
|
+
"""``text`` with the download link taken out.
|
|
250
|
+
|
|
251
|
+
The host redacts the link's token from everything a step reports; this
|
|
252
|
+
keeps the rest of the address out too, and does not depend on the host.
|
|
253
|
+
"""
|
|
254
|
+
if not url:
|
|
255
|
+
return text
|
|
256
|
+
return text.replace(url, _LINK_PLACEHOLDER)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def download_command(url: str, partial_path: str, timeout_seconds: int) -> str:
|
|
260
|
+
"""The curl line that fetches ``url`` into ``partial_path``.
|
|
261
|
+
|
|
262
|
+
``-f`` makes an HTTP error a failure instead of a saved error page;
|
|
263
|
+
``-sS`` drops the progress meter and keeps error messages. No ``-L``: a
|
|
264
|
+
download link answers with the file, so a redirect is not followed.
|
|
265
|
+
"""
|
|
266
|
+
return (
|
|
267
|
+
f"curl -fsS --max-time {int(timeout_seconds)} "
|
|
268
|
+
f"-o {shlex.quote(partial_path)} {shlex.quote(url)}"
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _first_line(text: str) -> str:
|
|
273
|
+
lines = text.strip().splitlines()
|
|
274
|
+
return lines[0] if lines else ""
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
#: What the step puts in its output, for the flow editor's variable picker.
|
|
278
|
+
_OUTPUT_FIELDS = (
|
|
279
|
+
OutputField("filename", "Name of the stored file", "string"),
|
|
280
|
+
OutputField("file_id", "Id of the stored file", "string"),
|
|
281
|
+
OutputField("dest_dir", "Directory the file went to on each host", "string"),
|
|
282
|
+
OutputField("dest_filename", "File name on each host", "string"),
|
|
283
|
+
OutputField("dest_path", "Full path of the file on each host", "string"),
|
|
284
|
+
OutputField("size_bytes", "Size of the stored file in bytes", "int"),
|
|
285
|
+
OutputField("sha256", "SHA-256 of the stored file", "string"),
|
|
286
|
+
OutputField("md5", "MD5 of the stored file (null if not stored)", "string"),
|
|
287
|
+
OutputField("sha512", "SHA-512 of the stored file (null if not stored)", "string"),
|
|
288
|
+
OutputField("content_type", "Content type of the stored file", "string"),
|
|
289
|
+
OutputField("metadata", "Metadata stored with the file", "object"),
|
|
290
|
+
OutputField("transferred_count", "Hosts that completed the download", "int"),
|
|
291
|
+
OutputField("repository", "Name of the file repository", "string"),
|
|
292
|
+
OutputField("repository_id", "Id of the file repository", "string"),
|
|
293
|
+
OutputField("path", "Path of the file in the repository (folder and name)", "string"),
|
|
294
|
+
OutputField("folder", "Folder of the file (empty for the repository root)", "string"),
|
|
295
|
+
OutputField(
|
|
296
|
+
"file",
|
|
297
|
+
"File reference to the same file, for another download step's File",
|
|
298
|
+
"object",
|
|
299
|
+
),
|
|
300
|
+
OutputField("verified", "Whether every host's copy passed the checksum check", "bool"),
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
class ShellFileDownloadHandler(BaseHandler):
|
|
305
|
+
"""Handler that has each Linux/Unix host download one file-repository file."""
|
|
306
|
+
|
|
307
|
+
handler_id = "shell.file_download"
|
|
308
|
+
supported_kinds = [StepKind.ACTION]
|
|
309
|
+
display_name = "Download File to Host"
|
|
310
|
+
description = (
|
|
311
|
+
"Have each Linux/Unix host download a file-repository file with curl, "
|
|
312
|
+
"through a short-lived link, and verify its checksum."
|
|
313
|
+
)
|
|
314
|
+
category = "Actions"
|
|
315
|
+
targeting = HandlerTargeting(roles=True, ips=False)
|
|
316
|
+
config_model = ShellFileDownloadConfig
|
|
317
|
+
output_fields = _OUTPUT_FIELDS
|
|
318
|
+
# "{{ inputs.package }}" reaches the handler as the file reference itself.
|
|
319
|
+
native_config_keys = frozenset({"file"})
|
|
320
|
+
default_config = {"dest_dir": "/var/tmp", "verify_checksum": True}
|
|
321
|
+
# A large file over a slow link takes a while, and every host has its own
|
|
322
|
+
# Transfer timeout within the step's.
|
|
323
|
+
default_timeout_seconds = 7200
|
|
324
|
+
|
|
325
|
+
async def execute(self, ctx: HandlerContext) -> HandlerResult:
|
|
326
|
+
"""Stage the configured file on every target host."""
|
|
327
|
+
target_devices = ctx.get_target_devices()
|
|
328
|
+
if not target_devices:
|
|
329
|
+
return HandlerResult(
|
|
330
|
+
success=False,
|
|
331
|
+
error="No target devices configured",
|
|
332
|
+
summary="Missing target device",
|
|
333
|
+
)
|
|
334
|
+
try:
|
|
335
|
+
config = self.parse_config(ctx.config, ShellFileDownloadConfig)
|
|
336
|
+
except InvalidConfigError as exc:
|
|
337
|
+
return exc.result()
|
|
338
|
+
|
|
339
|
+
file_ids = [item.strip() for item in config.file_ids if item.strip()]
|
|
340
|
+
ref: FileRef | None = None
|
|
341
|
+
file_id = ""
|
|
342
|
+
if not _is_unset(config.file):
|
|
343
|
+
if file_ids:
|
|
344
|
+
return HandlerResult(
|
|
345
|
+
success=False,
|
|
346
|
+
error="Set either File or File by id (legacy), not both",
|
|
347
|
+
summary="Conflicting file settings",
|
|
348
|
+
)
|
|
349
|
+
try:
|
|
350
|
+
ref = parse_file_ref(config.file)
|
|
351
|
+
except ValueError as exc:
|
|
352
|
+
return HandlerResult(success=False, error=str(exc), summary="Invalid file")
|
|
353
|
+
elif len(file_ids) != 1:
|
|
354
|
+
return HandlerResult(
|
|
355
|
+
success=False,
|
|
356
|
+
error=(
|
|
357
|
+
"Select exactly one file repository file"
|
|
358
|
+
if file_ids
|
|
359
|
+
else "No file repository file selected"
|
|
360
|
+
),
|
|
361
|
+
summary="Missing file",
|
|
362
|
+
)
|
|
363
|
+
else:
|
|
364
|
+
file_id = file_ids[0]
|
|
365
|
+
named_file: dict[str, Any] = (
|
|
366
|
+
{"repository": ref.repository, "path": ref.path} if ref else {"file_id": file_id}
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
services = ctx.require_services()
|
|
370
|
+
open_file_download = getattr(services, "open_file_download", None)
|
|
371
|
+
if open_file_download is None:
|
|
372
|
+
return HandlerResult(
|
|
373
|
+
success=False,
|
|
374
|
+
error=_NO_DOWNLOAD_LINKS,
|
|
375
|
+
summary="Download links not supported",
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
dest_dir = config.dest_dir.strip()
|
|
379
|
+
dest_filename_config = config.dest_filename.strip()
|
|
380
|
+
for label, value in (
|
|
381
|
+
("Destination Directory", dest_dir),
|
|
382
|
+
("Destination Filename", dest_filename_config),
|
|
383
|
+
):
|
|
384
|
+
if _has_control_characters(value):
|
|
385
|
+
return HandlerResult(
|
|
386
|
+
success=False,
|
|
387
|
+
error=f"{label} contains a line break or another control character",
|
|
388
|
+
summary="Invalid destination",
|
|
389
|
+
)
|
|
390
|
+
if not dest_dir.startswith("/"):
|
|
391
|
+
return HandlerResult(
|
|
392
|
+
success=False,
|
|
393
|
+
error="Destination Directory must be an absolute path (starting with /)",
|
|
394
|
+
summary="Invalid destination",
|
|
395
|
+
)
|
|
396
|
+
if dest_filename_config and not _is_plain_file_name(dest_filename_config):
|
|
397
|
+
return HandlerResult(
|
|
398
|
+
success=False,
|
|
399
|
+
error=(
|
|
400
|
+
"Destination Filename must be a file name without a directory part "
|
|
401
|
+
"(no slash); put the directory in Destination Directory"
|
|
402
|
+
),
|
|
403
|
+
summary="Invalid destination",
|
|
404
|
+
)
|
|
405
|
+
directory = dest_dir.rstrip("/")
|
|
406
|
+
many = len(target_devices) > 1
|
|
407
|
+
|
|
408
|
+
async def _process_device(device: dict) -> _DeviceOutcome:
|
|
409
|
+
outcome = _DeviceOutcome()
|
|
410
|
+
device_id = device.get("id", device.get("mgmt_host"))
|
|
411
|
+
device_name = device.get("name", device_id)
|
|
412
|
+
artifact_name = f"File download ({device_name})" if many else "File download"
|
|
413
|
+
|
|
414
|
+
def _fail(message: str, **details: Any) -> _DeviceOutcome:
|
|
415
|
+
outcome.success = False
|
|
416
|
+
outcome.errors.append(f"{device_name}: {message}")
|
|
417
|
+
outcome.evidence.append(
|
|
418
|
+
{
|
|
419
|
+
"kind": "cli_output",
|
|
420
|
+
"name": f"Error ({device_name})",
|
|
421
|
+
"device_id": device_id,
|
|
422
|
+
"content_text": message,
|
|
423
|
+
"content_json": {
|
|
424
|
+
"device_id": device_id,
|
|
425
|
+
"device_name": device_name,
|
|
426
|
+
**named_file,
|
|
427
|
+
"error": message,
|
|
428
|
+
"success": False,
|
|
429
|
+
**details,
|
|
430
|
+
},
|
|
431
|
+
}
|
|
432
|
+
)
|
|
433
|
+
return outcome
|
|
434
|
+
|
|
435
|
+
if not device.get("mgmt_host"):
|
|
436
|
+
return _fail("no mgmt_host configured")
|
|
437
|
+
|
|
438
|
+
# The connection first, the link second: the link's time is then
|
|
439
|
+
# not spent waiting for a host that is slow to answer.
|
|
440
|
+
try:
|
|
441
|
+
transport = await _open_shell(services, device, config.allow_legacy_ssh)
|
|
442
|
+
except Exception as exc: # noqa: BLE001 - one host's failure, reported
|
|
443
|
+
return _fail(str(exc) or type(exc).__name__)
|
|
444
|
+
|
|
445
|
+
url = ""
|
|
446
|
+
link_opened = False
|
|
447
|
+
dest_path = ""
|
|
448
|
+
#: The staging file, quoted for the command line, once curl may have made it.
|
|
449
|
+
staged: str | None = None
|
|
450
|
+
|
|
451
|
+
async def _discard() -> None:
|
|
452
|
+
"""Remove the staging file; never hides the failure that led here."""
|
|
453
|
+
if staged is None:
|
|
454
|
+
return
|
|
455
|
+
try:
|
|
456
|
+
await transport.run(f"rm -f {staged}", timeout=_FINISH_TIMEOUT_SECONDS)
|
|
457
|
+
except Exception: # noqa: BLE001 - cleanup is best-effort
|
|
458
|
+
logger.debug("staging file cleanup failed", exc_info=True)
|
|
459
|
+
|
|
460
|
+
try:
|
|
461
|
+
if ref is None:
|
|
462
|
+
link = open_file_download(file_id, ttl_seconds=_LINK_TTL_SECONDS)
|
|
463
|
+
else:
|
|
464
|
+
try:
|
|
465
|
+
link = open_file_download(
|
|
466
|
+
repository=ref.repository,
|
|
467
|
+
path=ref.path,
|
|
468
|
+
ttl_seconds=_LINK_TTL_SECONDS,
|
|
469
|
+
)
|
|
470
|
+
except TypeError:
|
|
471
|
+
# A host older than repository and path refuses them.
|
|
472
|
+
return _fail(_NO_REPOSITORY_PATHS)
|
|
473
|
+
async with link as download:
|
|
474
|
+
link_opened = True
|
|
475
|
+
url = download.url
|
|
476
|
+
outcome.file = _file_details(download)
|
|
477
|
+
if not url:
|
|
478
|
+
return _fail(
|
|
479
|
+
"the platform has no address hosts can reach it at; an "
|
|
480
|
+
"administrator sets HEGEMONY_API_EXTERNAL_URL"
|
|
481
|
+
)
|
|
482
|
+
dest_filename = dest_filename_config or download.filename
|
|
483
|
+
if _has_control_characters(dest_filename):
|
|
484
|
+
return _fail(
|
|
485
|
+
"the stored file's name contains a line break or another "
|
|
486
|
+
"control character; set Destination Filename"
|
|
487
|
+
)
|
|
488
|
+
if not _is_plain_file_name(dest_filename):
|
|
489
|
+
return _fail(
|
|
490
|
+
"the stored file's name has a directory part; set Destination Filename"
|
|
491
|
+
)
|
|
492
|
+
dest_path = f"{directory}/{dest_filename}"
|
|
493
|
+
# mv would put the file inside a directory of that name and
|
|
494
|
+
# still succeed, so the step would report a file that is
|
|
495
|
+
# not there. Asked before anything is downloaded.
|
|
496
|
+
is_directory = await transport.run(
|
|
497
|
+
f"test -d {shlex.quote(dest_path)}", timeout=_FINISH_TIMEOUT_SECONDS
|
|
498
|
+
)
|
|
499
|
+
if is_directory.exit_code == 0:
|
|
500
|
+
return _fail(
|
|
501
|
+
f"{dest_path} is a directory on the host; set Destination "
|
|
502
|
+
"Filename, or name the directory in Destination Directory"
|
|
503
|
+
)
|
|
504
|
+
# A name of its own for every transfer: two runs staging the
|
|
505
|
+
# same file on one host then never write to, verify or
|
|
506
|
+
# remove each other's download.
|
|
507
|
+
staging_path = f"{dest_path}.{secrets.token_hex(4)}{_PARTIAL_SUFFIX}"
|
|
508
|
+
staged = shlex.quote(staging_path)
|
|
509
|
+
fetched = await transport.run(
|
|
510
|
+
download_command(url, staging_path, config.transfer_timeout_seconds),
|
|
511
|
+
timeout=config.transfer_timeout_seconds + _RUN_MARGIN_SECONDS,
|
|
512
|
+
)
|
|
513
|
+
# The link is ended here: what follows needs the host only.
|
|
514
|
+
if fetched.exit_code != 0:
|
|
515
|
+
await _discard()
|
|
516
|
+
detail = _without_link(_first_line(fetched.stderr or fetched.stdout), url)
|
|
517
|
+
return _fail(
|
|
518
|
+
f"curl exited {fetched.exit_code}" + (f" ({detail})" if detail else ""),
|
|
519
|
+
exit_code=fetched.exit_code,
|
|
520
|
+
)
|
|
521
|
+
if config.verify_checksum:
|
|
522
|
+
# Read from standard input: given a file name with a
|
|
523
|
+
# backslash, sha256sum escapes the name and marks the line
|
|
524
|
+
# with a leading backslash, which is not part of the digest.
|
|
525
|
+
summed = await transport.run(
|
|
526
|
+
f"sha256sum < {staged}", timeout=_FINISH_TIMEOUT_SECONDS
|
|
527
|
+
)
|
|
528
|
+
words = summed.stdout.split()
|
|
529
|
+
actual = words[0].lower() if summed.exit_code == 0 and words else ""
|
|
530
|
+
expected = str(download.sha256 or "").lower()
|
|
531
|
+
if not actual or actual != expected:
|
|
532
|
+
await _discard()
|
|
533
|
+
if not actual:
|
|
534
|
+
reason = _first_line(summed.stderr or summed.stdout)
|
|
535
|
+
return _fail(
|
|
536
|
+
"the checksum could not be computed on the host"
|
|
537
|
+
+ (f" ({reason})" if reason else "")
|
|
538
|
+
+ "; install sha256sum or turn Verify checksum off"
|
|
539
|
+
)
|
|
540
|
+
return _fail(
|
|
541
|
+
"the downloaded file's SHA-256 does not match the stored file's",
|
|
542
|
+
expected_sha256=expected,
|
|
543
|
+
actual_sha256=actual,
|
|
544
|
+
)
|
|
545
|
+
outcome.verified = True
|
|
546
|
+
moved = await transport.run(
|
|
547
|
+
f"mv -f {staged} {shlex.quote(dest_path)}", timeout=_FINISH_TIMEOUT_SECONDS
|
|
548
|
+
)
|
|
549
|
+
if moved.exit_code != 0:
|
|
550
|
+
await _discard()
|
|
551
|
+
reason = _first_line(moved.stderr or moved.stdout)
|
|
552
|
+
return _fail(
|
|
553
|
+
"the downloaded file could not be moved into place"
|
|
554
|
+
+ (f" ({reason})" if reason else "")
|
|
555
|
+
)
|
|
556
|
+
# A directory that appeared under the name since the check above
|
|
557
|
+
# takes the file inside it, and mv still succeeds. Asked again,
|
|
558
|
+
# so that case is reported as the failure it is.
|
|
559
|
+
in_place = await transport.run(
|
|
560
|
+
f"test -f {shlex.quote(dest_path)}", timeout=_FINISH_TIMEOUT_SECONDS
|
|
561
|
+
)
|
|
562
|
+
if in_place.exit_code != 0:
|
|
563
|
+
return _fail(
|
|
564
|
+
f"{dest_path} is not a regular file after the move; a directory "
|
|
565
|
+
"of that name may have appeared during the download"
|
|
566
|
+
)
|
|
567
|
+
except NotImplementedError as exc:
|
|
568
|
+
await _discard()
|
|
569
|
+
if not link_opened:
|
|
570
|
+
# The host platform has the service but does not implement
|
|
571
|
+
# it (the SDK's default).
|
|
572
|
+
return _fail(_NO_DOWNLOAD_LINKS)
|
|
573
|
+
return _fail(_without_link(str(exc) or type(exc).__name__, url))
|
|
574
|
+
except Exception as exc: # noqa: BLE001 - one host's failure, reported
|
|
575
|
+
# A command that raised (a timeout, a dropped session) may have
|
|
576
|
+
# left the staging file: it is removed if the host still answers.
|
|
577
|
+
await _discard()
|
|
578
|
+
return _fail(_without_link(str(exc) or type(exc).__name__, url))
|
|
579
|
+
finally:
|
|
580
|
+
try:
|
|
581
|
+
await transport.close()
|
|
582
|
+
except Exception: # noqa: BLE001 - closing is best-effort
|
|
583
|
+
logger.debug("shell transport close failed", exc_info=True)
|
|
584
|
+
|
|
585
|
+
outcome.transferred = True
|
|
586
|
+
text = f"Downloaded to {dest_path}" + (", SHA-256 verified" if outcome.verified else "")
|
|
587
|
+
if not getattr(transport, "streams_exec_output", False):
|
|
588
|
+
await ctx.emit_output(text + "\n", device=device, command="file download")
|
|
589
|
+
outcome.evidence.append(
|
|
590
|
+
{
|
|
591
|
+
"kind": "cli_output",
|
|
592
|
+
"name": artifact_name,
|
|
593
|
+
"device_id": device_id,
|
|
594
|
+
"content_text": text,
|
|
595
|
+
"content_json": {
|
|
596
|
+
"device_id": device_id,
|
|
597
|
+
"device_name": device_name,
|
|
598
|
+
**(outcome.file or {}),
|
|
599
|
+
"file": _file_ref_of(outcome.file) or (ref.as_dict() if ref else None),
|
|
600
|
+
"dest_dir": dest_dir,
|
|
601
|
+
"dest_filename": dest_filename,
|
|
602
|
+
"dest_path": dest_path,
|
|
603
|
+
"verified": outcome.verified,
|
|
604
|
+
"success": True,
|
|
605
|
+
},
|
|
606
|
+
}
|
|
607
|
+
)
|
|
608
|
+
return outcome
|
|
609
|
+
|
|
610
|
+
outcomes = await for_each_target_device(
|
|
611
|
+
target_devices,
|
|
612
|
+
_process_device,
|
|
613
|
+
max_parallel=config.max_parallel_devices,
|
|
614
|
+
)
|
|
615
|
+
|
|
616
|
+
errors = [error for outcome in outcomes for error in outcome.errors]
|
|
617
|
+
all_success = all(outcome.success for outcome in outcomes)
|
|
618
|
+
# Each target opens its own link, so a path is looked up once per
|
|
619
|
+
# target. Should the path name another file part way through (the
|
|
620
|
+
# file was moved or replaced), the targets hold different files: the
|
|
621
|
+
# step fails rather than report one file for all of them.
|
|
622
|
+
got_ids = sorted(
|
|
623
|
+
{str(o.file["file_id"]) for o in outcomes if o.file and o.file.get("file_id")}
|
|
624
|
+
)
|
|
625
|
+
if len(got_ids) > 1:
|
|
626
|
+
all_success = False
|
|
627
|
+
errors.append(
|
|
628
|
+
f"the hosts did not all get the same file: {', '.join(got_ids)}; "
|
|
629
|
+
"the file at that path changed during the step"
|
|
630
|
+
)
|
|
631
|
+
transferred_count = sum(1 for outcome in outcomes if outcome.transferred)
|
|
632
|
+
file_info = next((outcome.file for outcome in outcomes if outcome.file), None) or {}
|
|
633
|
+
filename = str(file_info.get("filename") or "")
|
|
634
|
+
dest_name = dest_filename_config or filename
|
|
635
|
+
|
|
636
|
+
if all_success:
|
|
637
|
+
summary = f"{filename or 'File'} downloaded by {transferred_count} host(s)"
|
|
638
|
+
else:
|
|
639
|
+
summary = f"File download failed on some hosts: {'; '.join(errors)}"
|
|
640
|
+
|
|
641
|
+
return HandlerResult(
|
|
642
|
+
success=all_success,
|
|
643
|
+
error="; ".join(errors)[:MAX_CHAIN_OUTPUT_CHARS] if errors else None,
|
|
644
|
+
summary=summary[:MAX_CHAIN_OUTPUT_CHARS],
|
|
645
|
+
metrics={
|
|
646
|
+
"device_count": len(target_devices),
|
|
647
|
+
"transferred_count": transferred_count,
|
|
648
|
+
"size_bytes": file_info.get("size_bytes"),
|
|
649
|
+
},
|
|
650
|
+
evidence=[item for outcome in outcomes for item in outcome.evidence],
|
|
651
|
+
output={
|
|
652
|
+
"file_id": file_id or file_info.get("file_id") or None,
|
|
653
|
+
"filename": filename,
|
|
654
|
+
"dest_dir": dest_dir,
|
|
655
|
+
"dest_filename": dest_name,
|
|
656
|
+
"dest_path": f"{directory}/{dest_name}" if dest_name else "",
|
|
657
|
+
"size_bytes": file_info.get("size_bytes"),
|
|
658
|
+
"sha256": file_info.get("sha256"),
|
|
659
|
+
"md5": file_info.get("md5"),
|
|
660
|
+
"verified": config.verify_checksum and all_success,
|
|
661
|
+
"transferred_count": transferred_count,
|
|
662
|
+
**_file_output(file_info, ref),
|
|
663
|
+
},
|
|
664
|
+
)
|