dirigent-block-execute 0.17.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirigent_block_execute/__init__.py +43 -0
- dirigent_block_execute/build.py +319 -0
- dirigent_block_execute/capture.py +289 -0
- dirigent_block_execute/compose.py +675 -0
- dirigent_block_execute/docker.py +1262 -0
- dirigent_block_execute/environment.py +47 -0
- dirigent_block_execute/git.py +548 -0
- dirigent_block_execute/messages.py +183 -0
- dirigent_block_execute/py.typed +0 -0
- dirigent_block_execute/reap.py +174 -0
- dirigent_block_execute/secrets.py +42 -0
- dirigent_block_execute/shell.py +174 -0
- dirigent_block_execute/subprocess.py +219 -0
- dirigent_block_execute-0.17.1.dist-info/METADATA +26 -0
- dirigent_block_execute-0.17.1.dist-info/RECORD +18 -0
- dirigent_block_execute-0.17.1.dist-info/WHEEL +4 -0
- dirigent_block_execute-0.17.1.dist-info/entry_points.txt +3 -0
- dirigent_block_execute-0.17.1.dist-info/licenses/LICENSE +18 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""What a pipeline document may and may not pull out of the worker's own environment.
|
|
2
|
+
|
|
3
|
+
``shell.run`` and ``docker.run`` both let a step name worker environment variables to
|
|
4
|
+
inherit. That is the point of an allowlist -- a step that needs ``JAVA_HOME`` should be able
|
|
5
|
+
to say so -- but the worker's environment is also where dirigent's own secrets live, and
|
|
6
|
+
the allowlist is a field of a *pipeline document*. Without a denylist, anyone who can edit a
|
|
7
|
+
pipeline can write::
|
|
8
|
+
|
|
9
|
+
env_allowlist: [DIRIGENT_SECRET_KEY]
|
|
10
|
+
command: 'echo "$DIRIGENT_SECRET_KEY"'
|
|
11
|
+
|
|
12
|
+
and read the envelope key out of ``log_entries``, which every principal can read. That is a
|
|
13
|
+
privilege escalation from "can edit pipelines" to "can decrypt every stored connection
|
|
14
|
+
secret", which is precisely the boundary the unsafe-block allowlist exists to hold.
|
|
15
|
+
|
|
16
|
+
So the instance's own variables are refused when the document is validated, and filtered
|
|
17
|
+
again when the environment is built. Both, deliberately: validation is where a person is
|
|
18
|
+
told, and the filter is what holds for a document that never passed validation.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import os
|
|
22
|
+
from collections.abc import Iterable, Mapping
|
|
23
|
+
|
|
24
|
+
from dirigent_block_execute.messages import (
|
|
25
|
+
RESERVED_ENVIRONMENT,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
#: Must stay in step with the prefix ``Settings`` reads.
|
|
29
|
+
RESERVED_ENV_PREFIX = "DIRIGENT_"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def is_reserved(name: str) -> bool:
|
|
33
|
+
"""Report whether an environment variable belongs to the instance rather than the step."""
|
|
34
|
+
return name.upper().startswith(RESERVED_ENV_PREFIX)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def reject_reserved(allowlist: Iterable[str]) -> None:
|
|
38
|
+
"""Refuse an allowlist that reaches for the instance's own configuration."""
|
|
39
|
+
reserved = sorted({name for name in allowlist if is_reserved(name)})
|
|
40
|
+
if reserved:
|
|
41
|
+
raise ValueError(RESERVED_ENVIRONMENT.render(reserved=", ".join(reserved), prefix=RESERVED_ENV_PREFIX))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def allowed(names: Iterable[str], environ: Mapping[str, str] | None = None) -> dict[str, str]:
|
|
45
|
+
"""Read the named variables out of the worker's environment, minus the reserved ones."""
|
|
46
|
+
source = environ if environ is not None else os.environ
|
|
47
|
+
return {name: source[name] for name in names if name in source and not is_reserved(name)}
|
|
@@ -0,0 +1,548 @@
|
|
|
1
|
+
"""``git.checkout``: put a repository at a ref into the run's work directory.
|
|
2
|
+
|
|
3
|
+
Nothing else can bring an existing project into a run. ``storage.copy`` moves one object at a
|
|
4
|
+
time, the compose and build blocks read only what is already in the work directory, and
|
|
5
|
+
``shell.run`` with a ``git clone`` is an unsafe block on a worker that happens to have git and
|
|
6
|
+
a network. This block clones into a directory under the run's work directory and reports the
|
|
7
|
+
commit it landed on, so a downstream ``docker.compose.up`` names its compose file, a
|
|
8
|
+
``docker.build`` names its context, and a transform names its files, all relative to that
|
|
9
|
+
directory.
|
|
10
|
+
|
|
11
|
+
It is an **ordinary** block. It writes only under the run's work directory and reaches only the
|
|
12
|
+
remote its connection names, which is a narrower grant than ``shell.run`` and does not need the
|
|
13
|
+
unsafe allowlist. A checkout is a directory a tool opens, so it lands on the worker's own
|
|
14
|
+
filesystem rather than in storage, and a step that reads it runs on the same worker.
|
|
15
|
+
|
|
16
|
+
The credential never becomes an argument. A token reaches git through a ``GIT_ASKPASS`` helper
|
|
17
|
+
that reads it out of a file in a 0700 directory of its own; an ssh key is a 0600 file that
|
|
18
|
+
``GIT_SSH_COMMAND`` names. The remote is always given to git with any userinfo stripped, so the
|
|
19
|
+
checkout's ``.git/config`` holds no credential either, and the directory holding both goes away
|
|
20
|
+
when the step leaves however it leaves.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import contextlib
|
|
24
|
+
import re
|
|
25
|
+
import shutil
|
|
26
|
+
import tempfile
|
|
27
|
+
from collections.abc import Generator
|
|
28
|
+
from datetime import timedelta
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import ClassVar
|
|
31
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
32
|
+
|
|
33
|
+
from pydantic import BaseModel, Field, SecretStr, model_validator
|
|
34
|
+
|
|
35
|
+
from dirigent_block_execute import secrets, subprocess
|
|
36
|
+
from dirigent_block_execute.capture import log_stream, scrub, tail
|
|
37
|
+
from dirigent_block_execute.messages import (
|
|
38
|
+
CHECKOUT_OUTSIDE_THE_WORK_DIRECTORY,
|
|
39
|
+
CHECKOUT_STAYS_INSIDE,
|
|
40
|
+
CHECKOUT_THROUGH_A_SYMLINK,
|
|
41
|
+
GIT_EXITED,
|
|
42
|
+
GIT_KEY_NEEDS_SSH,
|
|
43
|
+
GIT_ONE_CREDENTIAL,
|
|
44
|
+
GIT_TOKEN_NEEDS_HTTPS,
|
|
45
|
+
NO_GIT,
|
|
46
|
+
)
|
|
47
|
+
from dirigent_common import BlockModel, Duration, HealthReport
|
|
48
|
+
from dirigent_plugin import (
|
|
49
|
+
BlockFailure,
|
|
50
|
+
ConnectionKind,
|
|
51
|
+
ConnectionRef,
|
|
52
|
+
ErrorClass,
|
|
53
|
+
Operator,
|
|
54
|
+
OperatorSpec,
|
|
55
|
+
RemoteHandle,
|
|
56
|
+
StepContext,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
#: How long a connection check may spend listing a remote's refs.
|
|
60
|
+
CHECK_TIMEOUT_SECONDS = 30.0
|
|
61
|
+
|
|
62
|
+
#: A ref written as a full object name, which no clone can resolve by ``--branch``.
|
|
63
|
+
COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$")
|
|
64
|
+
|
|
65
|
+
#: A remote that can carry a token: the credential is HTTP basic auth underneath.
|
|
66
|
+
HTTP_SCHEMES = ("http", "https")
|
|
67
|
+
|
|
68
|
+
#: A remote that can carry a key, either ``ssh://host/path`` or the scp-like ``user@host:path``.
|
|
69
|
+
SSH_SCHEME = "ssh"
|
|
70
|
+
|
|
71
|
+
#: What git says when the remote, rather than the request, is the problem.
|
|
72
|
+
UNREACHABLE = (
|
|
73
|
+
"could not resolve host",
|
|
74
|
+
"connection refused",
|
|
75
|
+
"connection timed out",
|
|
76
|
+
"operation timed out",
|
|
77
|
+
"early eof",
|
|
78
|
+
"the remote end hung up",
|
|
79
|
+
"temporary failure in name resolution",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
#: What git says when the request was understood and refused, or asked for something absent.
|
|
83
|
+
REFUSED = (
|
|
84
|
+
"authentication failed",
|
|
85
|
+
"permission denied",
|
|
86
|
+
"invalid username or password",
|
|
87
|
+
"could not read username",
|
|
88
|
+
"terminal prompts disabled",
|
|
89
|
+
"repository not found",
|
|
90
|
+
"remote branch",
|
|
91
|
+
"did not match any file",
|
|
92
|
+
"couldn't find remote ref",
|
|
93
|
+
"not our ref",
|
|
94
|
+
"access denied",
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
#: The askpass helper git runs. It is asked for a username first and a password second, and
|
|
98
|
+
#: answers each out of a file beside it so neither ever reaches an argument list or an
|
|
99
|
+
#: environment variable.
|
|
100
|
+
ASKPASS = """#!/bin/sh
|
|
101
|
+
case "$1" in
|
|
102
|
+
Username*) exec cat "{directory}/username" ;;
|
|
103
|
+
*) exec cat "{directory}/password" ;;
|
|
104
|
+
esac
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class GitConnectionConfig(BlockModel):
|
|
109
|
+
"""One remote repository and the credential, if any, that reaches it."""
|
|
110
|
+
|
|
111
|
+
url: str = Field(min_length=1)
|
|
112
|
+
"""The remote, in https form (``https://host/owner/repo.git``) or ssh form
|
|
113
|
+
(``ssh://git@host/owner/repo.git``, or the scp-like ``git@host:owner/repo.git``).
|
|
114
|
+
|
|
115
|
+
Any userinfo written into it is stripped before git is given it, so a credential belongs
|
|
116
|
+
in ``token`` or ``ssh_key`` rather than in the URL."""
|
|
117
|
+
|
|
118
|
+
username: str = "x-access-token"
|
|
119
|
+
"""The user half of the token credential. GitHub and GitLab both accept this literal for a
|
|
120
|
+
personal or deploy token, so a connection that has only a token needs nothing else."""
|
|
121
|
+
|
|
122
|
+
token: SecretStr | None = None
|
|
123
|
+
"""A personal, deploy or app token, presented as the password half of HTTP basic auth.
|
|
124
|
+
|
|
125
|
+
Sealed like every other connection secret: encrypted at rest, redacted in every API
|
|
126
|
+
response, and reaching git only through a file the askpass helper reads."""
|
|
127
|
+
|
|
128
|
+
ssh_key: SecretStr | None = None
|
|
129
|
+
"""A private key in OpenSSH form, for an ssh remote.
|
|
130
|
+
|
|
131
|
+
Written to a 0600 file for the length of one step and removed afterwards. A key with a
|
|
132
|
+
passphrase cannot be used: nothing here can answer the prompt."""
|
|
133
|
+
|
|
134
|
+
known_hosts: str | None = None
|
|
135
|
+
"""``known_hosts`` lines pinning the host's key, one per line, as ``ssh-keyscan`` prints them.
|
|
136
|
+
|
|
137
|
+
Given, the host key must match one of them. Left unset, the first key seen is accepted
|
|
138
|
+
(``StrictHostKeyChecking=accept-new``), which trusts the first connection."""
|
|
139
|
+
|
|
140
|
+
@model_validator(mode="after")
|
|
141
|
+
def _check_shape(self) -> "GitConnectionConfig":
|
|
142
|
+
"""Refuse two credentials at once, and a credential that does not fit the remote's form."""
|
|
143
|
+
if self.token is not None and self.ssh_key is not None:
|
|
144
|
+
raise ValueError(GIT_ONE_CREDENTIAL.render())
|
|
145
|
+
scheme = urlsplit(self.url).scheme
|
|
146
|
+
if self.token is not None and scheme not in HTTP_SCHEMES:
|
|
147
|
+
raise ValueError(GIT_TOKEN_NEEDS_HTTPS.render(url=repr(self.url)))
|
|
148
|
+
if self.ssh_key is not None and not _is_ssh(self.url):
|
|
149
|
+
raise ValueError(GIT_KEY_NEEDS_SSH.render(url=repr(self.url)))
|
|
150
|
+
return self
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class GitConnectionKind(ConnectionKind):
|
|
154
|
+
"""The connection kind ``git.checkout`` resolves its remote and credential through."""
|
|
155
|
+
|
|
156
|
+
id: ClassVar[str] = "git"
|
|
157
|
+
config_model: ClassVar[type[BaseModel]] = GitConnectionConfig
|
|
158
|
+
|
|
159
|
+
async def check(self, config: BaseModel) -> HealthReport:
|
|
160
|
+
"""List the remote's refs, which is the smallest thing that proves reach and credential."""
|
|
161
|
+
settings = GitConnectionConfig.model_validate(config.model_dump())
|
|
162
|
+
binary = shutil.which("git")
|
|
163
|
+
if binary is None:
|
|
164
|
+
return HealthReport(healthy=False, detail="git is not on this host's PATH")
|
|
165
|
+
with tempfile.TemporaryDirectory(prefix="dirigent-git-check-") as home:
|
|
166
|
+
root = Path(home)
|
|
167
|
+
with credentials(settings, root) as sealed:
|
|
168
|
+
environ = {**subprocess.environment([], {}, root), **sealed.environ}
|
|
169
|
+
try:
|
|
170
|
+
code, out, err = await subprocess.output(
|
|
171
|
+
argv=[binary, "ls-remote", "--heads", "--", public_url(settings.url)],
|
|
172
|
+
directory=root,
|
|
173
|
+
environ=environ,
|
|
174
|
+
timeout_seconds=CHECK_TIMEOUT_SECONDS,
|
|
175
|
+
what="git ls-remote",
|
|
176
|
+
)
|
|
177
|
+
except BlockFailure as error:
|
|
178
|
+
return HealthReport(healthy=False, detail=scrub(str(error), sealed.secrets))
|
|
179
|
+
if code != 0:
|
|
180
|
+
detail = _first_line(err) or f"git exited {code}"
|
|
181
|
+
return HealthReport(healthy=False, detail=scrub(detail, sealed.secrets))
|
|
182
|
+
return HealthReport(healthy=True, detail=f"{len(_first_lines(out))} branches")
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
class GitCheckoutConfig(BlockModel):
|
|
186
|
+
"""Which repository to put where, at what ref, and how much of its history."""
|
|
187
|
+
|
|
188
|
+
connection: ConnectionRef
|
|
189
|
+
"""The ``git`` connection naming the remote and holding its credential."""
|
|
190
|
+
|
|
191
|
+
ref: str | None = None
|
|
192
|
+
"""The branch, tag or full commit sha to land on. Unset takes the remote's default branch."""
|
|
193
|
+
|
|
194
|
+
target: str = ""
|
|
195
|
+
"""The directory the checkout lands in, inside the run's work directory; never absolute,
|
|
196
|
+
never climbing out. Empty takes the step's own name, so a step named ``checkout`` writes
|
|
197
|
+
``checkout/`` and a downstream ``docker.build`` names ``checkout`` as its context."""
|
|
198
|
+
|
|
199
|
+
depth: int = Field(default=1, ge=0)
|
|
200
|
+
"""How many commits of history to fetch. ``1`` is a shallow checkout of the ref alone,
|
|
201
|
+
which is what a build wants; ``0`` fetches the whole history, which a step reading the log
|
|
202
|
+
or describing a tag needs."""
|
|
203
|
+
|
|
204
|
+
submodules: bool = False
|
|
205
|
+
"""Also check out the repository's submodules, recursively."""
|
|
206
|
+
|
|
207
|
+
timeout: Duration = timedelta(minutes=10)
|
|
208
|
+
"""The deadline on each git invocation, after which it is killed as transient."""
|
|
209
|
+
|
|
210
|
+
@model_validator(mode="after")
|
|
211
|
+
def _check_shape(self) -> "GitCheckoutConfig":
|
|
212
|
+
"""Refuse a target that is absolute or climbs out of the run's work directory."""
|
|
213
|
+
if self.target and (Path(self.target).is_absolute() or ".." in Path(self.target).parts):
|
|
214
|
+
raise ValueError(CHECKOUT_STAYS_INSIDE.render())
|
|
215
|
+
return self
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
class GitCheckoutOutput(BlockModel):
|
|
219
|
+
"""Where the working tree is and what is in it, for the steps that read it."""
|
|
220
|
+
|
|
221
|
+
commit: str
|
|
222
|
+
"""The full sha the checkout landed on, which is the only exact name for what was built."""
|
|
223
|
+
|
|
224
|
+
ref: str
|
|
225
|
+
"""The ref that was asked for, or the default branch the clone landed on when none was."""
|
|
226
|
+
|
|
227
|
+
target: str
|
|
228
|
+
"""The checkout's directory, relative to the run's work directory, as a downstream block names it."""
|
|
229
|
+
|
|
230
|
+
remote: str
|
|
231
|
+
"""The remote it came from, with any credential stripped."""
|
|
232
|
+
|
|
233
|
+
stdout_uri: str = ""
|
|
234
|
+
"""Where the whole of the fetching command's stdout was written; empty when the checkout was
|
|
235
|
+
already standing and nothing was fetched."""
|
|
236
|
+
|
|
237
|
+
stderr_uri: str = ""
|
|
238
|
+
"""Where the whole of the fetching command's stderr was written, which is where git prints
|
|
239
|
+
what it is doing; empty when the checkout was already standing and nothing was fetched."""
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
class GitCheckoutOperator(Operator[GitCheckoutConfig, GitCheckoutOutput]):
|
|
243
|
+
"""Clones a repository at a ref into the run's work directory and reports the commit."""
|
|
244
|
+
|
|
245
|
+
spec = OperatorSpec(
|
|
246
|
+
id="git.checkout",
|
|
247
|
+
summary="Check a repository out into the run's work directory.",
|
|
248
|
+
idempotent=True,
|
|
249
|
+
)
|
|
250
|
+
config_model: ClassVar[type[BaseModel]] = GitCheckoutConfig
|
|
251
|
+
output_model: ClassVar[type[BaseModel]] = GitCheckoutOutput
|
|
252
|
+
|
|
253
|
+
async def execute(self, config: GitCheckoutConfig, ctx: StepContext) -> GitCheckoutOutput | RemoteHandle:
|
|
254
|
+
"""Resolve the connection, land the working tree, and report what is in it."""
|
|
255
|
+
root = ctx.work
|
|
256
|
+
settings = ctx.connection(config.connection, GitConnectionConfig)
|
|
257
|
+
target = config.target or ctx.step
|
|
258
|
+
destination = _destination(root, target)
|
|
259
|
+
remote = public_url(settings.url)
|
|
260
|
+
binary = shutil.which("git", path=subprocess.environment([], {}, root).get("PATH"))
|
|
261
|
+
if binary is None:
|
|
262
|
+
raise BlockFailure(NO_GIT, error_class=ErrorClass.REJECTED)
|
|
263
|
+
|
|
264
|
+
with credentials(settings, root) as sealed:
|
|
265
|
+
environ = {**subprocess.environment([], {}, root), **sealed.environ}
|
|
266
|
+
git = _Git(binary, environ, config.timeout.total_seconds(), sealed.secrets, ctx)
|
|
267
|
+
standing = await _standing(git, config, destination, remote)
|
|
268
|
+
if not standing:
|
|
269
|
+
_clear(destination)
|
|
270
|
+
await _land(git, config, destination, remote)
|
|
271
|
+
commit = await git.read(destination, "rev-parse", "HEAD")
|
|
272
|
+
landed = config.ref or await git.read(destination, "rev-parse", "--abbrev-ref", "HEAD")
|
|
273
|
+
|
|
274
|
+
message = "checkout already there" if standing else "checked out"
|
|
275
|
+
ctx.log.info(message, commit=commit, ref=landed, target=target, remote=remote)
|
|
276
|
+
return GitCheckoutOutput(
|
|
277
|
+
commit=commit,
|
|
278
|
+
ref=landed,
|
|
279
|
+
target=target,
|
|
280
|
+
remote=remote,
|
|
281
|
+
stdout_uri=git.stdout_uri,
|
|
282
|
+
stderr_uri=git.stderr_uri,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class Sealed:
|
|
287
|
+
"""What one step's credential became: an environment for git, and the strings to scrub."""
|
|
288
|
+
|
|
289
|
+
def __init__(self, environ: dict[str, str], secrets: list[str]) -> None:
|
|
290
|
+
"""Hold the variables git is given and the values that must never be logged."""
|
|
291
|
+
self.environ = environ
|
|
292
|
+
self.secrets = secrets
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
@contextlib.contextmanager
|
|
296
|
+
def credentials(settings: GitConnectionConfig, parent: Path) -> Generator[Sealed]:
|
|
297
|
+
"""Materialise the connection's credential into a private directory, and take it away again.
|
|
298
|
+
|
|
299
|
+
The directory is 0700 and holds a 0700 askpass helper and its two 0600 answers, or a 0600
|
|
300
|
+
private key and the host keys to check against. Nothing in it outlives the step: the
|
|
301
|
+
directory goes whether the checkout succeeded, failed, or was cancelled.
|
|
302
|
+
"""
|
|
303
|
+
with secrets.private_directory(parent, "dirigent-git-") as directory:
|
|
304
|
+
yield _materialise(settings, directory)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _materialise(settings: GitConnectionConfig, directory: Path) -> Sealed:
|
|
308
|
+
"""Write the credential's files and name the environment that points git at them."""
|
|
309
|
+
# git never asks a terminal for anything: there is none, and a prompt would hang the step
|
|
310
|
+
# until its deadline instead of failing it with what went wrong.
|
|
311
|
+
environ = {"GIT_TERMINAL_PROMPT": "0", "GIT_CONFIG_NOSYSTEM": "1"}
|
|
312
|
+
hidden: list[str] = []
|
|
313
|
+
if settings.token is not None:
|
|
314
|
+
token = settings.token.get_secret_value()
|
|
315
|
+
secrets.write(directory / "username", settings.username)
|
|
316
|
+
secrets.write(directory / "password", token)
|
|
317
|
+
helper = directory / "askpass.sh"
|
|
318
|
+
secrets.write(helper, ASKPASS.format(directory=directory), secrets.EXECUTABLE)
|
|
319
|
+
environ["GIT_ASKPASS"] = str(helper)
|
|
320
|
+
hidden.append(token)
|
|
321
|
+
if settings.ssh_key is not None:
|
|
322
|
+
key = directory / "id"
|
|
323
|
+
secrets.write(key, settings.ssh_key.get_secret_value().rstrip("\n") + "\n")
|
|
324
|
+
options = ["-i", str(key), "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes"]
|
|
325
|
+
if settings.known_hosts is not None:
|
|
326
|
+
hosts = directory / "known_hosts"
|
|
327
|
+
secrets.write(hosts, settings.known_hosts.rstrip("\n") + "\n")
|
|
328
|
+
options += ["-o", "StrictHostKeyChecking=yes", "-o", f"UserKnownHostsFile={hosts}"]
|
|
329
|
+
else:
|
|
330
|
+
options += ["-o", "StrictHostKeyChecking=accept-new"]
|
|
331
|
+
environ["GIT_SSH_COMMAND"] = " ".join(["ssh", *(_quote(one) for one in options)])
|
|
332
|
+
hidden += [settings.ssh_key.get_secret_value(), str(key)]
|
|
333
|
+
return Sealed(environ, hidden)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
class _Git:
|
|
337
|
+
"""One repository's git, with the binary, the environment and the redaction already bound."""
|
|
338
|
+
|
|
339
|
+
def __init__(
|
|
340
|
+
self, binary: str, environ: dict[str, str], timeout: float, secrets: list[str], ctx: StepContext
|
|
341
|
+
) -> None:
|
|
342
|
+
"""Hold what every invocation of this step's git shares."""
|
|
343
|
+
self.binary = binary
|
|
344
|
+
self.environ = environ
|
|
345
|
+
self.timeout = timeout
|
|
346
|
+
self.secrets = secrets
|
|
347
|
+
self.ctx = ctx
|
|
348
|
+
self.stdout_uri = ""
|
|
349
|
+
self.stderr_uri = ""
|
|
350
|
+
|
|
351
|
+
async def call(self, directory: Path, *argv: str) -> tuple[int, bytes, bytes]:
|
|
352
|
+
"""Run one git command in a directory and hand back its exit code and both streams."""
|
|
353
|
+
code, out, err = await subprocess.output(
|
|
354
|
+
argv=[self.binary, *argv],
|
|
355
|
+
directory=directory,
|
|
356
|
+
environ=self.environ,
|
|
357
|
+
timeout_seconds=self.timeout,
|
|
358
|
+
what=f"git {argv[0]}",
|
|
359
|
+
)
|
|
360
|
+
for line in _first_lines(err):
|
|
361
|
+
self.ctx.log.debug(scrub(line, self.secrets), stream="stderr")
|
|
362
|
+
return code, out, err
|
|
363
|
+
|
|
364
|
+
async def run(self, directory: Path, *argv: str) -> bytes:
|
|
365
|
+
"""Run one git command that must succeed, failing the step with git's own words if it does not."""
|
|
366
|
+
code, out, err = await self.call(directory, *argv)
|
|
367
|
+
if code != 0:
|
|
368
|
+
detail = tail(err, redact=self.secrets) or tail(out, redact=self.secrets) or "no output"
|
|
369
|
+
raise BlockFailure(GIT_EXITED, error_class=classify(err), command=argv[0], code=code, detail=detail)
|
|
370
|
+
return out
|
|
371
|
+
|
|
372
|
+
async def read(self, directory: Path, *argv: str) -> str:
|
|
373
|
+
"""Run one git command that prints a single value, and return that value."""
|
|
374
|
+
return (await self.run(directory, *argv)).decode("utf-8", errors="replace").strip()
|
|
375
|
+
|
|
376
|
+
async def stream(self, directory: Path, *argv: str) -> None:
|
|
377
|
+
"""Run one git command that talks to the remote, draining both streams to storage as it prints.
|
|
378
|
+
|
|
379
|
+
A clone of a large repository prints for minutes, so its lines reach the run log where
|
|
380
|
+
they happen rather than when the process exits, and the whole of each stream is an
|
|
381
|
+
artifact rather than a buffer in the worker. Each command writes its own pair, named for
|
|
382
|
+
its subcommand; the pair the last one wrote is what the output carries.
|
|
383
|
+
"""
|
|
384
|
+
artifacts = f"{subprocess.prefix(self.ctx, 'git')}-{argv[0]}"
|
|
385
|
+
stdout_uri = f"{artifacts}-stdout.txt"
|
|
386
|
+
stderr_uri = f"{artifacts}-stderr.txt"
|
|
387
|
+
code, out, err = await subprocess.run(
|
|
388
|
+
directory=directory,
|
|
389
|
+
ctx=self.ctx,
|
|
390
|
+
stdout_uri=stdout_uri,
|
|
391
|
+
stderr_uri=stderr_uri,
|
|
392
|
+
timeout_seconds=self.timeout,
|
|
393
|
+
environ=self.environ,
|
|
394
|
+
argv=[self.binary, *argv],
|
|
395
|
+
what=f"git {argv[0]}",
|
|
396
|
+
redact=self.secrets,
|
|
397
|
+
)
|
|
398
|
+
self.stdout_uri = stdout_uri
|
|
399
|
+
self.stderr_uri = stderr_uri
|
|
400
|
+
log_stream(self.ctx, "stdout", out, self.secrets)
|
|
401
|
+
log_stream(self.ctx, "stderr", err, self.secrets)
|
|
402
|
+
if code != 0:
|
|
403
|
+
detail = tail(err.tail, redact=self.secrets) or tail(out.tail, redact=self.secrets) or "no output"
|
|
404
|
+
raise BlockFailure(GIT_EXITED, error_class=classify(err.tail), command=argv[0], code=code, detail=detail)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
async def _standing(git: _Git, config: GitCheckoutConfig, destination: Path, remote: str) -> bool:
|
|
408
|
+
"""Whether the target already holds a working tree at the commit being asked for.
|
|
409
|
+
|
|
410
|
+
A retry of a step, or a second run into a work directory that already holds the checkout,
|
|
411
|
+
should cost one ref listing rather than a second clone. Anything that is not a working
|
|
412
|
+
tree at the wanted commit answers false, and the target is then rebuilt from scratch.
|
|
413
|
+
|
|
414
|
+
A target that is itself a symlink is never standing: what it points at is some other
|
|
415
|
+
directory, so it is unlinked and replaced rather than looked into.
|
|
416
|
+
"""
|
|
417
|
+
if destination.is_symlink() or not (destination / ".git").exists():
|
|
418
|
+
return False
|
|
419
|
+
code, out, _err = await git.call(destination, "rev-parse", "HEAD")
|
|
420
|
+
if code != 0:
|
|
421
|
+
return False
|
|
422
|
+
head = out.decode("utf-8", errors="replace").strip()
|
|
423
|
+
if config.ref is not None and COMMIT_SHA.match(config.ref):
|
|
424
|
+
return head == config.ref
|
|
425
|
+
code, out, _err = await git.call(destination, "ls-remote", "--", remote, config.ref or "HEAD")
|
|
426
|
+
if code != 0:
|
|
427
|
+
return False
|
|
428
|
+
lines = _first_lines(out)
|
|
429
|
+
wanted = lines[0].split("\t", 1)[0] if lines else ""
|
|
430
|
+
return bool(wanted) and head == wanted
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
async def _land(git: _Git, config: GitCheckoutConfig, destination: Path, remote: str) -> None:
|
|
434
|
+
"""Put the working tree at the ref, by whichever of git's two routes reaches it.
|
|
435
|
+
|
|
436
|
+
A branch or tag is what ``clone --branch`` takes. A commit sha is not: the ref namespace
|
|
437
|
+
has no entry for it, so the repository is created empty, the one commit is fetched into
|
|
438
|
+
``FETCH_HEAD``, and the working tree is detached onto that.
|
|
439
|
+
"""
|
|
440
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
441
|
+
if config.ref is not None and COMMIT_SHA.match(config.ref):
|
|
442
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
443
|
+
await git.run(destination, "init", "--quiet")
|
|
444
|
+
await git.run(destination, "remote", "add", "origin", "--", remote)
|
|
445
|
+
await git.stream(destination, "fetch", *_depth(config), "origin", config.ref)
|
|
446
|
+
await git.run(destination, "checkout", "--quiet", "--detach", "FETCH_HEAD")
|
|
447
|
+
if config.submodules:
|
|
448
|
+
await git.stream(destination, "submodule", "update", "--init", "--recursive", *_depth(config))
|
|
449
|
+
return
|
|
450
|
+
argv = ["clone", *_depth(config)]
|
|
451
|
+
if config.ref is not None:
|
|
452
|
+
argv += ["--branch", config.ref]
|
|
453
|
+
if config.submodules:
|
|
454
|
+
argv += ["--recurse-submodules"]
|
|
455
|
+
await git.stream(destination.parent, *argv, "--", remote, str(destination))
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def _depth(config: GitCheckoutConfig) -> list[str]:
|
|
459
|
+
"""``--depth`` when the checkout is shallow, and nothing at all when it is not."""
|
|
460
|
+
return ["--depth", str(config.depth)] if config.depth > 0 else []
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def public_url(url: str) -> str:
|
|
464
|
+
"""The remote with any userinfo removed, which is what git and the output are both given.
|
|
465
|
+
|
|
466
|
+
A URL that carries its own credential would put it in the checkout's ``.git/config``,
|
|
467
|
+
where it outlives the step and reaches every later reader of the work directory.
|
|
468
|
+
"""
|
|
469
|
+
split = urlsplit(url)
|
|
470
|
+
if not split.scheme or "@" not in split.netloc:
|
|
471
|
+
return url
|
|
472
|
+
return urlunsplit(split._replace(netloc=split.netloc.rsplit("@", 1)[1]))
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def _is_ssh(url: str) -> bool:
|
|
476
|
+
"""Whether the remote is an ssh one, in either the URL form or the scp-like one."""
|
|
477
|
+
split = urlsplit(url)
|
|
478
|
+
return split.scheme == SSH_SCHEME or (not split.scheme and "@" in url and ":" in url)
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def classify(stderr: bytes) -> ErrorClass:
|
|
482
|
+
"""A remote that could not be reached is transient; one that said no is not."""
|
|
483
|
+
text = stderr.decode("utf-8", errors="replace").lower()
|
|
484
|
+
if any(marker in text for marker in UNREACHABLE):
|
|
485
|
+
return ErrorClass.TRANSIENT
|
|
486
|
+
if any(marker in text for marker in REFUSED):
|
|
487
|
+
return ErrorClass.REJECTED
|
|
488
|
+
return ErrorClass.UNKNOWN
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _destination(root: Path, target: str) -> Path:
|
|
492
|
+
"""Where the checkout lands, once it is certain that path is inside the run's work directory.
|
|
493
|
+
|
|
494
|
+
The config refuses a target that is absolute or carries ``..``, which is a check on the
|
|
495
|
+
text; a symlink an earlier step left in the work directory is not. So every component
|
|
496
|
+
between the root and the target must be a real directory here, and the directory the
|
|
497
|
+
checkout lands in must resolve inside the root, before anything looks at the target,
|
|
498
|
+
clears it or clones into it. The target itself may be a symlink: it is unlinked rather
|
|
499
|
+
than followed.
|
|
500
|
+
"""
|
|
501
|
+
base = root.resolve()
|
|
502
|
+
parts = Path(target).parts
|
|
503
|
+
walked = base
|
|
504
|
+
for part in parts[:-1]:
|
|
505
|
+
walked = walked / part
|
|
506
|
+
if walked.is_symlink():
|
|
507
|
+
raise BlockFailure(
|
|
508
|
+
CHECKOUT_THROUGH_A_SYMLINK,
|
|
509
|
+
error_class=ErrorClass.REJECTED,
|
|
510
|
+
target=repr(target),
|
|
511
|
+
walked=walked,
|
|
512
|
+
base=base,
|
|
513
|
+
)
|
|
514
|
+
destination = walked / parts[-1] if parts else walked
|
|
515
|
+
landing = destination.parent.resolve()
|
|
516
|
+
if landing != base and base not in landing.parents:
|
|
517
|
+
raise BlockFailure(
|
|
518
|
+
CHECKOUT_OUTSIDE_THE_WORK_DIRECTORY,
|
|
519
|
+
error_class=ErrorClass.REJECTED,
|
|
520
|
+
target=repr(target),
|
|
521
|
+
destination=destination,
|
|
522
|
+
base=base,
|
|
523
|
+
)
|
|
524
|
+
return destination
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _clear(destination: Path) -> None:
|
|
528
|
+
"""Empty the target so a clone into it starts from nothing, whatever was there before."""
|
|
529
|
+
if destination.is_symlink() or destination.is_file():
|
|
530
|
+
destination.unlink()
|
|
531
|
+
elif destination.is_dir():
|
|
532
|
+
shutil.rmtree(destination)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _quote(value: str) -> str:
|
|
536
|
+
"""Quote one ssh option for the command line git splits ``GIT_SSH_COMMAND`` into."""
|
|
537
|
+
return value if re.fullmatch(r"[\w@%+=:,./-]+", value) else "'" + value.replace("'", "'\\''") + "'"
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _first_line(payload: bytes) -> str:
|
|
541
|
+
"""The first non-blank line of a stream, which is the sentence a health report wants."""
|
|
542
|
+
lines = _first_lines(payload)
|
|
543
|
+
return lines[0] if lines else ""
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def _first_lines(payload: bytes) -> list[str]:
|
|
547
|
+
"""Every non-blank line of a captured stream."""
|
|
548
|
+
return [line for line in payload.decode("utf-8", errors="replace").splitlines() if line.strip()]
|