autoforge-cli 0.1.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autoforge/__init__.py +5 -0
- autoforge/agent_worktree.py +653 -0
- autoforge/certificate.py +495 -0
- autoforge/child_env.py +871 -0
- autoforge/claims.py +726 -0
- autoforge/claude_stream.py +501 -0
- autoforge/cli.py +2884 -0
- autoforge/config.py +2178 -0
- autoforge/config_fingerprint.py +374 -0
- autoforge/doctor.py +1551 -0
- autoforge/doctor_sandbox.py +691 -0
- autoforge/effect_ops.py +1272 -0
- autoforge/effects.py +3210 -0
- autoforge/engine.py +16637 -0
- autoforge/epic_commands.py +1486 -0
- autoforge/epic_graph.py +852 -0
- autoforge/errors.py +536 -0
- autoforge/evidence.py +1722 -0
- autoforge/executor.py +1601 -0
- autoforge/executor_duplex.py +985 -0
- autoforge/executor_supervised.py +144 -0
- autoforge/git_transport.py +970 -0
- autoforge/github.py +3402 -0
- autoforge/interaction.py +1027 -0
- autoforge/local_workspace.py +1253 -0
- autoforge/locking.py +1050 -0
- autoforge/loop_detect.py +660 -0
- autoforge/loop_guard.py +404 -0
- autoforge/materialize.py +722 -0
- autoforge/ownership.py +240 -0
- autoforge/pi_rpc.py +713 -0
- autoforge/premerge.py +181 -0
- autoforge/profiles.py +106 -0
- autoforge/progress.py +392 -0
- autoforge/prompts/__init__.py +173 -0
- autoforge/prompts/acceptance_actions.md +39 -0
- autoforge/prompts/analyze_execute.md +128 -0
- autoforge/prompts/common.md +132 -0
- autoforge/prompts/correction.md +12 -0
- autoforge/prompts/fix.md +252 -0
- autoforge/prompts/local_analyze_execute.md +69 -0
- autoforge/prompts/local_common.md +149 -0
- autoforge/prompts/local_fix.md +96 -0
- autoforge/prompts/local_review.md +103 -0
- autoforge/prompts/operator_acceptance.md +61 -0
- autoforge/prompts/replan_reexecute.md +991 -0
- autoforge/prompts/review.md +312 -0
- autoforge/prompts/update_epic.md +120 -0
- autoforge/prompts/update_epic_rerequest.md +85 -0
- autoforge/prompts/validation_evidence.md +26 -0
- autoforge/prompts/validation_failure.md +22 -0
- autoforge/prompts/worktree_state.md +45 -0
- autoforge/providers.py +2893 -0
- autoforge/read_context.py +389 -0
- autoforge/redaction.py +408 -0
- autoforge/replan.py +208 -0
- autoforge/replan_txn.py +1930 -0
- autoforge/result_parser.py +2234 -0
- autoforge/roadmap.py +151 -0
- autoforge/run_contract.py +408 -0
- autoforge/run_namespace.py +839 -0
- autoforge/runlog.py +764 -0
- autoforge/safefs.py +1717 -0
- autoforge/sandbox.py +1313 -0
- autoforge/sandbox_bwrap.py +815 -0
- autoforge/sandbox_pin.py +170 -0
- autoforge/sandbox_policy.py +1107 -0
- autoforge/scheduler.py +1143 -0
- autoforge/scheduler_state.py +2316 -0
- autoforge/scheduler_stop.py +421 -0
- autoforge/sdk_policy.py +988 -0
- autoforge/state.py +2417 -0
- autoforge/transitions.py +388 -0
- autoforge/validation.py +324 -0
- autoforge/worker.py +568 -0
- autoforge_cli-0.1.0a0.dist-info/METADATA +525 -0
- autoforge_cli-0.1.0a0.dist-info/RECORD +81 -0
- autoforge_cli-0.1.0a0.dist-info/WHEEL +5 -0
- autoforge_cli-0.1.0a0.dist-info/entry_points.txt +2 -0
- autoforge_cli-0.1.0a0.dist-info/licenses/LICENSE +21 -0
- autoforge_cli-0.1.0a0.dist-info/top_level.txt +1 -0
autoforge/evidence.py
ADDED
|
@@ -0,0 +1,1722 @@
|
|
|
1
|
+
"""Candidate validation evidence: definitions, policy, records (ADR 0006, #172).
|
|
2
|
+
|
|
3
|
+
The pure part of candidate validation. A **definition** is one trusted argv
|
|
4
|
+
the operator configured (``validation.commands`` for REMOTE, the run
|
|
5
|
+
contract's ``local.validation_commands`` for LOCAL), bound by a
|
|
6
|
+
``definition_digest`` over what runs (D3.5). **Policy P** is how it runs: the
|
|
7
|
+
sandbox mode, a path-independent template of the outer sandbox, the
|
|
8
|
+
environment allow-list's names, the timeout, ``contain_orphans`` and the
|
|
9
|
+
materialization method, bound by a ``policy_digest`` (D8.1). A **candidate**
|
|
10
|
+
is exactly one of two identities, :class:`RemoteCandidate` (repository,
|
|
11
|
+
object format, commit and tree, plus its provenance) or
|
|
12
|
+
:class:`LocalCandidate` (workspace fingerprint, workspace-policy digest and
|
|
13
|
+
run id), under a ``kind`` discriminator (D2.1-D2.7).
|
|
14
|
+
|
|
15
|
+
A ``ValidationRecord`` is what the controller observed when it ran one
|
|
16
|
+
definition against one candidate under one policy (D10.1). It has one of
|
|
17
|
+
two shapes, chosen by its ``outcome``:
|
|
18
|
+
|
|
19
|
+
- **observed** (``PASS``, ``FAIL``, ``TIMEOUT``): the enforcement facts the
|
|
20
|
+
executor returned, the exit status, the timing, ADR 0002's leftovers and
|
|
21
|
+
the bounded output, and for LOCAL the fingerprints the walks before and
|
|
22
|
+
after the command read;
|
|
23
|
+
- **interrupted** (``INTERRUPTED``): built by the resume path from a
|
|
24
|
+
leftover attempt marker alone (D10.7). It claims no launch and is never
|
|
25
|
+
evidence.
|
|
26
|
+
|
|
27
|
+
A launch, setup or materialization failure is no record at all: it is
|
|
28
|
+
inconclusive (D10.5), and the caller re-checks within its bound.
|
|
29
|
+
|
|
30
|
+
Everything here is persisted in ``state.json`` (D11.1), validated on load
|
|
31
|
+
field by field against the bounds of D10.3, digest-checked, and checked for
|
|
32
|
+
referential integrity (:func:`check_evidence`). A value that fails is
|
|
33
|
+
corruption (:class:`~autoforge.errors.StateError`): never clipped, defaulted
|
|
34
|
+
or repaired. The writers build values through this module only, so a
|
|
35
|
+
writer never produces one the loader would refuse. Loading is read-only
|
|
36
|
+
(D11.3): converting a leftover marker (:func:`recover_attempt`) is the
|
|
37
|
+
engine's, under the repository lock.
|
|
38
|
+
|
|
39
|
+
This module does no I/O and knows no provider; the engine launches the
|
|
40
|
+
validators and saves the state.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
import hashlib
|
|
46
|
+
import json
|
|
47
|
+
import re
|
|
48
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
49
|
+
from dataclasses import dataclass, field
|
|
50
|
+
from datetime import datetime
|
|
51
|
+
from enum import StrEnum
|
|
52
|
+
from typing import NoReturn
|
|
53
|
+
|
|
54
|
+
from .errors import (
|
|
55
|
+
ConfigurationError,
|
|
56
|
+
StateError,
|
|
57
|
+
ValidationDefinitionError,
|
|
58
|
+
ValidationInterruptedError,
|
|
59
|
+
)
|
|
60
|
+
from .executor import MAX_DEADLINE_SECONDS, ExecutionResult
|
|
61
|
+
from .materialize import METHOD as METHOD_BYTE_EXACT
|
|
62
|
+
from .redaction import MAX_GROWTH_FACTOR, redact, redact_argv
|
|
63
|
+
from .result_parser import MAX_URL_CHARS
|
|
64
|
+
from .runlog import RUN_ID_RE, STEP_DIR_RE
|
|
65
|
+
from .sandbox import SandboxFacts
|
|
66
|
+
from .validation import parse_comment_url, parse_pr_url
|
|
67
|
+
|
|
68
|
+
# -- bounds (D10.3) --------------------------------------------------------------------
|
|
69
|
+
# The one bound definition for every Wave 2B persisted field; the writers and
|
|
70
|
+
# the loader use the same constants. Once merged they are part of the state
|
|
71
|
+
# protocol: lowering one is a protocol change.
|
|
72
|
+
|
|
73
|
+
SCHEMA = 1
|
|
74
|
+
|
|
75
|
+
# Lists.
|
|
76
|
+
MAX_DEFINITIONS = 16
|
|
77
|
+
MAX_RECORDS = 64
|
|
78
|
+
MAX_BOUND_RECORDS = MAX_DEFINITIONS
|
|
79
|
+
MAX_ARGV_ELEMENTS = 32
|
|
80
|
+
|
|
81
|
+
# An argv as configured, and as a record or a marker stores it: redacted, at
|
|
82
|
+
# most the configured bounds times redaction's growth factor.
|
|
83
|
+
MAX_ARGV_ELEMENT_CHARS = 1024
|
|
84
|
+
MAX_ARGV_CHARS = 4096
|
|
85
|
+
MAX_STORED_ARGV_ELEMENT_CHARS = MAX_ARGV_ELEMENT_CHARS * MAX_GROWTH_FACTOR
|
|
86
|
+
MAX_STORED_ARGV_CHARS = MAX_ARGV_CHARS * MAX_GROWTH_FACTOR
|
|
87
|
+
|
|
88
|
+
# Each stored output tail, after escaping: today's VerificationError tail.
|
|
89
|
+
MAX_TAIL_CHARS = 2000
|
|
90
|
+
|
|
91
|
+
MAX_REPOSITORY_CHARS = 200
|
|
92
|
+
MAX_REF_CHARS = 255
|
|
93
|
+
MAX_WORKTREE_ID_CHARS = 255
|
|
94
|
+
MAX_RUN_LOG_CHARS = 256
|
|
95
|
+
MAX_BACKEND_CHARS = 64
|
|
96
|
+
MAX_TEARDOWN_CHARS = 128
|
|
97
|
+
MAX_TIMESTAMP_CHARS = 32
|
|
98
|
+
|
|
99
|
+
# Integers.
|
|
100
|
+
MAX_SEQ = 2**31 - 1
|
|
101
|
+
MAX_BYTE_COUNT = 2**53 - 1
|
|
102
|
+
MAX_COMMENT_ID = 2**53 - 1
|
|
103
|
+
MIN_EXIT_CODE = -128
|
|
104
|
+
MAX_EXIT_CODE = 255
|
|
105
|
+
# D5.2's bound on a producer's validation failures (#173 counts them).
|
|
106
|
+
MAX_FAILURE_COUNT = 3
|
|
107
|
+
# More INTERRUPTED records than this for one key enters BLOCKED (D10.7).
|
|
108
|
+
MAX_INTERRUPTIONS = 2
|
|
109
|
+
|
|
110
|
+
# Enumerations.
|
|
111
|
+
KIND_REMOTE = "remote"
|
|
112
|
+
KIND_LOCAL = "local"
|
|
113
|
+
PROVENANCE_WORKTREE = "worktree"
|
|
114
|
+
PROVENANCE_PR_HEAD = "pr-head"
|
|
115
|
+
OBJECT_FORMATS = {"sha1": 40, "sha256": 64}
|
|
116
|
+
SCOPE = "candidate-tree"
|
|
117
|
+
CWD_ROLE = "candidate-root"
|
|
118
|
+
MODES = ("disabled", "preferred", "required")
|
|
119
|
+
NETWORKS = ("host", "isolated", "")
|
|
120
|
+
FAILURE_PHASES = ("ANALYZE_EXECUTE", "FIX")
|
|
121
|
+
METHOD_IN_PLACE = "in-place"
|
|
122
|
+
METHODS = (METHOD_BYTE_EXACT, METHOD_IN_PLACE)
|
|
123
|
+
|
|
124
|
+
# The canonical encodings' version labels (D3.5, D8.1).
|
|
125
|
+
DEFINITION_ENCODING = "autoforge-validation-definition-v1"
|
|
126
|
+
SET_ENCODING = "autoforge-validation-set-v1"
|
|
127
|
+
POLICY_ENCODING = "autoforge-validation-policy-v1"
|
|
128
|
+
ALLOWLIST_ENCODING = "autoforge-validation-env-names-v1"
|
|
129
|
+
|
|
130
|
+
# The template under ``sandbox.mode: disabled``: there is no outer sandbox,
|
|
131
|
+
# so there is nothing of one to describe (D8.1).
|
|
132
|
+
NO_SANDBOX_TEMPLATE: Mapping[str, object] = {"outer_sandbox": "none"}
|
|
133
|
+
|
|
134
|
+
_HEX64_RE = re.compile(r"^[0-9a-f]{64}\Z")
|
|
135
|
+
_REPOSITORY_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z")
|
|
136
|
+
_REF_FORBIDDEN_RE = re.compile(r"[\x00-\x20\x7f~^:?*\[\\]|\.\.|@\{|//|/\.|\.lock(?:/|\Z)")
|
|
137
|
+
_COMPONENT_FORBIDDEN_RE = re.compile(r"[/\x00-\x1f\x7f-\x9f]")
|
|
138
|
+
# What a stored tail never holds unescaped: every C0 and C1 control
|
|
139
|
+
# character but newline and tab, DEL, and the two Unicode line separators.
|
|
140
|
+
_UNESCAPED_RE = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f\u2028\u2029]")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class Outcome(StrEnum):
|
|
144
|
+
"""D10.4. ``INTERRUPTED`` is never observed: only recovery writes it."""
|
|
145
|
+
|
|
146
|
+
PASS = "PASS"
|
|
147
|
+
FAIL = "FAIL"
|
|
148
|
+
TIMEOUT = "TIMEOUT"
|
|
149
|
+
INTERRUPTED = "INTERRUPTED"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
OBSERVED_OUTCOMES = (Outcome.PASS, Outcome.FAIL, Outcome.TIMEOUT)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# -- canonical encoding ----------------------------------------------------------------
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def canonical_json(value: object) -> str:
|
|
159
|
+
"""Sorted keys, no whitespace, ASCII: the encoding every digest here is over."""
|
|
160
|
+
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _sha256(value: object) -> str:
|
|
164
|
+
return hashlib.sha256(canonical_json(value).encode("ascii")).hexdigest()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# -- definitions (D3.1-D3.6, D4.2) -----------------------------------------------------
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def definition_problem(commands: Sequence[Sequence[str]]) -> str:
|
|
171
|
+
"""Why ``commands`` cannot be a validation set (D10.3), or "".
|
|
172
|
+
|
|
173
|
+
Names the definition by its index and the bound it is beyond, never the
|
|
174
|
+
argv (it may hold what the operator should not have put there).
|
|
175
|
+
"""
|
|
176
|
+
if len(commands) > MAX_DEFINITIONS:
|
|
177
|
+
return f"{len(commands)} definitions, over the bound of {MAX_DEFINITIONS} per set"
|
|
178
|
+
for index, argv in enumerate(commands):
|
|
179
|
+
if not argv:
|
|
180
|
+
return f"definition [{index}] is empty"
|
|
181
|
+
if len(argv) > MAX_ARGV_ELEMENTS:
|
|
182
|
+
return (
|
|
183
|
+
f"definition [{index}] has {len(argv)} argv elements, over the bound of "
|
|
184
|
+
f"{MAX_ARGV_ELEMENTS}"
|
|
185
|
+
)
|
|
186
|
+
for position, element in enumerate(argv):
|
|
187
|
+
if not isinstance(element, str) or not element.strip():
|
|
188
|
+
return f"definition [{index}] element {position} is blank"
|
|
189
|
+
if len(element) > MAX_ARGV_ELEMENT_CHARS:
|
|
190
|
+
return (
|
|
191
|
+
f"definition [{index}] element {position} is {len(element)} characters, "
|
|
192
|
+
f"over the bound of {MAX_ARGV_ELEMENT_CHARS}"
|
|
193
|
+
)
|
|
194
|
+
total = sum(len(element) for element in argv)
|
|
195
|
+
if total > MAX_ARGV_CHARS:
|
|
196
|
+
return (
|
|
197
|
+
f"definition [{index}] is {total} characters in all, over the bound of "
|
|
198
|
+
f"{MAX_ARGV_CHARS}"
|
|
199
|
+
)
|
|
200
|
+
return ""
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def check_definitions(commands: Sequence[Sequence[str]]) -> None:
|
|
204
|
+
"""Raise :class:`ValidationDefinitionError` when ``commands`` is beyond D10.3."""
|
|
205
|
+
problem = definition_problem(commands)
|
|
206
|
+
if problem:
|
|
207
|
+
raise ValidationDefinitionError(problem)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def check_scope(scope: object) -> None:
|
|
211
|
+
"""D4.2: ``candidate-tree`` is the only scope Wave 2B implements."""
|
|
212
|
+
if scope != SCOPE:
|
|
213
|
+
raise ConfigurationError(
|
|
214
|
+
f"validation.scope must be {SCOPE!r}, the only scope there is, got {scope!r}: "
|
|
215
|
+
"base-bound and merge-result-bound validation do not exist"
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def definition_digest(argv: Sequence[str], index: int) -> str:
|
|
220
|
+
"""D3.5: SHA-256 over what runs, the configured (unredacted) argv included."""
|
|
221
|
+
return _sha256(
|
|
222
|
+
{
|
|
223
|
+
"version": DEFINITION_ENCODING,
|
|
224
|
+
"argv": list(argv),
|
|
225
|
+
"cwd": CWD_ROLE,
|
|
226
|
+
"scope": SCOPE,
|
|
227
|
+
"index": index,
|
|
228
|
+
}
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def set_digest(digests: Sequence[str]) -> str:
|
|
233
|
+
"""D3.5, D3.6: the ordered definition digests; the empty set has one too."""
|
|
234
|
+
return _sha256({"version": SET_ENCODING, "definitions": list(digests)})
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
@dataclass(frozen=True)
|
|
238
|
+
class Definition:
|
|
239
|
+
"""One trusted definition of a set, at its index (D3.5)."""
|
|
240
|
+
|
|
241
|
+
index: int
|
|
242
|
+
argv: tuple[str, ...]
|
|
243
|
+
|
|
244
|
+
@property
|
|
245
|
+
def digest(self) -> str:
|
|
246
|
+
return definition_digest(self.argv, self.index)
|
|
247
|
+
|
|
248
|
+
@property
|
|
249
|
+
def shown(self) -> str:
|
|
250
|
+
"""The argv as it may be displayed: redacted, space-joined."""
|
|
251
|
+
return " ".join(redact_argv(list(self.argv)))
|
|
252
|
+
|
|
253
|
+
def stored(self) -> dict:
|
|
254
|
+
"""The form a record and a marker hold: the argv redacted."""
|
|
255
|
+
return {
|
|
256
|
+
"index": self.index,
|
|
257
|
+
"argv": redact_argv(list(self.argv)),
|
|
258
|
+
"definition_digest": self.digest,
|
|
259
|
+
"scope": SCOPE,
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def definitions(commands: Sequence[Sequence[str]]) -> tuple[Definition, ...]:
|
|
264
|
+
"""The set ``commands`` configures, checked against D10.3."""
|
|
265
|
+
check_definitions(commands)
|
|
266
|
+
return tuple(Definition(i, tuple(argv)) for i, argv in enumerate(commands))
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def validation_set_digest(defs: Sequence[Definition]) -> str:
|
|
270
|
+
return set_digest([d.digest for d in defs])
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
# -- policy P (D8.1, D8.2) -------------------------------------------------------------
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
@dataclass(frozen=True)
|
|
277
|
+
class Policy:
|
|
278
|
+
"""How a set runs. Path-independent, so two exports agree (D8.1)."""
|
|
279
|
+
|
|
280
|
+
mode: str
|
|
281
|
+
template: Mapping[str, object]
|
|
282
|
+
env_names: tuple[str, ...]
|
|
283
|
+
timeout_seconds: int
|
|
284
|
+
contain_orphans: bool
|
|
285
|
+
method: str
|
|
286
|
+
|
|
287
|
+
@property
|
|
288
|
+
def template_digest(self) -> str:
|
|
289
|
+
return _sha256(self.template)
|
|
290
|
+
|
|
291
|
+
@property
|
|
292
|
+
def env_names_digest(self) -> str:
|
|
293
|
+
return _sha256({"version": ALLOWLIST_ENCODING, "names": sorted(set(self.env_names))})
|
|
294
|
+
|
|
295
|
+
@property
|
|
296
|
+
def digest(self) -> str:
|
|
297
|
+
return _sha256(
|
|
298
|
+
{
|
|
299
|
+
"version": POLICY_ENCODING,
|
|
300
|
+
"mode": self.mode,
|
|
301
|
+
"template": self.template_digest,
|
|
302
|
+
"env_allowlist": self.env_names_digest,
|
|
303
|
+
"timeout_seconds": self.timeout_seconds,
|
|
304
|
+
"contain_orphans": self.contain_orphans,
|
|
305
|
+
"method": self.method,
|
|
306
|
+
}
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
@property
|
|
310
|
+
def requires_enforcement(self) -> bool:
|
|
311
|
+
"""D8.3: a record applies under this policy only when it was enforced.
|
|
312
|
+
|
|
313
|
+
Under ``required``, and under ``preferred`` in REMOTE, which refuses
|
|
314
|
+
as ``required`` does (ADR 0005 D12.2). LOCAL's ``preferred`` falls
|
|
315
|
+
back to logical isolation on a host that cannot enforce, and its
|
|
316
|
+
records say ``enforced: false`` honestly; the in-place method is
|
|
317
|
+
LOCAL's, so it tells the two apart.
|
|
318
|
+
"""
|
|
319
|
+
if self.mode == "preferred":
|
|
320
|
+
return self.method != METHOD_IN_PLACE
|
|
321
|
+
return self.mode == "required"
|
|
322
|
+
|
|
323
|
+
def stored(self) -> dict:
|
|
324
|
+
return {
|
|
325
|
+
"policy_digest": self.digest,
|
|
326
|
+
"mode": self.mode,
|
|
327
|
+
"timeout_seconds": self.timeout_seconds,
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
def describe(self) -> str:
|
|
331
|
+
"""One line for the dry-run plan and ``status``."""
|
|
332
|
+
return (
|
|
333
|
+
f"mode {self.mode}, digest {self.digest[:16]}..., timeout {self.timeout_seconds}s, "
|
|
334
|
+
f"method {self.method}"
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
# -- candidates (D2.1-D2.7) ------------------------------------------------------------
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
@dataclass(frozen=True)
|
|
342
|
+
class WorktreeProvenance:
|
|
343
|
+
"""An issue worktree's candidate (D2.3); displayed, never compared."""
|
|
344
|
+
|
|
345
|
+
issue_number: int
|
|
346
|
+
worktree_id: str
|
|
347
|
+
strict_import: bool
|
|
348
|
+
# The base revision the phase's entry observation required; "" for none.
|
|
349
|
+
base_revision: str = ""
|
|
350
|
+
|
|
351
|
+
def to_dict(self) -> dict:
|
|
352
|
+
return {
|
|
353
|
+
"kind": PROVENANCE_WORKTREE,
|
|
354
|
+
"issue_number": self.issue_number,
|
|
355
|
+
"worktree_id": self.worktree_id,
|
|
356
|
+
"strict_import": self.strict_import,
|
|
357
|
+
"base_revision": self.base_revision,
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
@dataclass(frozen=True)
|
|
362
|
+
class PrHeadProvenance:
|
|
363
|
+
"""A published candidate: the head of the canonical PR URL (D2.3)."""
|
|
364
|
+
|
|
365
|
+
pr_url: str
|
|
366
|
+
|
|
367
|
+
def to_dict(self) -> dict:
|
|
368
|
+
return {"kind": PROVENANCE_PR_HEAD, "pr_url": self.pr_url}
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
Provenance = WorktreeProvenance | PrHeadProvenance
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
@dataclass(frozen=True)
|
|
375
|
+
class RemoteCandidate:
|
|
376
|
+
"""REMOTE's validation identity: a commit and the tree it names (D2.1)."""
|
|
377
|
+
|
|
378
|
+
repository: str
|
|
379
|
+
object_format: str
|
|
380
|
+
commit: str
|
|
381
|
+
tree: str
|
|
382
|
+
provenance: Provenance
|
|
383
|
+
|
|
384
|
+
kind = KIND_REMOTE
|
|
385
|
+
|
|
386
|
+
@property
|
|
387
|
+
def identity(self) -> tuple[str, ...]:
|
|
388
|
+
"""The key records are compared under: provenance is not part of it."""
|
|
389
|
+
return (KIND_REMOTE, self.repository.lower(), self.object_format, self.commit, self.tree)
|
|
390
|
+
|
|
391
|
+
def to_dict(self) -> dict:
|
|
392
|
+
return {
|
|
393
|
+
"kind": KIND_REMOTE,
|
|
394
|
+
"repository": self.repository,
|
|
395
|
+
"object_format": self.object_format,
|
|
396
|
+
"commit": self.commit,
|
|
397
|
+
"tree": self.tree,
|
|
398
|
+
"provenance": self.provenance.to_dict(),
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
def describe(self) -> str:
|
|
402
|
+
return f"commit {self.commit[:12]} (tree {self.tree[:12]}) of {self.repository}"
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
@dataclass(frozen=True)
|
|
406
|
+
class LocalCandidate:
|
|
407
|
+
"""LOCAL's validation identity: the workspace as ADR 0001 fingerprints it (D2.6)."""
|
|
408
|
+
|
|
409
|
+
fingerprint: str
|
|
410
|
+
workspace_policy_digest: str
|
|
411
|
+
run_id: str
|
|
412
|
+
|
|
413
|
+
kind = KIND_LOCAL
|
|
414
|
+
|
|
415
|
+
@property
|
|
416
|
+
def identity(self) -> tuple[str, ...]:
|
|
417
|
+
return (KIND_LOCAL, self.fingerprint, self.workspace_policy_digest, self.run_id)
|
|
418
|
+
|
|
419
|
+
def to_dict(self) -> dict:
|
|
420
|
+
return {
|
|
421
|
+
"kind": KIND_LOCAL,
|
|
422
|
+
"fingerprint": self.fingerprint,
|
|
423
|
+
"workspace_policy_digest": self.workspace_policy_digest,
|
|
424
|
+
"run_id": self.run_id,
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
def describe(self) -> str:
|
|
428
|
+
return f"workspace {self.fingerprint[:16]}..."
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
Candidate = RemoteCandidate | LocalCandidate
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
# -- field checks ----------------------------------------------------------------------
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _fail(what: str, message: str) -> NoReturn:
|
|
438
|
+
raise StateError(f"{what} {message}")
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _exact(raw: object, keys: Iterable[str], what: str) -> dict:
|
|
442
|
+
if not isinstance(raw, dict):
|
|
443
|
+
_fail(what, f"must be an object, got {type(raw).__name__}")
|
|
444
|
+
wanted = tuple(keys)
|
|
445
|
+
unknown = sorted(str(k) for k in raw if k not in wanted)
|
|
446
|
+
if unknown:
|
|
447
|
+
_fail(what, f"has unknown key(s) {unknown}")
|
|
448
|
+
missing = [k for k in wanted if k not in raw]
|
|
449
|
+
if missing:
|
|
450
|
+
_fail(what, f"is missing key(s) {missing}")
|
|
451
|
+
return raw
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _str(value: object, what: str, max_chars: int) -> str:
|
|
455
|
+
if not isinstance(value, str):
|
|
456
|
+
_fail(what, f"must be a string, got {type(value).__name__}")
|
|
457
|
+
if len(value) > max_chars:
|
|
458
|
+
_fail(what, f"is {len(value)} characters, over its bound of {max_chars}")
|
|
459
|
+
return value
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _int(value: object, what: str, minimum: int, maximum: int) -> int:
|
|
463
|
+
if not isinstance(value, int) or isinstance(value, bool):
|
|
464
|
+
_fail(what, f"must be an integer, got {type(value).__name__}")
|
|
465
|
+
if value < minimum or value > maximum:
|
|
466
|
+
_fail(what, f"must be in [{minimum}, {maximum}], got {value}")
|
|
467
|
+
return value
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _bool(value: object, what: str) -> bool:
|
|
471
|
+
if not isinstance(value, bool):
|
|
472
|
+
_fail(what, f"must be a boolean, got {type(value).__name__}")
|
|
473
|
+
return value
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _enum(value: object, what: str, allowed: Sequence[str]) -> str:
|
|
477
|
+
if not isinstance(value, str) or value not in allowed:
|
|
478
|
+
_fail(what, f"must be one of {list(allowed)}, got {value!r:.80}")
|
|
479
|
+
return value
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _hex64(value: object, what: str) -> str:
|
|
483
|
+
if not isinstance(value, str) or not _HEX64_RE.match(value):
|
|
484
|
+
_fail(what, "must be a 64-character lowercase SHA-256 hex digest")
|
|
485
|
+
return value
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def _oid(value: object, what: str, object_format: str) -> str:
|
|
489
|
+
width = OBJECT_FORMATS[object_format]
|
|
490
|
+
if (
|
|
491
|
+
not isinstance(value, str)
|
|
492
|
+
or len(value) != width
|
|
493
|
+
or not all(c in "0123456789abcdef" for c in value)
|
|
494
|
+
):
|
|
495
|
+
_fail(what, f"must be a full {width}-character lowercase {object_format} object id")
|
|
496
|
+
return value
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _timestamp(value: object, what: str) -> str:
|
|
500
|
+
text = _str(value, what, MAX_TIMESTAMP_CHARS)
|
|
501
|
+
try:
|
|
502
|
+
moment = datetime.fromisoformat(text)
|
|
503
|
+
except ValueError:
|
|
504
|
+
_fail(what, f"is not an ISO 8601 time: {text!r}")
|
|
505
|
+
offset = moment.utcoffset()
|
|
506
|
+
if offset is None or offset.total_seconds() != 0 or "T" not in text:
|
|
507
|
+
_fail(what, f"must be an ISO 8601 UTC time as utcnow_iso writes it, got {text!r}")
|
|
508
|
+
return text
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _repository(value: object, what: str) -> str:
|
|
512
|
+
text = _str(value, what, MAX_REPOSITORY_CHARS)
|
|
513
|
+
if not _REPOSITORY_RE.match(text):
|
|
514
|
+
_fail(what, "must be an <owner>/<repo> repository name")
|
|
515
|
+
return text
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _ref(value: object, what: str) -> str:
|
|
519
|
+
text = _str(value, what, MAX_REF_CHARS)
|
|
520
|
+
branch = text.removeprefix("refs/heads/")
|
|
521
|
+
if (
|
|
522
|
+
not branch
|
|
523
|
+
or branch.startswith(("-", "/", "refs/"))
|
|
524
|
+
or branch.endswith(("/", "."))
|
|
525
|
+
or _REF_FORBIDDEN_RE.search(branch)
|
|
526
|
+
):
|
|
527
|
+
_fail(what, "must be a plain branch name")
|
|
528
|
+
return text
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _pr_url(value: object, what: str, repository: str) -> str:
|
|
532
|
+
text = _str(value, what, MAX_URL_CHARS)
|
|
533
|
+
try:
|
|
534
|
+
ref = parse_pr_url(text)
|
|
535
|
+
except ConfigurationError:
|
|
536
|
+
_fail(what, "is not a GitHub pull request URL")
|
|
537
|
+
if ref.canonical != text:
|
|
538
|
+
_fail(what, "must be stored in its canonical form")
|
|
539
|
+
if not ref.same_repository(repository):
|
|
540
|
+
_fail(what, f"is not a pull request of {repository}")
|
|
541
|
+
return text
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def _run_id(value: object, what: str) -> str:
|
|
545
|
+
text = _str(value, what, 128)
|
|
546
|
+
if not RUN_ID_RE.match(text) or text in (".", ".."):
|
|
547
|
+
_fail(what, "must be a run id (a single safe path component)")
|
|
548
|
+
return text
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _run_log(value: object, what: str) -> str:
|
|
552
|
+
text = _str(value, what, MAX_RUN_LOG_CHARS)
|
|
553
|
+
run_id, slash, step = text.partition("/")
|
|
554
|
+
if (
|
|
555
|
+
not slash
|
|
556
|
+
or not RUN_ID_RE.match(run_id)
|
|
557
|
+
or run_id in (".", "..")
|
|
558
|
+
or not (STEP_DIR_RE.match(step))
|
|
559
|
+
):
|
|
560
|
+
_fail(what, "must be <run id>/<step directory>")
|
|
561
|
+
return text
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _worktree_id(value: object, what: str) -> str:
|
|
565
|
+
text = _str(value, what, MAX_WORKTREE_ID_CHARS)
|
|
566
|
+
if not text or text in (".", "..") or _COMPONENT_FORBIDDEN_RE.search(text):
|
|
567
|
+
_fail(what, "must be one path component with no '/', NUL or control character")
|
|
568
|
+
return text
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _stored_argv(value: object, what: str) -> list[str]:
|
|
572
|
+
if not isinstance(value, list) or not value:
|
|
573
|
+
_fail(what, "must be a non-empty list of strings")
|
|
574
|
+
if len(value) > MAX_ARGV_ELEMENTS:
|
|
575
|
+
_fail(what, f"holds {len(value)} elements, over its bound of {MAX_ARGV_ELEMENTS}")
|
|
576
|
+
for i, item in enumerate(value):
|
|
577
|
+
_str(item, f"{what}[{i}]", MAX_STORED_ARGV_ELEMENT_CHARS)
|
|
578
|
+
total = sum(len(item) for item in value)
|
|
579
|
+
if total > MAX_STORED_ARGV_CHARS:
|
|
580
|
+
_fail(what, f"is {total} characters in all, over its bound of {MAX_STORED_ARGV_CHARS}")
|
|
581
|
+
return value
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _tail(value: object, what: str) -> str:
|
|
585
|
+
text = _str(value, what, MAX_TAIL_CHARS)
|
|
586
|
+
if _UNESCAPED_RE.search(text):
|
|
587
|
+
_fail(what, "holds an unescaped control character")
|
|
588
|
+
return text
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
# -- parsing ---------------------------------------------------------------------------
|
|
592
|
+
|
|
593
|
+
_REMOTE_KEYS = ("kind", "repository", "object_format", "commit", "tree", "provenance")
|
|
594
|
+
_LOCAL_KEYS = ("kind", "fingerprint", "workspace_policy_digest", "run_id")
|
|
595
|
+
_WORKTREE_KEYS = ("kind", "issue_number", "worktree_id", "strict_import", "base_revision")
|
|
596
|
+
_PR_HEAD_KEYS = ("kind", "pr_url")
|
|
597
|
+
_DEFINITION_KEYS = ("index", "argv", "definition_digest", "scope")
|
|
598
|
+
_POLICY_KEYS = ("policy_digest", "mode", "timeout_seconds")
|
|
599
|
+
_ENFORCEMENT_KEYS = ("enforced", "backend", "backend_version", "network", "teardown")
|
|
600
|
+
_LEFTOVER_KEYS = (
|
|
601
|
+
"descendants_killed",
|
|
602
|
+
"group_survived_kill",
|
|
603
|
+
"orphans_killed",
|
|
604
|
+
"orphan_survived_kill",
|
|
605
|
+
"orphans_unchecked",
|
|
606
|
+
)
|
|
607
|
+
_OUTPUT_KEYS = (
|
|
608
|
+
"stdout_tail",
|
|
609
|
+
"stderr_tail",
|
|
610
|
+
"stdout_bytes",
|
|
611
|
+
"stderr_bytes",
|
|
612
|
+
"stdout_tail_truncated",
|
|
613
|
+
"stderr_tail_truncated",
|
|
614
|
+
)
|
|
615
|
+
_BOTH_KEYS = ("schema", "seq", "digest", "candidate", "definition", "policy", "outcome", "run_log")
|
|
616
|
+
_OBSERVED_KEYS = ("enforcement", "exit_code", "started_at", "finished_at", "leftovers", "output")
|
|
617
|
+
_INTERRUPTED_KEYS = ("marked_at", "recovered_at")
|
|
618
|
+
_MARKER_KEYS = ("candidate", "definition", "policy", "run_log", "marked_at")
|
|
619
|
+
_BINDING_KEYS = ("review_round", "candidate", "records", "validation_set_digest", "policy_digest")
|
|
620
|
+
_PAIR_KEYS = ("seq", "digest")
|
|
621
|
+
_COUNTER_KEYS = ("phase", "count", "candidate_commit", "last_record_seq")
|
|
622
|
+
_CERTIFICATE_BOTH = (
|
|
623
|
+
"schema",
|
|
624
|
+
"digest",
|
|
625
|
+
"kind",
|
|
626
|
+
"candidate",
|
|
627
|
+
"review_round",
|
|
628
|
+
"records",
|
|
629
|
+
"validation_set_digest",
|
|
630
|
+
"policy_digest",
|
|
631
|
+
"clean",
|
|
632
|
+
"issued_at",
|
|
633
|
+
)
|
|
634
|
+
_CERTIFICATE_REMOTE = (
|
|
635
|
+
"reviewed_pr_url",
|
|
636
|
+
"reviewed_head_sha",
|
|
637
|
+
"reviewed_base_ref",
|
|
638
|
+
"reviewed_merge_base_sha",
|
|
639
|
+
"comment",
|
|
640
|
+
)
|
|
641
|
+
_CERTIFICATE_LOCAL = ("reviewed_workspace_fingerprint",)
|
|
642
|
+
_COMMENT_KEYS = ("url", "id", "marker_key")
|
|
643
|
+
_MARKER_KEY_KEYS = (
|
|
644
|
+
"review_round",
|
|
645
|
+
"reviewed_head_sha",
|
|
646
|
+
"reviewed_base_ref",
|
|
647
|
+
"reviewed_merge_base_sha",
|
|
648
|
+
)
|
|
649
|
+
_CLEAN_KEYS = ("needs_fix_round", "actionable_findings")
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def _parse_provenance(raw: object, what: str, repository: str) -> Provenance:
|
|
653
|
+
if not isinstance(raw, dict):
|
|
654
|
+
_fail(what, f"must be an object, got {type(raw).__name__}")
|
|
655
|
+
kind = raw.get("kind")
|
|
656
|
+
if kind == PROVENANCE_WORKTREE:
|
|
657
|
+
_exact(raw, _WORKTREE_KEYS, what)
|
|
658
|
+
base = _str(raw["base_revision"], f"{what}.base_revision", 64)
|
|
659
|
+
if base and not (len(base) in (40, 64) and _hex(base)):
|
|
660
|
+
_fail(f"{what}.base_revision", "must be a full lowercase object id or empty")
|
|
661
|
+
return WorktreeProvenance(
|
|
662
|
+
issue_number=_int(raw["issue_number"], f"{what}.issue_number", 1, MAX_SEQ),
|
|
663
|
+
worktree_id=_worktree_id(raw["worktree_id"], f"{what}.worktree_id"),
|
|
664
|
+
strict_import=_bool(raw["strict_import"], f"{what}.strict_import"),
|
|
665
|
+
base_revision=base,
|
|
666
|
+
)
|
|
667
|
+
if kind == PROVENANCE_PR_HEAD:
|
|
668
|
+
_exact(raw, _PR_HEAD_KEYS, what)
|
|
669
|
+
return PrHeadProvenance(_pr_url(raw["pr_url"], f"{what}.pr_url", repository))
|
|
670
|
+
_fail(f"{what}.kind", f"must be {PROVENANCE_WORKTREE!r} or {PROVENANCE_PR_HEAD!r}")
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def _hex(text: str) -> bool:
|
|
674
|
+
return all(c in "0123456789abcdef" for c in text)
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def parse_candidate(raw: object, what: str = "candidate") -> Candidate:
|
|
678
|
+
"""A candidate of exactly one kind (D2.7): mixed fields are refused."""
|
|
679
|
+
if not isinstance(raw, dict):
|
|
680
|
+
_fail(what, f"must be an object, got {type(raw).__name__}")
|
|
681
|
+
kind = raw.get("kind")
|
|
682
|
+
if kind == KIND_REMOTE:
|
|
683
|
+
_exact(raw, _REMOTE_KEYS, what)
|
|
684
|
+
repository = _repository(raw["repository"], f"{what}.repository")
|
|
685
|
+
object_format = _enum(raw["object_format"], f"{what}.object_format", tuple(OBJECT_FORMATS))
|
|
686
|
+
commit = _oid(raw["commit"], f"{what}.commit", object_format)
|
|
687
|
+
tree = _oid(raw["tree"], f"{what}.tree", object_format)
|
|
688
|
+
provenance = _parse_provenance(raw["provenance"], f"{what}.provenance", repository)
|
|
689
|
+
if isinstance(provenance, WorktreeProvenance) and provenance.base_revision:
|
|
690
|
+
_oid(provenance.base_revision, f"{what}.provenance.base_revision", object_format)
|
|
691
|
+
return RemoteCandidate(repository, object_format, commit, tree, provenance)
|
|
692
|
+
if kind == KIND_LOCAL:
|
|
693
|
+
_exact(raw, _LOCAL_KEYS, what)
|
|
694
|
+
return LocalCandidate(
|
|
695
|
+
fingerprint=_hex64(raw["fingerprint"], f"{what}.fingerprint"),
|
|
696
|
+
workspace_policy_digest=_hex64(
|
|
697
|
+
raw["workspace_policy_digest"], f"{what}.workspace_policy_digest"
|
|
698
|
+
),
|
|
699
|
+
run_id=_run_id(raw["run_id"], f"{what}.run_id"),
|
|
700
|
+
)
|
|
701
|
+
_fail(f"{what}.kind", f"must be {KIND_REMOTE!r} or {KIND_LOCAL!r}, got {kind!r:.40}")
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
def _parse_definition(raw: object, what: str) -> dict:
|
|
705
|
+
_exact(raw, _DEFINITION_KEYS, what)
|
|
706
|
+
assert isinstance(raw, dict)
|
|
707
|
+
_int(raw["index"], f"{what}.index", 0, MAX_DEFINITIONS - 1)
|
|
708
|
+
_stored_argv(raw["argv"], f"{what}.argv")
|
|
709
|
+
_hex64(raw["definition_digest"], f"{what}.definition_digest")
|
|
710
|
+
_enum(raw["scope"], f"{what}.scope", (SCOPE,))
|
|
711
|
+
return raw
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def _parse_policy(raw: object, what: str) -> dict:
|
|
715
|
+
_exact(raw, _POLICY_KEYS, what)
|
|
716
|
+
assert isinstance(raw, dict)
|
|
717
|
+
_hex64(raw["policy_digest"], f"{what}.policy_digest")
|
|
718
|
+
_enum(raw["mode"], f"{what}.mode", MODES)
|
|
719
|
+
_int(raw["timeout_seconds"], f"{what}.timeout_seconds", 1, MAX_DEADLINE_SECONDS)
|
|
720
|
+
return raw
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def _parse_enforcement(raw: object, what: str) -> None:
|
|
724
|
+
_exact(raw, _ENFORCEMENT_KEYS, what)
|
|
725
|
+
assert isinstance(raw, dict)
|
|
726
|
+
_bool(raw["enforced"], f"{what}.enforced")
|
|
727
|
+
_str(raw["backend"], f"{what}.backend", MAX_BACKEND_CHARS)
|
|
728
|
+
_str(raw["backend_version"], f"{what}.backend_version", MAX_BACKEND_CHARS)
|
|
729
|
+
_enum(raw["network"], f"{what}.network", NETWORKS)
|
|
730
|
+
_str(raw["teardown"], f"{what}.teardown", MAX_TEARDOWN_CHARS)
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _parse_output(raw: object, what: str) -> None:
|
|
734
|
+
_exact(raw, _OUTPUT_KEYS, what)
|
|
735
|
+
assert isinstance(raw, dict)
|
|
736
|
+
for stream in ("stdout", "stderr"):
|
|
737
|
+
_tail(raw[f"{stream}_tail"], f"{what}.{stream}_tail")
|
|
738
|
+
_int(raw[f"{stream}_bytes"], f"{what}.{stream}_bytes", 0, MAX_BYTE_COUNT)
|
|
739
|
+
_bool(raw[f"{stream}_tail_truncated"], f"{what}.{stream}_tail_truncated")
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def record_digest(record: Mapping[str, object]) -> str:
|
|
743
|
+
"""SHA-256 over the canonical encoding of every field but ``digest`` (D10.1)."""
|
|
744
|
+
return _sha256({k: v for k, v in record.items() if k != "digest"})
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
@dataclass(frozen=True)
|
|
748
|
+
class RecordView:
|
|
749
|
+
"""A loaded record's fields the integrity checks and the readers compare."""
|
|
750
|
+
|
|
751
|
+
raw: dict
|
|
752
|
+
seq: int
|
|
753
|
+
digest: str
|
|
754
|
+
candidate: Candidate
|
|
755
|
+
index: int
|
|
756
|
+
argv: tuple[str, ...]
|
|
757
|
+
definition_digest: str
|
|
758
|
+
policy_digest: str
|
|
759
|
+
mode: str
|
|
760
|
+
outcome: Outcome
|
|
761
|
+
run_log: str
|
|
762
|
+
enforced: bool = False
|
|
763
|
+
exit_code: int | None = None
|
|
764
|
+
start_fingerprint: str = ""
|
|
765
|
+
end_fingerprint: str = ""
|
|
766
|
+
|
|
767
|
+
@property
|
|
768
|
+
def is_observed(self) -> bool:
|
|
769
|
+
return self.outcome is not Outcome.INTERRUPTED
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
def parse_record(raw: object, what: str = "validation record") -> RecordView:
|
|
773
|
+
"""One record of exactly one shape and one kind, digest-checked (D10.1, D11.1)."""
|
|
774
|
+
if not isinstance(raw, dict):
|
|
775
|
+
_fail(what, f"must be an object, got {type(raw).__name__}")
|
|
776
|
+
outcome_value = raw.get("outcome")
|
|
777
|
+
if outcome_value not in tuple(Outcome):
|
|
778
|
+
_fail(f"{what}.outcome", f"must be one of {[o.value for o in Outcome]}")
|
|
779
|
+
outcome = Outcome(str(outcome_value))
|
|
780
|
+
candidate_raw = raw.get("candidate")
|
|
781
|
+
local = isinstance(candidate_raw, dict) and candidate_raw.get("kind") == KIND_LOCAL
|
|
782
|
+
keys = list(_BOTH_KEYS)
|
|
783
|
+
keys += _INTERRUPTED_KEYS if outcome is Outcome.INTERRUPTED else _OBSERVED_KEYS
|
|
784
|
+
if local:
|
|
785
|
+
keys.append("start_fingerprint")
|
|
786
|
+
if outcome is not Outcome.INTERRUPTED:
|
|
787
|
+
keys.append("end_fingerprint")
|
|
788
|
+
_exact(raw, keys, what)
|
|
789
|
+
_int(raw["schema"], f"{what}.schema", SCHEMA, SCHEMA)
|
|
790
|
+
seq = _int(raw["seq"], f"{what}.seq", 1, MAX_SEQ)
|
|
791
|
+
digest = _hex64(raw["digest"], f"{what}.digest")
|
|
792
|
+
candidate = parse_candidate(candidate_raw, f"{what}.candidate")
|
|
793
|
+
definition = _parse_definition(raw["definition"], f"{what}.definition")
|
|
794
|
+
policy = _parse_policy(raw["policy"], f"{what}.policy")
|
|
795
|
+
run_log = _run_log(raw["run_log"], f"{what}.run_log")
|
|
796
|
+
start = end = ""
|
|
797
|
+
if local:
|
|
798
|
+
start = _hex64(raw["start_fingerprint"], f"{what}.start_fingerprint")
|
|
799
|
+
assert isinstance(candidate, LocalCandidate)
|
|
800
|
+
if start != candidate.fingerprint:
|
|
801
|
+
_fail(f"{what}.start_fingerprint", "must equal the candidate's fingerprint")
|
|
802
|
+
enforced = False
|
|
803
|
+
exit_code: int | None = None
|
|
804
|
+
if outcome is Outcome.INTERRUPTED:
|
|
805
|
+
_timestamp(raw["marked_at"], f"{what}.marked_at")
|
|
806
|
+
_timestamp(raw["recovered_at"], f"{what}.recovered_at")
|
|
807
|
+
else:
|
|
808
|
+
_parse_enforcement(raw["enforcement"], f"{what}.enforcement")
|
|
809
|
+
enforced = raw["enforcement"]["enforced"]
|
|
810
|
+
code = raw["exit_code"]
|
|
811
|
+
if outcome is Outcome.TIMEOUT:
|
|
812
|
+
if code is not None:
|
|
813
|
+
_fail(f"{what}.exit_code", "must be null for a TIMEOUT")
|
|
814
|
+
else:
|
|
815
|
+
exit_code = _int(code, f"{what}.exit_code", MIN_EXIT_CODE, MAX_EXIT_CODE)
|
|
816
|
+
if (exit_code == 0) != (outcome is Outcome.PASS):
|
|
817
|
+
_fail(f"{what}.exit_code", f"{exit_code} contradicts the outcome {outcome.value}")
|
|
818
|
+
_timestamp(raw["started_at"], f"{what}.started_at")
|
|
819
|
+
_timestamp(raw["finished_at"], f"{what}.finished_at")
|
|
820
|
+
_exact(raw["leftovers"], _LEFTOVER_KEYS, f"{what}.leftovers")
|
|
821
|
+
for key in _LEFTOVER_KEYS:
|
|
822
|
+
_bool(raw["leftovers"][key], f"{what}.leftovers.{key}")
|
|
823
|
+
_parse_output(raw["output"], f"{what}.output")
|
|
824
|
+
if local:
|
|
825
|
+
end = _hex64(raw["end_fingerprint"], f"{what}.end_fingerprint")
|
|
826
|
+
if record_digest(raw) != digest:
|
|
827
|
+
_fail(f"{what}.digest", "does not match the record: it was edited or corrupted")
|
|
828
|
+
return RecordView(
|
|
829
|
+
raw=raw,
|
|
830
|
+
seq=seq,
|
|
831
|
+
digest=digest,
|
|
832
|
+
candidate=candidate,
|
|
833
|
+
index=definition["index"],
|
|
834
|
+
argv=tuple(definition["argv"]),
|
|
835
|
+
definition_digest=definition["definition_digest"],
|
|
836
|
+
policy_digest=policy["policy_digest"],
|
|
837
|
+
mode=policy["mode"],
|
|
838
|
+
outcome=outcome,
|
|
839
|
+
run_log=run_log,
|
|
840
|
+
enforced=enforced,
|
|
841
|
+
exit_code=exit_code,
|
|
842
|
+
start_fingerprint=start,
|
|
843
|
+
end_fingerprint=end,
|
|
844
|
+
)
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
@dataclass(frozen=True)
|
|
848
|
+
class MarkerView:
|
|
849
|
+
raw: dict
|
|
850
|
+
candidate: Candidate
|
|
851
|
+
definition_digest: str
|
|
852
|
+
policy_digest: str
|
|
853
|
+
run_log: str
|
|
854
|
+
index: int
|
|
855
|
+
argv: tuple[str, ...]
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
def parse_marker(raw: object, what: str = "validation_attempt") -> MarkerView:
|
|
859
|
+
"""The attempt marker (D11.2): what an interrupted record copies, nothing observed."""
|
|
860
|
+
if not isinstance(raw, dict):
|
|
861
|
+
_fail(what, f"must be an object, got {type(raw).__name__}")
|
|
862
|
+
candidate_raw = raw.get("candidate")
|
|
863
|
+
local = isinstance(candidate_raw, dict) and candidate_raw.get("kind") == KIND_LOCAL
|
|
864
|
+
_exact(raw, (*_MARKER_KEYS, *(("start_fingerprint",) if local else ())), what)
|
|
865
|
+
candidate = parse_candidate(candidate_raw, f"{what}.candidate")
|
|
866
|
+
definition = _parse_definition(raw["definition"], f"{what}.definition")
|
|
867
|
+
policy = _parse_policy(raw["policy"], f"{what}.policy")
|
|
868
|
+
run_log = _run_log(raw["run_log"], f"{what}.run_log")
|
|
869
|
+
_timestamp(raw["marked_at"], f"{what}.marked_at")
|
|
870
|
+
if local:
|
|
871
|
+
assert isinstance(candidate, LocalCandidate)
|
|
872
|
+
if _hex64(raw["start_fingerprint"], f"{what}.start_fingerprint") != candidate.fingerprint:
|
|
873
|
+
_fail(f"{what}.start_fingerprint", "must equal the candidate's fingerprint")
|
|
874
|
+
return MarkerView(
|
|
875
|
+
raw=raw,
|
|
876
|
+
candidate=candidate,
|
|
877
|
+
definition_digest=definition["definition_digest"],
|
|
878
|
+
policy_digest=policy["policy_digest"],
|
|
879
|
+
run_log=run_log,
|
|
880
|
+
index=definition["index"],
|
|
881
|
+
argv=tuple(definition["argv"]),
|
|
882
|
+
)
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
def _parse_pairs(raw: object, what: str) -> list[tuple[int, str]]:
|
|
886
|
+
if not isinstance(raw, list):
|
|
887
|
+
_fail(what, "must be a list")
|
|
888
|
+
if len(raw) > MAX_BOUND_RECORDS:
|
|
889
|
+
_fail(what, f"holds {len(raw)} records, over its bound of {MAX_BOUND_RECORDS}")
|
|
890
|
+
pairs: list[tuple[int, str]] = []
|
|
891
|
+
for i, item in enumerate(raw):
|
|
892
|
+
_exact(item, _PAIR_KEYS, f"{what}[{i}]")
|
|
893
|
+
pairs.append(
|
|
894
|
+
(
|
|
895
|
+
_int(item["seq"], f"{what}[{i}].seq", 1, MAX_SEQ),
|
|
896
|
+
_hex64(item["digest"], f"{what}[{i}].digest"),
|
|
897
|
+
)
|
|
898
|
+
)
|
|
899
|
+
return pairs
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
# -- integrity (D11.1) -----------------------------------------------------------------
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
@dataclass(frozen=True)
|
|
906
|
+
class EvidenceFields:
|
|
907
|
+
"""The six persisted additions of the Wave 2B migration (D11.2)."""
|
|
908
|
+
|
|
909
|
+
validation_records: list = field(default_factory=list)
|
|
910
|
+
validation_attempt: dict = field(default_factory=dict)
|
|
911
|
+
review_evidence: dict = field(default_factory=dict)
|
|
912
|
+
review_certificate: dict = field(default_factory=dict)
|
|
913
|
+
producer_validation_failure: dict = field(default_factory=dict)
|
|
914
|
+
validation_override_seq: int | None = None
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
def check_evidence(
|
|
918
|
+
fields_: EvidenceFields, *, local: bool, repository: str, run_id: str
|
|
919
|
+
) -> list[RecordView]:
|
|
920
|
+
"""Validate the six fields together; return the records, parsed. Read-only.
|
|
921
|
+
|
|
922
|
+
``local`` is the state's mode, ``repository`` its REMOTE repository and
|
|
923
|
+
``run_id`` its run: a record of the other mode's kind, of another
|
|
924
|
+
repository or run, or a run log of another run is refused (D2.7).
|
|
925
|
+
"""
|
|
926
|
+
records_raw = fields_.validation_records
|
|
927
|
+
if not isinstance(records_raw, list):
|
|
928
|
+
_fail("state field 'validation_records'", "must be a list")
|
|
929
|
+
if len(records_raw) > MAX_RECORDS:
|
|
930
|
+
_fail(
|
|
931
|
+
"state field 'validation_records'",
|
|
932
|
+
f"holds {len(records_raw)} records, over its bound of {MAX_RECORDS}",
|
|
933
|
+
)
|
|
934
|
+
records: list[RecordView] = []
|
|
935
|
+
for i, raw in enumerate(records_raw):
|
|
936
|
+
where = f"state field 'validation_records'[{i}]"
|
|
937
|
+
view = parse_record(raw, where)
|
|
938
|
+
_check_candidate(view.candidate, f"{where}.candidate", local, repository, run_id)
|
|
939
|
+
_check_run_log(view.run_log, f"{where}.run_log", run_id)
|
|
940
|
+
if records and view.seq <= records[-1].seq:
|
|
941
|
+
_fail(f"{where}.seq", "must increase strictly")
|
|
942
|
+
records.append(view)
|
|
943
|
+
by_seq = {r.seq: r for r in records}
|
|
944
|
+
|
|
945
|
+
marker = fields_.validation_attempt
|
|
946
|
+
if not isinstance(marker, dict):
|
|
947
|
+
_fail("state field 'validation_attempt'", "must be an object")
|
|
948
|
+
if marker:
|
|
949
|
+
where = "state field 'validation_attempt'"
|
|
950
|
+
pending = parse_marker(marker, where)
|
|
951
|
+
_check_candidate(pending.candidate, f"{where}.candidate", local, repository, run_id)
|
|
952
|
+
_check_run_log(pending.run_log, f"{where}.run_log", run_id)
|
|
953
|
+
|
|
954
|
+
binding = fields_.review_evidence
|
|
955
|
+
if not isinstance(binding, dict):
|
|
956
|
+
_fail("state field 'review_evidence'", "must be an object")
|
|
957
|
+
if binding:
|
|
958
|
+
_check_binding(binding, by_seq, local, repository, run_id)
|
|
959
|
+
|
|
960
|
+
certificate = fields_.review_certificate
|
|
961
|
+
if not isinstance(certificate, dict):
|
|
962
|
+
_fail("state field 'review_certificate'", "must be an object")
|
|
963
|
+
if certificate:
|
|
964
|
+
_check_certificate(certificate, by_seq, local, repository, run_id)
|
|
965
|
+
|
|
966
|
+
counter = fields_.producer_validation_failure
|
|
967
|
+
if not isinstance(counter, dict):
|
|
968
|
+
_fail("state field 'producer_validation_failure'", "must be an object")
|
|
969
|
+
if counter:
|
|
970
|
+
if local:
|
|
971
|
+
_fail("state field 'producer_validation_failure'", "must be empty for a LOCAL run")
|
|
972
|
+
_check_counter(counter, by_seq)
|
|
973
|
+
|
|
974
|
+
watermark = fields_.validation_override_seq
|
|
975
|
+
if watermark is not None:
|
|
976
|
+
what = "state field 'validation_override_seq'"
|
|
977
|
+
if local:
|
|
978
|
+
_fail(what, "must be null for a LOCAL run: LOCAL has no unblock")
|
|
979
|
+
_int(watermark, what, 0, MAX_SEQ)
|
|
980
|
+
highest = records[-1].seq if records else 0
|
|
981
|
+
if watermark > highest:
|
|
982
|
+
_fail(what, f"is {watermark}, above the highest retained record seq {highest}")
|
|
983
|
+
return records
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
def _check_candidate(
|
|
987
|
+
candidate: Candidate, where: str, local: bool, repository: str, run_id: str
|
|
988
|
+
) -> None:
|
|
989
|
+
"""D2.7: the state's own kind, repository and run."""
|
|
990
|
+
if local != isinstance(candidate, LocalCandidate):
|
|
991
|
+
mode = "LOCAL" if local else "REMOTE"
|
|
992
|
+
_fail(where, f"holds a {candidate.kind} candidate in a {mode} run")
|
|
993
|
+
if isinstance(candidate, LocalCandidate):
|
|
994
|
+
if candidate.run_id != run_id:
|
|
995
|
+
_fail(where, f"names run {candidate.run_id!r}, not this run {run_id!r}")
|
|
996
|
+
elif repository and candidate.repository.lower() != repository.lower():
|
|
997
|
+
_fail(where, f"names repository {candidate.repository!r}, not {repository!r}")
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
def _check_run_log(run_log: str, where: str, run_id: str) -> None:
|
|
1001
|
+
if run_log.partition("/")[0] != run_id:
|
|
1002
|
+
_fail(where, f"is not in this run's log directory ({run_id})")
|
|
1003
|
+
|
|
1004
|
+
|
|
1005
|
+
def _check_named(
|
|
1006
|
+
pairs: list[tuple[int, str]],
|
|
1007
|
+
by_seq: Mapping[int, RecordView],
|
|
1008
|
+
candidate: Candidate,
|
|
1009
|
+
policy_digest: str,
|
|
1010
|
+
what: str,
|
|
1011
|
+
) -> list[RecordView]:
|
|
1012
|
+
named: list[RecordView] = []
|
|
1013
|
+
for i, (seq, digest) in enumerate(pairs):
|
|
1014
|
+
where = f"{what}.records[{i}]"
|
|
1015
|
+
record = by_seq.get(seq)
|
|
1016
|
+
if record is None:
|
|
1017
|
+
_fail(where, f"names record seq {seq}, which is not retained")
|
|
1018
|
+
if record.digest != digest:
|
|
1019
|
+
_fail(where, f"names record seq {seq} under another digest")
|
|
1020
|
+
if record.outcome is Outcome.INTERRUPTED:
|
|
1021
|
+
_fail(where, f"names record seq {seq}, which is INTERRUPTED and never evidence")
|
|
1022
|
+
if record.candidate.identity != candidate.identity:
|
|
1023
|
+
_fail(where, f"names record seq {seq}, which is for another candidate")
|
|
1024
|
+
if record.policy_digest != policy_digest:
|
|
1025
|
+
_fail(where, f"names record seq {seq}, which ran under another policy")
|
|
1026
|
+
if record.index != i:
|
|
1027
|
+
_fail(where, f"names record seq {seq}, which is definition {record.index}, not {i}")
|
|
1028
|
+
named.append(record)
|
|
1029
|
+
return named
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def _check_binding(
|
|
1033
|
+
raw: dict, by_seq: Mapping[int, RecordView], local: bool, repository: str, run_id: str
|
|
1034
|
+
) -> None:
|
|
1035
|
+
what = "state field 'review_evidence'"
|
|
1036
|
+
_exact(raw, _BINDING_KEYS, what)
|
|
1037
|
+
_int(raw["review_round"], f"{what}.review_round", 1, MAX_SEQ)
|
|
1038
|
+
candidate = parse_candidate(raw["candidate"], f"{what}.candidate")
|
|
1039
|
+
_check_candidate(candidate, f"{what}.candidate", local, repository, run_id)
|
|
1040
|
+
pairs = _parse_pairs(raw["records"], f"{what}.records")
|
|
1041
|
+
set_value = _hex64(raw["validation_set_digest"], f"{what}.validation_set_digest")
|
|
1042
|
+
policy_digest = _hex64(raw["policy_digest"], f"{what}.policy_digest")
|
|
1043
|
+
named = _check_named(pairs, by_seq, candidate, policy_digest, what)
|
|
1044
|
+
# D10.6: the set stops at its first non-PASS, so only the last record named
|
|
1045
|
+
# may be one. A binding naming PASS records only (none, for an empty set)
|
|
1046
|
+
# names every definition of its set, and their digests encode to its set
|
|
1047
|
+
# digest (D3.5). After a non-PASS the definitions that got no record are
|
|
1048
|
+
# not in the binding, so that set digest cannot be recomputed here: D5.5
|
|
1049
|
+
# compares it with the set in force, and D12.2 issues no certificate from it.
|
|
1050
|
+
for i, record in enumerate(named[:-1]):
|
|
1051
|
+
if record.outcome is not Outcome.PASS:
|
|
1052
|
+
_fail(
|
|
1053
|
+
f"{what}.records[{i}]",
|
|
1054
|
+
f"names a {record.outcome.value} record before the last; "
|
|
1055
|
+
"the set stops at its first non-PASS",
|
|
1056
|
+
)
|
|
1057
|
+
if all(record.outcome is Outcome.PASS for record in named):
|
|
1058
|
+
if set_digest([r.definition_digest for r in named]) != set_value:
|
|
1059
|
+
_fail(f"{what}.validation_set_digest", "is not the digest of the records' definitions")
|
|
1060
|
+
|
|
1061
|
+
|
|
1062
|
+
def certificate_digest(certificate: Mapping[str, object]) -> str:
|
|
1063
|
+
return _sha256({k: v for k, v in certificate.items() if k != "digest"})
|
|
1064
|
+
|
|
1065
|
+
|
|
1066
|
+
def check_certificate(
|
|
1067
|
+
raw: Mapping[str, object],
|
|
1068
|
+
records: Sequence[RecordView],
|
|
1069
|
+
*,
|
|
1070
|
+
local: bool,
|
|
1071
|
+
repository: str,
|
|
1072
|
+
run_id: str,
|
|
1073
|
+
) -> None:
|
|
1074
|
+
"""One certificate against the retained ``records``, exactly as a load checks it (D11.1).
|
|
1075
|
+
|
|
1076
|
+
The writer's check (:mod:`autoforge.certificate` never builds one a load
|
|
1077
|
+
refuses) and the applicability check's first step.
|
|
1078
|
+
"""
|
|
1079
|
+
if not isinstance(raw, dict):
|
|
1080
|
+
_fail("state field 'review_certificate'", "must be an object")
|
|
1081
|
+
_check_certificate(raw, {r.seq: r for r in records}, local, repository, run_id)
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
def _check_certificate(
|
|
1085
|
+
raw: dict, by_seq: Mapping[int, RecordView], local: bool, repository: str, run_id: str
|
|
1086
|
+
) -> None:
|
|
1087
|
+
what = "state field 'review_certificate'"
|
|
1088
|
+
kind = raw.get("kind")
|
|
1089
|
+
if kind not in (KIND_REMOTE, KIND_LOCAL):
|
|
1090
|
+
_fail(f"{what}.kind", f"must be {KIND_REMOTE!r} or {KIND_LOCAL!r}")
|
|
1091
|
+
extra = _CERTIFICATE_LOCAL if kind == KIND_LOCAL else _CERTIFICATE_REMOTE
|
|
1092
|
+
_exact(raw, (*_CERTIFICATE_BOTH, *extra), what)
|
|
1093
|
+
_int(raw["schema"], f"{what}.schema", SCHEMA, SCHEMA)
|
|
1094
|
+
digest = _hex64(raw["digest"], f"{what}.digest")
|
|
1095
|
+
candidate = parse_candidate(raw["candidate"], f"{what}.candidate")
|
|
1096
|
+
if candidate.kind != kind:
|
|
1097
|
+
_fail(f"{what}.kind", "differs from its candidate's kind")
|
|
1098
|
+
_check_candidate(candidate, f"{what}.candidate", local, repository, run_id)
|
|
1099
|
+
review_round = _int(raw["review_round"], f"{what}.review_round", 1, MAX_SEQ)
|
|
1100
|
+
pairs = _parse_pairs(raw["records"], f"{what}.records")
|
|
1101
|
+
set_value = _hex64(raw["validation_set_digest"], f"{what}.validation_set_digest")
|
|
1102
|
+
policy_digest = _hex64(raw["policy_digest"], f"{what}.policy_digest")
|
|
1103
|
+
named = _check_named(pairs, by_seq, candidate, policy_digest, what)
|
|
1104
|
+
for i, record in enumerate(named):
|
|
1105
|
+
if record.outcome is not Outcome.PASS:
|
|
1106
|
+
_fail(
|
|
1107
|
+
f"{what}.records[{i}]",
|
|
1108
|
+
f"names a {record.outcome.value} record; a certificate names PASS records only",
|
|
1109
|
+
)
|
|
1110
|
+
if set_digest([r.definition_digest for r in named]) != set_value:
|
|
1111
|
+
_fail(f"{what}.validation_set_digest", "is not the digest of the records' definitions")
|
|
1112
|
+
clean = _exact(raw["clean"], _CLEAN_KEYS, f"{what}.clean")
|
|
1113
|
+
if clean["needs_fix_round"] is not False:
|
|
1114
|
+
_fail(f"{what}.clean.needs_fix_round", "must be false")
|
|
1115
|
+
if _int(clean["actionable_findings"], f"{what}.clean.actionable_findings", 0, 0) != 0:
|
|
1116
|
+
_fail(f"{what}.clean.actionable_findings", "must be 0")
|
|
1117
|
+
_timestamp(raw["issued_at"], f"{what}.issued_at")
|
|
1118
|
+
if isinstance(candidate, LocalCandidate):
|
|
1119
|
+
name = f"{what}.reviewed_workspace_fingerprint"
|
|
1120
|
+
if _hex64(raw["reviewed_workspace_fingerprint"], name) != candidate.fingerprint:
|
|
1121
|
+
_fail(name, "differs from the candidate's")
|
|
1122
|
+
else:
|
|
1123
|
+
pr_url = _pr_url(raw["reviewed_pr_url"], f"{what}.reviewed_pr_url", candidate.repository)
|
|
1124
|
+
head = _oid(raw["reviewed_head_sha"], f"{what}.reviewed_head_sha", candidate.object_format)
|
|
1125
|
+
if head != candidate.commit:
|
|
1126
|
+
_fail(f"{what}.reviewed_head_sha", "differs from the candidate's commit")
|
|
1127
|
+
base_ref = _ref(raw["reviewed_base_ref"], f"{what}.reviewed_base_ref")
|
|
1128
|
+
merge_base = _oid(
|
|
1129
|
+
raw["reviewed_merge_base_sha"],
|
|
1130
|
+
f"{what}.reviewed_merge_base_sha",
|
|
1131
|
+
candidate.object_format,
|
|
1132
|
+
)
|
|
1133
|
+
comment = _exact(raw["comment"], _COMMENT_KEYS, f"{what}.comment")
|
|
1134
|
+
url = _str(comment["url"], f"{what}.comment.url", MAX_URL_CHARS)
|
|
1135
|
+
try:
|
|
1136
|
+
ref = parse_comment_url(url)
|
|
1137
|
+
except ConfigurationError:
|
|
1138
|
+
_fail(f"{what}.comment.url", "is not a GitHub comment URL")
|
|
1139
|
+
if ref.canonical != url:
|
|
1140
|
+
_fail(f"{what}.comment.url", "must be stored in its canonical form")
|
|
1141
|
+
if not ref.on(parse_pr_url(pr_url)):
|
|
1142
|
+
_fail(f"{what}.comment.url", "is not on the reviewed pull request")
|
|
1143
|
+
if _int(comment["id"], f"{what}.comment.id", 1, MAX_COMMENT_ID) != ref.comment_id:
|
|
1144
|
+
_fail(f"{what}.comment.id", "differs from the comment URL's id")
|
|
1145
|
+
key_what = f"{what}.comment.marker_key"
|
|
1146
|
+
key = _exact(comment["marker_key"], _MARKER_KEY_KEYS, key_what)
|
|
1147
|
+
# The round is parsed strictly first: equality alone takes JSON 2.0
|
|
1148
|
+
# (or true) for the integer it equals. The other three are compared
|
|
1149
|
+
# with strings the certificate's own fields were parsed into, which
|
|
1150
|
+
# only an equal string equals.
|
|
1151
|
+
_int(key["review_round"], f"{key_what}.review_round", 1, MAX_SEQ)
|
|
1152
|
+
expected = {
|
|
1153
|
+
"review_round": review_round,
|
|
1154
|
+
"reviewed_head_sha": head,
|
|
1155
|
+
"reviewed_base_ref": base_ref,
|
|
1156
|
+
"reviewed_merge_base_sha": merge_base,
|
|
1157
|
+
}
|
|
1158
|
+
for name, value in expected.items():
|
|
1159
|
+
if key[name] != value:
|
|
1160
|
+
_fail(f"{key_what}.{name}", "differs from the certificate's")
|
|
1161
|
+
if certificate_digest(raw) != digest:
|
|
1162
|
+
_fail(f"{what}.digest", "does not match the certificate: it was edited or corrupted")
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
def _check_counter(raw: dict, by_seq: Mapping[int, RecordView]) -> None:
|
|
1166
|
+
what = "state field 'producer_validation_failure'"
|
|
1167
|
+
_exact(raw, _COUNTER_KEYS, what)
|
|
1168
|
+
_enum(raw["phase"], f"{what}.phase", FAILURE_PHASES)
|
|
1169
|
+
_int(raw["count"], f"{what}.count", 1, MAX_FAILURE_COUNT)
|
|
1170
|
+
commit = _str(raw["candidate_commit"], f"{what}.candidate_commit", 64)
|
|
1171
|
+
seq = _int(raw["last_record_seq"], f"{what}.last_record_seq", 1, MAX_SEQ)
|
|
1172
|
+
record = by_seq.get(seq)
|
|
1173
|
+
if record is None:
|
|
1174
|
+
_fail(f"{what}.last_record_seq", f"names record seq {seq}, which is not retained")
|
|
1175
|
+
if record.outcome not in (Outcome.FAIL, Outcome.TIMEOUT):
|
|
1176
|
+
_fail(
|
|
1177
|
+
f"{what}.last_record_seq",
|
|
1178
|
+
f"names a {record.outcome.value} record, not a FAIL or TIMEOUT",
|
|
1179
|
+
)
|
|
1180
|
+
candidate = record.candidate
|
|
1181
|
+
assert isinstance(candidate, RemoteCandidate)
|
|
1182
|
+
if commit != candidate.commit:
|
|
1183
|
+
_fail(f"{what}.candidate_commit", "differs from the commit of the record it names")
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def pinned_seqs(
|
|
1187
|
+
review_evidence: Mapping[str, object],
|
|
1188
|
+
review_certificate: Mapping[str, object],
|
|
1189
|
+
producer_validation_failure: Mapping[str, object],
|
|
1190
|
+
) -> set[int]:
|
|
1191
|
+
"""The seqs no eviction may remove (D11.4)."""
|
|
1192
|
+
pinned: set[int] = set()
|
|
1193
|
+
for holder in (review_evidence, review_certificate):
|
|
1194
|
+
pairs = holder.get("records")
|
|
1195
|
+
for pair in pairs if isinstance(pairs, list) else ():
|
|
1196
|
+
if isinstance(pair, dict) and isinstance(pair.get("seq"), int):
|
|
1197
|
+
pinned.add(pair["seq"])
|
|
1198
|
+
seq = producer_validation_failure.get("last_record_seq")
|
|
1199
|
+
if isinstance(seq, int):
|
|
1200
|
+
pinned.add(seq)
|
|
1201
|
+
return pinned
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
# -- writers ---------------------------------------------------------------------------
|
|
1205
|
+
|
|
1206
|
+
|
|
1207
|
+
def _clip(text: str, limit: int) -> str:
|
|
1208
|
+
"""``SandboxFacts``' rule for an over-long display fact: cut, with an ellipsis."""
|
|
1209
|
+
return text if len(text) <= limit else text[: limit - 1] + "…"
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
def enforcement_facts(facts: SandboxFacts | None) -> dict:
|
|
1213
|
+
"""D8.2: what of the executor's facts a record copies; a launch with none is unenforced."""
|
|
1214
|
+
if facts is None:
|
|
1215
|
+
return {
|
|
1216
|
+
"enforced": False,
|
|
1217
|
+
"backend": "",
|
|
1218
|
+
"backend_version": "",
|
|
1219
|
+
"network": "",
|
|
1220
|
+
"teardown": "",
|
|
1221
|
+
}
|
|
1222
|
+
return {
|
|
1223
|
+
"enforced": bool(facts.enforced),
|
|
1224
|
+
"backend": _clip(redact(facts.backend), MAX_BACKEND_CHARS),
|
|
1225
|
+
"backend_version": _clip(redact(facts.backend_version), MAX_BACKEND_CHARS),
|
|
1226
|
+
"network": facts.network if facts.network in NETWORKS else "",
|
|
1227
|
+
"teardown": _clip(redact(facts.teardown), MAX_TEARDOWN_CHARS),
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
|
|
1231
|
+
def _escape(char: str) -> str:
|
|
1232
|
+
if char in "\n\t" or not _UNESCAPED_RE.match(char):
|
|
1233
|
+
return char
|
|
1234
|
+
return char.encode("unicode_escape").decode("ascii")
|
|
1235
|
+
|
|
1236
|
+
|
|
1237
|
+
def bounded_tail(text: str, captured_truncated: bool) -> tuple[str, bool]:
|
|
1238
|
+
"""D10.2 on one decoded stream: redact it whole, escape, cut, say whether it was cut.
|
|
1239
|
+
|
|
1240
|
+
``text`` is the executor's capture, already decoded with replacement
|
|
1241
|
+
characters under its byte caps. The cut keeps the longest suffix of at
|
|
1242
|
+
most :data:`MAX_TAIL_CHARS` escaped characters, so it never splits an
|
|
1243
|
+
escape. The flag is True when the capture omitted bytes or the cut did.
|
|
1244
|
+
"""
|
|
1245
|
+
redacted = redact(text)
|
|
1246
|
+
pieces: list[str] = []
|
|
1247
|
+
size = 0
|
|
1248
|
+
# Each character escapes to at least one, so the last MAX_TAIL_CHARS
|
|
1249
|
+
# characters always hold the suffix.
|
|
1250
|
+
for char in reversed(redacted[-MAX_TAIL_CHARS:]):
|
|
1251
|
+
piece = _escape(char)
|
|
1252
|
+
if size + len(piece) > MAX_TAIL_CHARS:
|
|
1253
|
+
break
|
|
1254
|
+
pieces.append(piece)
|
|
1255
|
+
size += len(piece)
|
|
1256
|
+
kept = len(pieces)
|
|
1257
|
+
return "".join(reversed(pieces)), captured_truncated or kept < len(redacted)
|
|
1258
|
+
|
|
1259
|
+
|
|
1260
|
+
def outcome_of(result: ExecutionResult) -> Outcome:
|
|
1261
|
+
"""D10.4: a timeout is TIMEOUT, exit 0 is PASS, anything else FAIL."""
|
|
1262
|
+
if result.timed_out:
|
|
1263
|
+
return Outcome.TIMEOUT
|
|
1264
|
+
return Outcome.PASS if result.exit_code == 0 else Outcome.FAIL
|
|
1265
|
+
|
|
1266
|
+
|
|
1267
|
+
def observed_record(
|
|
1268
|
+
*,
|
|
1269
|
+
seq: int,
|
|
1270
|
+
candidate: Candidate,
|
|
1271
|
+
definition: Definition,
|
|
1272
|
+
policy: Policy,
|
|
1273
|
+
result: ExecutionResult,
|
|
1274
|
+
run_log: str,
|
|
1275
|
+
end_fingerprint: str = "",
|
|
1276
|
+
) -> dict:
|
|
1277
|
+
"""The record of one launched definition (D10.1), digest included."""
|
|
1278
|
+
outcome = outcome_of(result)
|
|
1279
|
+
stdout_tail, stdout_cut = bounded_tail(result.stdout or "", result.stdout_truncated)
|
|
1280
|
+
stderr_tail, stderr_cut = bounded_tail(result.stderr or "", result.stderr_truncated)
|
|
1281
|
+
record: dict = {
|
|
1282
|
+
"schema": SCHEMA,
|
|
1283
|
+
"seq": seq,
|
|
1284
|
+
"candidate": candidate.to_dict(),
|
|
1285
|
+
"definition": definition.stored(),
|
|
1286
|
+
"policy": policy.stored(),
|
|
1287
|
+
"outcome": outcome.value,
|
|
1288
|
+
"run_log": run_log,
|
|
1289
|
+
"enforcement": enforcement_facts(result.sandbox),
|
|
1290
|
+
"exit_code": None if outcome is Outcome.TIMEOUT else result.exit_code,
|
|
1291
|
+
"started_at": result.started_at,
|
|
1292
|
+
"finished_at": result.finished_at,
|
|
1293
|
+
"leftovers": {
|
|
1294
|
+
"descendants_killed": bool(result.descendants_killed),
|
|
1295
|
+
"group_survived_kill": bool(result.group_survived_kill),
|
|
1296
|
+
"orphans_killed": bool(result.orphans_killed),
|
|
1297
|
+
"orphan_survived_kill": bool(result.orphan_survived_kill),
|
|
1298
|
+
"orphans_unchecked": bool(result.orphans_unchecked),
|
|
1299
|
+
},
|
|
1300
|
+
"output": {
|
|
1301
|
+
"stdout_tail": stdout_tail,
|
|
1302
|
+
"stderr_tail": stderr_tail,
|
|
1303
|
+
"stdout_bytes": min(max(result.stdout_bytes, 0), MAX_BYTE_COUNT),
|
|
1304
|
+
"stderr_bytes": min(max(result.stderr_bytes, 0), MAX_BYTE_COUNT),
|
|
1305
|
+
"stdout_tail_truncated": stdout_cut,
|
|
1306
|
+
"stderr_tail_truncated": stderr_cut,
|
|
1307
|
+
},
|
|
1308
|
+
}
|
|
1309
|
+
if isinstance(candidate, LocalCandidate):
|
|
1310
|
+
record["start_fingerprint"] = candidate.fingerprint
|
|
1311
|
+
record["end_fingerprint"] = end_fingerprint
|
|
1312
|
+
record["digest"] = record_digest(record)
|
|
1313
|
+
# A writer never produces a value the loader would refuse (D10.3): a
|
|
1314
|
+
# runner that reported a malformed time, say, fails here, before the save.
|
|
1315
|
+
parse_record(record, "the new validation record")
|
|
1316
|
+
return record
|
|
1317
|
+
|
|
1318
|
+
|
|
1319
|
+
def attempt_marker(
|
|
1320
|
+
*,
|
|
1321
|
+
candidate: Candidate,
|
|
1322
|
+
definition: Definition,
|
|
1323
|
+
policy: Policy,
|
|
1324
|
+
run_log: str,
|
|
1325
|
+
marked_at: str,
|
|
1326
|
+
) -> dict:
|
|
1327
|
+
"""The marker written immediately before a launch (D11.2)."""
|
|
1328
|
+
marker: dict = {
|
|
1329
|
+
"candidate": candidate.to_dict(),
|
|
1330
|
+
"definition": definition.stored(),
|
|
1331
|
+
"policy": policy.stored(),
|
|
1332
|
+
"run_log": run_log,
|
|
1333
|
+
"marked_at": marked_at,
|
|
1334
|
+
}
|
|
1335
|
+
if isinstance(candidate, LocalCandidate):
|
|
1336
|
+
marker["start_fingerprint"] = candidate.fingerprint
|
|
1337
|
+
parse_marker(marker, "the new validation attempt marker")
|
|
1338
|
+
return marker
|
|
1339
|
+
|
|
1340
|
+
|
|
1341
|
+
def interrupted_record(*, seq: int, marker: Mapping[str, object], recovered_at: str) -> dict:
|
|
1342
|
+
"""D10.7: built from the marker alone; claims no launch."""
|
|
1343
|
+
record: dict = {
|
|
1344
|
+
"schema": SCHEMA,
|
|
1345
|
+
"seq": seq,
|
|
1346
|
+
"candidate": marker["candidate"],
|
|
1347
|
+
"definition": marker["definition"],
|
|
1348
|
+
"policy": marker["policy"],
|
|
1349
|
+
"outcome": Outcome.INTERRUPTED.value,
|
|
1350
|
+
"run_log": marker["run_log"],
|
|
1351
|
+
"marked_at": marker["marked_at"],
|
|
1352
|
+
"recovered_at": recovered_at,
|
|
1353
|
+
}
|
|
1354
|
+
if "start_fingerprint" in marker:
|
|
1355
|
+
record["start_fingerprint"] = marker["start_fingerprint"]
|
|
1356
|
+
record["digest"] = record_digest(record)
|
|
1357
|
+
parse_record(record, "the new interrupted record")
|
|
1358
|
+
return record
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
def next_seq(records: Sequence[Mapping[str, object]]) -> int:
|
|
1362
|
+
"""The seq the next record takes: one above the highest retained."""
|
|
1363
|
+
highest = 0
|
|
1364
|
+
for record in records:
|
|
1365
|
+
seq = record.get("seq")
|
|
1366
|
+
if isinstance(seq, int) and seq > highest:
|
|
1367
|
+
highest = seq
|
|
1368
|
+
return highest + 1
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
def appended(records: Sequence[dict], record: dict, pinned: set[int]) -> list[dict]:
|
|
1372
|
+
"""``records`` with ``record`` appended, evicted oldest first to the bound (D11.4)."""
|
|
1373
|
+
out = [*records, record]
|
|
1374
|
+
while len(out) > MAX_RECORDS:
|
|
1375
|
+
for i, old in enumerate(out):
|
|
1376
|
+
if old.get("seq") not in pinned:
|
|
1377
|
+
del out[i]
|
|
1378
|
+
break
|
|
1379
|
+
else: # pragma: no cover - pinned: a binding, a certificate, the counter's
|
|
1380
|
+
# record and the set being run, at most 3 x 16 + 1
|
|
1381
|
+
break
|
|
1382
|
+
return out
|
|
1383
|
+
|
|
1384
|
+
|
|
1385
|
+
# -- readers ---------------------------------------------------------------------------
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
def applicable(record: RecordView, candidate: Candidate, definition: str, policy: Policy) -> bool:
|
|
1389
|
+
"""D8.3: an observed record for this candidate, definition and policy.
|
|
1390
|
+
|
|
1391
|
+
Equal candidate identity (never provenance), definition digest and
|
|
1392
|
+
policy digest; under ``required`` (or REMOTE's ``preferred``) the record
|
|
1393
|
+
must also show the launch enforced. An ``INTERRUPTED`` record never applies.
|
|
1394
|
+
"""
|
|
1395
|
+
return (
|
|
1396
|
+
record.is_observed
|
|
1397
|
+
and record.candidate.identity == candidate.identity
|
|
1398
|
+
and record.definition_digest == definition
|
|
1399
|
+
and record.policy_digest == policy.digest
|
|
1400
|
+
and (record.enforced or not policy.requires_enforcement)
|
|
1401
|
+
)
|
|
1402
|
+
|
|
1403
|
+
|
|
1404
|
+
def interruptions(
|
|
1405
|
+
records: Sequence[RecordView],
|
|
1406
|
+
identity: tuple[str, ...],
|
|
1407
|
+
definition: str,
|
|
1408
|
+
policy: str,
|
|
1409
|
+
watermark: int | None,
|
|
1410
|
+
) -> int:
|
|
1411
|
+
"""D10.7: the retained INTERRUPTED records of one key above the watermark."""
|
|
1412
|
+
floor = watermark or 0
|
|
1413
|
+
return sum(
|
|
1414
|
+
1
|
|
1415
|
+
for r in records
|
|
1416
|
+
if r.outcome is Outcome.INTERRUPTED
|
|
1417
|
+
and r.seq > floor
|
|
1418
|
+
and r.candidate.identity == identity
|
|
1419
|
+
and r.definition_digest == definition
|
|
1420
|
+
and r.policy_digest == policy
|
|
1421
|
+
)
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
def recover_attempt(
|
|
1425
|
+
records_raw: Sequence[dict],
|
|
1426
|
+
marker_raw: Mapping[str, object],
|
|
1427
|
+
*,
|
|
1428
|
+
watermark: int | None,
|
|
1429
|
+
pinned: set[int],
|
|
1430
|
+
recovered_at: str,
|
|
1431
|
+
) -> list[dict]:
|
|
1432
|
+
"""D10.7, D11.3: the records the resume path writes for a leftover marker.
|
|
1433
|
+
|
|
1434
|
+
The caller clears the marker in the same save. A record that matches the
|
|
1435
|
+
marker (its candidate identity, definition, policy and run log) means
|
|
1436
|
+
the marker is stale, a hand edit: the record wins, and the records come
|
|
1437
|
+
back unchanged. Otherwise an ``INTERRUPTED`` record is appended; more
|
|
1438
|
+
than :data:`MAX_INTERRUPTIONS` of them for the key (those at or below
|
|
1439
|
+
the watermark not counted) raise :class:`ValidationInterruptedError`
|
|
1440
|
+
carrying the records, which the caller saves in its ``BLOCKED`` write.
|
|
1441
|
+
"""
|
|
1442
|
+
marker = parse_marker(marker_raw)
|
|
1443
|
+
for view in (parse_record(r) for r in records_raw):
|
|
1444
|
+
if (
|
|
1445
|
+
view.candidate.identity == marker.candidate.identity
|
|
1446
|
+
and view.definition_digest == marker.definition_digest
|
|
1447
|
+
and view.policy_digest == marker.policy_digest
|
|
1448
|
+
and view.run_log == marker.run_log
|
|
1449
|
+
):
|
|
1450
|
+
return list(records_raw)
|
|
1451
|
+
record = interrupted_record(
|
|
1452
|
+
seq=next_seq(records_raw), marker=marker_raw, recovered_at=recovered_at
|
|
1453
|
+
)
|
|
1454
|
+
records = appended(list(records_raw), record, pinned)
|
|
1455
|
+
count = interruptions(
|
|
1456
|
+
[parse_record(r) for r in records],
|
|
1457
|
+
marker.candidate.identity,
|
|
1458
|
+
marker.definition_digest,
|
|
1459
|
+
marker.policy_digest,
|
|
1460
|
+
watermark,
|
|
1461
|
+
)
|
|
1462
|
+
if count > MAX_INTERRUPTIONS:
|
|
1463
|
+
raise ValidationInterruptedError(
|
|
1464
|
+
f"candidate validation of {marker.candidate.describe()} was interrupted {count} "
|
|
1465
|
+
f"times for definition [{marker.index}] {' '.join(marker.argv)!r} under policy "
|
|
1466
|
+
f"{marker.policy_digest[:16]}..., over the bound of {MAX_INTERRUPTIONS}, so the "
|
|
1467
|
+
"controller will not run it again unattended. Find what ended the validator (a "
|
|
1468
|
+
"crash, an interrupt, a kill), then resume",
|
|
1469
|
+
records,
|
|
1470
|
+
)
|
|
1471
|
+
return records
|
|
1472
|
+
|
|
1473
|
+
|
|
1474
|
+
def record_summary(view: RecordView) -> str:
|
|
1475
|
+
"""One display line for ``status``: redacted, bounded by the stored bounds."""
|
|
1476
|
+
argv = redact(" ".join(view.argv))
|
|
1477
|
+
if len(argv) > 200:
|
|
1478
|
+
argv = argv[:199] + "…"
|
|
1479
|
+
if view.outcome is Outcome.INTERRUPTED:
|
|
1480
|
+
detail = "interrupted (no launch claimed)"
|
|
1481
|
+
elif view.outcome is Outcome.TIMEOUT:
|
|
1482
|
+
detail = "timed out"
|
|
1483
|
+
else:
|
|
1484
|
+
detail = f"exit {view.exit_code}"
|
|
1485
|
+
enforced = ""
|
|
1486
|
+
if view.is_observed:
|
|
1487
|
+
enforced = ", sandbox enforced" if view.enforced else ", unsandboxed"
|
|
1488
|
+
return f"#{view.seq} [{view.index}] {argv!r}: {view.outcome.value} ({detail}{enforced})"
|
|
1489
|
+
|
|
1490
|
+
|
|
1491
|
+
@dataclass(frozen=True)
|
|
1492
|
+
class ValidationRun:
|
|
1493
|
+
"""What one call of the REMOTE entry point observed, in set order."""
|
|
1494
|
+
|
|
1495
|
+
candidate: Candidate
|
|
1496
|
+
validation_set_digest: str
|
|
1497
|
+
policy: Policy
|
|
1498
|
+
records: tuple[dict, ...]
|
|
1499
|
+
definitions: int
|
|
1500
|
+
|
|
1501
|
+
# The seqs of ``records`` this call reused rather than ran (#173).
|
|
1502
|
+
reused: tuple[int, ...] = ()
|
|
1503
|
+
|
|
1504
|
+
@property
|
|
1505
|
+
def passed(self) -> bool:
|
|
1506
|
+
"""Every definition of the set ran and passed (D10.6 stops at a non-PASS)."""
|
|
1507
|
+
return len(self.records) == self.definitions and all(
|
|
1508
|
+
r["outcome"] == Outcome.PASS.value for r in self.records
|
|
1509
|
+
)
|
|
1510
|
+
|
|
1511
|
+
def evidence(self) -> Evidence:
|
|
1512
|
+
return Evidence(
|
|
1513
|
+
candidate=self.candidate,
|
|
1514
|
+
validation_set_digest=self.validation_set_digest,
|
|
1515
|
+
policy=self.policy,
|
|
1516
|
+
records=tuple(parse_record(r) for r in self.records),
|
|
1517
|
+
definitions=self.definitions,
|
|
1518
|
+
)
|
|
1519
|
+
|
|
1520
|
+
|
|
1521
|
+
# -- eligibility, reuse and the binding (D5.5, D7.2, D8.3, D10.6; #173) ----------------
|
|
1522
|
+
|
|
1523
|
+
|
|
1524
|
+
def eligible(
|
|
1525
|
+
view: RecordView,
|
|
1526
|
+
candidate: Candidate,
|
|
1527
|
+
definition: Definition,
|
|
1528
|
+
policy: Policy,
|
|
1529
|
+
watermark: int | None,
|
|
1530
|
+
) -> bool:
|
|
1531
|
+
"""Whether ``view`` stands for ``definition`` of ``candidate`` under ``policy`` now.
|
|
1532
|
+
|
|
1533
|
+
The one eligibility rule every gate uses. The record is applicable
|
|
1534
|
+
(:func:`applicable`: the candidate key, the definition digest at its
|
|
1535
|
+
index, the policy digest, enforcement where the policy requires it; an
|
|
1536
|
+
``INTERRUPTED`` record never is), and:
|
|
1537
|
+
|
|
1538
|
+
- a ``PASS`` is evidence, and stays so across an override; in LOCAL only
|
|
1539
|
+
when the run ended where it started (D7.2): a validator that changed
|
|
1540
|
+
covered bytes passed on a tree that no longer exists;
|
|
1541
|
+
- a ``FAIL`` or ``TIMEOUT`` is terminal, reused rather than run again,
|
|
1542
|
+
unless its seq is at or below the operator's override watermark
|
|
1543
|
+
(D14.2): it is then historical and the definition runs again.
|
|
1544
|
+
"""
|
|
1545
|
+
if not applicable(view, candidate, definition.digest, policy):
|
|
1546
|
+
return False
|
|
1547
|
+
if view.outcome is Outcome.PASS:
|
|
1548
|
+
return view.candidate.kind != KIND_LOCAL or view.end_fingerprint == view.start_fingerprint
|
|
1549
|
+
return view.seq > (watermark or 0)
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
def eligible_record(
|
|
1553
|
+
records: Sequence[RecordView],
|
|
1554
|
+
candidate: Candidate,
|
|
1555
|
+
definition: Definition,
|
|
1556
|
+
policy: Policy,
|
|
1557
|
+
watermark: int | None,
|
|
1558
|
+
) -> RecordView | None:
|
|
1559
|
+
"""The newest :func:`eligible` record for ``definition``, or ``None``."""
|
|
1560
|
+
for view in reversed(records):
|
|
1561
|
+
if eligible(view, candidate, definition, policy, watermark):
|
|
1562
|
+
return view
|
|
1563
|
+
return None
|
|
1564
|
+
|
|
1565
|
+
|
|
1566
|
+
@dataclass(frozen=True)
|
|
1567
|
+
class Evidence:
|
|
1568
|
+
"""What the retained records say about one candidate, set and policy (D10.6).
|
|
1569
|
+
|
|
1570
|
+
``records`` are the eligible records in set order, stopping at the
|
|
1571
|
+
first non-PASS, as a run of the set would. The evidence is *complete*
|
|
1572
|
+
when every definition has a ``PASS``, *failed* when it stops at a
|
|
1573
|
+
``FAIL`` or ``TIMEOUT``, and otherwise has a definition no eligible
|
|
1574
|
+
record covers, which a gate runs (REMOTE) or refuses (LOCAL REVIEW).
|
|
1575
|
+
"""
|
|
1576
|
+
|
|
1577
|
+
candidate: Candidate
|
|
1578
|
+
validation_set_digest: str
|
|
1579
|
+
policy: Policy
|
|
1580
|
+
records: tuple[RecordView, ...]
|
|
1581
|
+
definitions: int
|
|
1582
|
+
|
|
1583
|
+
@property
|
|
1584
|
+
def complete(self) -> bool:
|
|
1585
|
+
return len(self.records) == self.definitions and all(
|
|
1586
|
+
r.outcome is Outcome.PASS for r in self.records
|
|
1587
|
+
)
|
|
1588
|
+
|
|
1589
|
+
@property
|
|
1590
|
+
def failed(self) -> RecordView | None:
|
|
1591
|
+
"""The ``FAIL`` or ``TIMEOUT`` the set stops at, or ``None``."""
|
|
1592
|
+
if self.records and self.records[-1].outcome is not Outcome.PASS:
|
|
1593
|
+
return self.records[-1]
|
|
1594
|
+
return None
|
|
1595
|
+
|
|
1596
|
+
@property
|
|
1597
|
+
def decided(self) -> bool:
|
|
1598
|
+
"""Complete or failed: nothing is left to run."""
|
|
1599
|
+
return self.complete or self.failed is not None
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
def find_evidence(
|
|
1603
|
+
records: Sequence[RecordView],
|
|
1604
|
+
candidate: Candidate,
|
|
1605
|
+
defs: Sequence[Definition],
|
|
1606
|
+
policy: Policy,
|
|
1607
|
+
watermark: int | None,
|
|
1608
|
+
) -> Evidence:
|
|
1609
|
+
"""The candidate's evidence for ``defs`` from ``records`` alone; runs nothing."""
|
|
1610
|
+
found: list[RecordView] = []
|
|
1611
|
+
for definition in defs:
|
|
1612
|
+
view = eligible_record(records, candidate, definition, policy, watermark)
|
|
1613
|
+
if view is None:
|
|
1614
|
+
break
|
|
1615
|
+
found.append(view)
|
|
1616
|
+
if view.outcome is not Outcome.PASS:
|
|
1617
|
+
break
|
|
1618
|
+
return Evidence(
|
|
1619
|
+
candidate=candidate,
|
|
1620
|
+
validation_set_digest=validation_set_digest(defs),
|
|
1621
|
+
policy=policy,
|
|
1622
|
+
records=tuple(found),
|
|
1623
|
+
definitions=len(defs),
|
|
1624
|
+
)
|
|
1625
|
+
|
|
1626
|
+
|
|
1627
|
+
def binding(review_round: int, evidence: Evidence) -> dict:
|
|
1628
|
+
"""The ``review_evidence`` value naming ``evidence`` for ``review_round`` (D11.2)."""
|
|
1629
|
+
if not evidence.decided:
|
|
1630
|
+
raise ValueError("a binding names complete or failed evidence only")
|
|
1631
|
+
return {
|
|
1632
|
+
"review_round": review_round,
|
|
1633
|
+
"candidate": evidence.candidate.to_dict(),
|
|
1634
|
+
"records": [{"seq": r.seq, "digest": r.digest} for r in evidence.records],
|
|
1635
|
+
"validation_set_digest": evidence.validation_set_digest,
|
|
1636
|
+
"policy_digest": evidence.policy.digest,
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
|
|
1640
|
+
def bound_records(raw: Mapping[str, object], records: Sequence[RecordView]) -> list[RecordView]:
|
|
1641
|
+
"""The records a loaded binding names, in its order (:func:`check_evidence` proved them)."""
|
|
1642
|
+
by_seq = {r.seq: r for r in records}
|
|
1643
|
+
pairs = raw.get("records")
|
|
1644
|
+
named: list[RecordView] = []
|
|
1645
|
+
for pair in pairs if isinstance(pairs, list) else ():
|
|
1646
|
+
view = by_seq.get(pair["seq"])
|
|
1647
|
+
if view is None or view.digest != pair["digest"]:
|
|
1648
|
+
raise StateError(
|
|
1649
|
+
f"state field 'review_evidence' names record seq {pair['seq']}, "
|
|
1650
|
+
"which is not retained under that digest"
|
|
1651
|
+
)
|
|
1652
|
+
named.append(view)
|
|
1653
|
+
return named
|
|
1654
|
+
|
|
1655
|
+
|
|
1656
|
+
def binding_reusable(
|
|
1657
|
+
raw: Mapping[str, object],
|
|
1658
|
+
records: Sequence[RecordView],
|
|
1659
|
+
*,
|
|
1660
|
+
review_round: int,
|
|
1661
|
+
candidate: Candidate,
|
|
1662
|
+
validation_set_digest: str,
|
|
1663
|
+
policy_digest: str,
|
|
1664
|
+
watermark: int | None,
|
|
1665
|
+
) -> bool:
|
|
1666
|
+
"""D5.5, D5.7: whether a persisted binding is evidence for this launch, unchanged.
|
|
1667
|
+
|
|
1668
|
+
Its round is the upcoming one, its candidate is the one just bound (the
|
|
1669
|
+
PR head's commit in REMOTE, the bound fingerprint's identity in LOCAL),
|
|
1670
|
+
its set and policy digests are the ones in force, and no record it
|
|
1671
|
+
names is a ``FAIL`` or ``TIMEOUT`` the operator's override made
|
|
1672
|
+
historical.
|
|
1673
|
+
"""
|
|
1674
|
+
if not raw:
|
|
1675
|
+
return False
|
|
1676
|
+
bound = parse_candidate(raw.get("candidate"), "state field 'review_evidence'.candidate")
|
|
1677
|
+
if isinstance(candidate, RemoteCandidate):
|
|
1678
|
+
if not isinstance(bound, RemoteCandidate) or bound.commit != candidate.commit:
|
|
1679
|
+
return False
|
|
1680
|
+
elif bound.identity != candidate.identity:
|
|
1681
|
+
return False
|
|
1682
|
+
if (
|
|
1683
|
+
raw.get("review_round") != review_round
|
|
1684
|
+
or raw.get("validation_set_digest") != validation_set_digest
|
|
1685
|
+
or raw.get("policy_digest") != policy_digest
|
|
1686
|
+
):
|
|
1687
|
+
return False
|
|
1688
|
+
floor = watermark or 0
|
|
1689
|
+
return not any(
|
|
1690
|
+
r.outcome is not Outcome.PASS and r.seq <= floor for r in bound_records(raw, records)
|
|
1691
|
+
)
|
|
1692
|
+
|
|
1693
|
+
|
|
1694
|
+
def failure_counter(phase: str, previous: Mapping[str, object], record: RecordView) -> dict:
|
|
1695
|
+
"""``producer_validation_failure`` after one more failure of ``phase`` (D5.2).
|
|
1696
|
+
|
|
1697
|
+
Counted per phase entry, whichever candidate failed: the count goes on
|
|
1698
|
+
from the phase's own counter, and starts at 1 otherwise. The bound is
|
|
1699
|
+
the caller's (:data:`MAX_FAILURE_COUNT`), and the count never passes it.
|
|
1700
|
+
"""
|
|
1701
|
+
candidate = record.candidate
|
|
1702
|
+
if not isinstance(candidate, RemoteCandidate) or record.outcome not in (
|
|
1703
|
+
Outcome.FAIL,
|
|
1704
|
+
Outcome.TIMEOUT,
|
|
1705
|
+
):
|
|
1706
|
+
raise ValueError("the counter names a REMOTE FAIL or TIMEOUT record")
|
|
1707
|
+
count = previous.get("count") if previous.get("phase") == phase else 0
|
|
1708
|
+
return {
|
|
1709
|
+
"phase": phase,
|
|
1710
|
+
"count": min((count if isinstance(count, int) else 0) + 1, MAX_FAILURE_COUNT),
|
|
1711
|
+
"candidate_commit": candidate.commit,
|
|
1712
|
+
"last_record_seq": record.seq,
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
|
|
1716
|
+
def duration_seconds(view: RecordView) -> float | None:
|
|
1717
|
+
"""How long an observed record's launch ran, from its own times; ``None`` for none."""
|
|
1718
|
+
if not view.is_observed:
|
|
1719
|
+
return None
|
|
1720
|
+
started = datetime.fromisoformat(view.raw["started_at"])
|
|
1721
|
+
finished = datetime.fromisoformat(view.raw["finished_at"])
|
|
1722
|
+
return max((finished - started).total_seconds(), 0.0)
|